Migrate to microservice repository template (#32)

* Apply microservice repository template

* JSON auto fixes

* Python auto fixes

* Python manual fixes

* Python manual fixes (mypy)

* Migrate README

* Re-run README generation

* Remove OpenAPI template files

* Remove debug folder

* Remove debug folder

* Fix license headers

* Template update (license checker)

* Fix badge URL in readme template

* Re-integrate custom config docs generator

* Re-add PyPI publish workflow

* Clean up devcontainer compose file

* Apply suggestions from code review

Co-authored-by: Kersten Breuer <kersten-breuer@outlook.com>

* Remove dev_launcher

* Remove README generation

* Remove config schema

* Ignore removed template files

---------

Co-authored-by: Kersten Breuer <kersten-breuer@outlook.com>

.coveragerc

@@ -0,0 +1,5 @@
+[paths]
+source =
+  src
+  /workspace/src
+  **/lib/python*/site-packages

.deprecated_files

@@ -0,0 +1,24 @@
+# List of all deprecated files and directories that
+# should not exist any more in this repo.
+# This list is similar to the `./mandatory_files`,
+# however, the entries here will be removed if they
+# still exist.
+
+.devcontainer/library-scripts/docker-in-docker-debian.sh
+.devcontainer/library-scripts
+
+.github/workflows/check_mandatory_and_static_files.yaml
+.github/workflows/dev_cd.yaml
+.github/workflows/unit_and_int_tests.yaml
+
+scripts/check_mandatory_and_static_files.py
+scripts/update_static_files.py
+
+docs
+
+setup.py
+setup.cfg
+
+.pylintrc
+.flake8
+.editorconfig

.deprecated_files_ignore

@@ -0,0 +1,2 @@
+# Optional list of files which are actually deprecated in the template
+# but are still allowed to be used in the current repository

.devcontainer/Dockerfile

@@ -1,12 +1,7 @@
-# [Choice] Python version: 3, 3.8, 3.7, 3.6
-ARG VARIANT=3.9
-FROM mcr.microsoft.com/vscode/devcontainers/python:0-${VARIANT}
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.9-bullseye
 
 ENV PYTHONUNBUFFERED 1
 
-# Name of the python package, overwitten in docker-compose.yaml
-ARG USER_UID=data_steward_scripts
-
 # Update args in docker-compose.yaml to set the UID/GID of the "vscode" user.
 ARG USER_UID=1000
 ARG USER_GID=$USER_UID
@@ -17,20 +12,8 @@ ARG INSTALL_NODE="true"
 ARG NODE_VERSION="lts/*"
 RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/local/share/nvm/nvm.sh && nvm install ${NODE_VERSION} 2>&1"; fi
 
-# Install additional OS packages.
-RUN apt update \
-    && export DEBIAN_FRONTEND=noninteractive \
-    && apt -y install --no-install-recommends postgresql-client-common \
-    && apt -y install --fix-missing libcurl4-openssl-dev libssl-dev
-
 # Copy install and launcher script to bin:
 COPY ./dev_install /bin
+COPY ./dev_launcher /bin
 
-# make docker in docker work:
-# (https://github.com/microsoft/vscode-dev-containers/blob/main/script-library/docs/docker-in-docker.md#script-use)
-COPY library-scripts/*.sh /tmp/library-scripts/
-ENV DOCKER_BUILDKIT=1
-RUN apt-get update && /bin/bash /tmp/library-scripts/docker-in-docker-debian.sh
-ENTRYPOINT ["/usr/local/share/docker-init.sh"]
-VOLUME [ "/var/lib/docker" ]
 CMD ["sleep", "infinity"]

.devcontainer/dev_install

@@ -6,11 +6,11 @@ cd /workspace
 # upgrade pip
 python -m pip install --upgrade pip
 
-# install with all extras in editable mode
-pip install -e .[all]
-
 # install or upgrade dependencies for development and testing
-pip install -r requirements-dev.txt
+pip install --no-deps -r requirements-dev.txt
+
+# install the package itself in edit mode:
+pip install --no-deps -e .
 
 # install pre-commit hooks to git
 pre-commit install

.devcontainer/devcontainer.json

@@ -2,54 +2,45 @@
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/python-3-postgres
 // Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6
 {
-	"name": "Data Steward Kit",
+	"name": "${localWorkspaceFolderBasename}",
 	"dockerComposeFile": "docker-compose.yml",
 	"service": "app",
 	"workspaceFolder": "/workspace",
 	"customizations": {
 		"vscode": {
 			// Set *default* container specific settings.json values on container create.
 			"settings": {
+				"files.eol": "\n",
 				"terminal.integrated.profiles.linux": {
 					"bash": {
 						"path": "/bin/bash"
 					}
 				},
 				"python.pythonPath": "/usr/local/bin/python",
 				"python.languageServer": "Pylance",
-				"python.linting.enabled": true,
-				"python.linting.pylintEnabled": true,
-				"python.formatting.autopep8Path": "/usr/local/py-utils/bin/autopep8",
-				"python.formatting.blackPath": "/usr/local/py-utils/bin/black",
-				"python.formatting.yapfPath": "/usr/local/py-utils/bin/yapf",
-				"python.formatting.provider": "black",
 				"python.analysis.typeCheckingMode": "basic",
-				"python.linting.banditPath": "/usr/local/py-utils/bin/bandit",
-				"python.linting.mypyPath": "/usr/local/py-utils/bin/mypy",
-				"python.linting.pycodestylePath": "/usr/local/py-utils/bin/pycodestyle",
-				"python.linting.pydocstylePath": "/usr/local/py-utils/bin/pydocstyle",
-				"python.linting.pylintPath": "/usr/local/py-utils/bin/pylint",
 				"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest",
+				"python.testing.pytestArgs": [
+					"--profile"
+				],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true,
 				"editor.formatOnSave": true,
 				"editor.renderWhitespace": "all",
 				"editor.rulers": [
 					88
 				],
+				"editor.defaultFormatter": "ms-python.black-formatter",
 				"licenser.license": "Custom",
 				"licenser.customHeaderFile": "/workspace/.devcontainer/license_header.txt"
 			},
 			// Add the IDs of extensions you want installed when the container is created.
 			"extensions": [
 				"mikestead.dotenv",
 				"ms-azuretools.vscode-docker",
-				"ms-python.black-formatter",
 				"ms-python.python",
-				"ms-python.isort",
 				"ms-python.vscode-pylance",
 				"ms-toolsai.jupyter",
-				"vtenentes.bdd",
 				"njpwerner.autodocstring",
 				"redhat.vscode-yaml",
 				"42crunch.vscode-openapi",
@@ -60,7 +51,9 @@
 				"yzhang.markdown-all-in-one",
 				"visualstudioexptteam.vscodeintellicode",
 				"ymotongpoo.licenser",
-				"editorconfig.editorconfig"
+				"charliermarsh.ruff",
+				"ms-python.black-formatter",
+				"ms-python.mypy-type-checker"
 			]
 		}
 	},
@@ -70,12 +63,13 @@
 	"postCreateCommand": "dev_install",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.
 	"remoteUser": "vscode",
+	"containerEnv": {
+		// for testcontainers to connect to the docker host:
+		"TC_HOST": "host.docker.internal",
+		"DOCKER_HOST": "unix:///var/run/docker.sock"
+	},
 	"features": {
-		"ghcr.io/devcontainers/features/docker-in-docker:2": {
-			"version": "latest",
-			"enableNonRootDocker": "true",
-			"moby": true,
-			"azureDnsAutoDetection": false
-		}
+		// details can be found here: https://github.com/devcontainers/features/tree/main/src/docker-outside-of-docker
+		"ghcr.io/devcontainers/features/docker-outside-of-docker:1": {}
 	}
 }

.devcontainer/docker-compose.yml

@@ -11,6 +11,8 @@ services:
         # [Choice] Install Node.js
         INSTALL_NODE: "true"
         NODE_VERSION: "lts/*"
+        # Please adapt to package name:
+        PACKAGE_NAME: "ghga_datasteward_kit"
         # On Linux, you may need to update USER_UID and USER_GID below if not your local UID is not 1000.
         USER_UID: 1000
         USER_GID: 1000
@@ -24,11 +26,8 @@ services:
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity
 
+    # Uncomment the next line to use a non-root user for all processes.
     user: vscode
-    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
-    # (Adding the "ports" property to this file will not forward from a Codespace.)
-
-
 
   localstack:
     image: localstack/localstack
@@ -47,16 +46,7 @@ services:
         target: /var/lib/localstack
         volume:
           nocopy: true
-    ports:
-      - "4566:4566"
-    expose:
-      - "4566"
-
-  mongodb:
-    image: mongo:5.0.4
-    restart: unless-stopped
-    volumes:
-      - mongo_fs:/data/db
+    # useful ports: 4566 - AWS API
 
 volumes:
   s3_fs: {}