Service layout changes + dep updates

.devcontainer/Dockerfile

@@ -14,6 +14,6 @@ RUN if [ "${INSTALL_NODE}" = "true" ]; then su vscode -c "umask 0002 && . /usr/l
 
 # Copy install and launcher script to bin:
 COPY ./dev_install /bin
-COPY ./supdate /bin
+COPY ./update_service_files /bin
 
 CMD ["sleep", "infinity"]

.devcontainer/dev_install

@@ -6,10 +6,10 @@ cd /workspace
 # upgrade pip
 python -m pip install --upgrade pip
 
-# install or upgrade dependencies for development and testing
+# install global depencies for all services
 pip install --no-deps -r ./lock/requirements-dev.txt
 
-# install the package itself in edit mode:
+# install services as packages in edit mode:
 for dir in ./services/*; do
     pip install --no-deps -e $dir
 done

.devcontainer/devcontainer.json

@@ -1,6 +1,5 @@
 // For format details, see https://aka.ms/devcontainer.json. For config options, see the README at:
 // https://github.com/microsoft/vscode-dev-containers/tree/v0.177.0/containers/python-3-postgres
-// Update the VARIANT arg in docker-compose.yml to pick a Python version: 3, 3.8, 3.7, 3.6
 {
 	"name": "${localWorkspaceFolderBasename}",
 	"dockerComposeFile": "docker-compose.yml",
@@ -20,13 +19,10 @@
 				"python.languageServer": "Pylance",
 				"python.analysis.typeCheckingMode": "basic",
 				"python.testing.pytestPath": "/usr/local/py-utils/bin/pytest",
-				"python.testing.pytestArgs": [
-					"--profile"
-				],
 				"python.testing.unittestEnabled": false,
 				"python.testing.pytestEnabled": true,
 				"editor.codeActionsOnSave": {
-					"source.organizeImports": true
+					"source.organizeImports": "explicit"
 				},
 				"editor.formatOnSave": true,
 				"editor.renderWhitespace": "all",
@@ -60,8 +56,6 @@
 			]
 		}
 	},
-	// Use 'forwardPorts' to make a list of ports inside the container available locally.
-	// "forwardPorts": [5000, 5432],
 	// Use 'postCreateCommand' to run commands after the container is created.
 	"postCreateCommand": "dev_install",
 	// Comment out connect as root instead. More info: https://aka.ms/vscode-remote/containers/non-root.

.devcontainer/docker-compose.yml

@@ -6,9 +6,6 @@ services:
       context: .
       dockerfile: ./Dockerfile
       args:
-        # [Choice] Python version: 3, 3.8, 3.7, 3.6
-        VARIANT: 3.9
-        # [Choice] Install Node.js
         INSTALL_NODE: "true"
         NODE_VERSION: "lts/*"
         PACKAGE_NAME: "file-services-backend"
@@ -17,21 +14,21 @@ services:
         USER_GID: 1000
 
     init: true
-    privileged: true
+
+    # Makes testcontainers work on linux based hosts
+    extra_hosts:
+      - host.docker.internal:host-gateway
 
     volumes:
       - ..:/workspace:cached
 
     # Overrides default command so things don't shut down after the process ends.
     command: sleep infinity
-
     # Uncomment the next line to use a non-root user for all processes.
     user: vscode
 
     # define environment variables
     environment:
-      IFRS_CONFIG_YAML: /workspace/services/ifrs/.dev/.dev_config.yaml
-      IRS_CONFIG_YAML: /workspace/services/irs/.dev/.dev_config.yaml
-      PCS_CONFIG_YAML: /workspace/services/pcs/.dev/.dev_config.yaml
-    # Use "forwardPorts" in **devcontainer.json** to forward an app port locally.
-    # (Adding the "ports" property to this file will not forward from a Codespace.)
+      IFRS_CONFIG_YAML: /workspace/services/ifrs/dev_config.yaml
+      IRS_CONFIG_YAML: /workspace/services/irs/dev_config.yaml
+      PCS_CONFIG_YAML: /workspace/services/pcs/dev_config.yaml

.devcontainer/update_service_files

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+python3 ./scripts/update_service_files.py "$@"

.github/workflows/monorepo_tests.yaml

@@ -0,0 +1,55 @@
+name: Monorepo Tests
+
+on: push
+
+jobs:
+  check-changes:
+    runs-on: ubuntu-latest
+    outputs:
+      services: ${{ steps.services-changed.outputs.affected }}
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          filter: tree:0
+          fetch-depth: 0
+
+      - name: Install Typer to check changed services
+        id: install-typer
+        run: pip install typer>=0.9.0
+
+      - name: Generate list of changed services
+        id: services-changed
+        run: |
+          echo "affected=$(python3 ./scripts/get_affected_services.py)" >> $GITHUB_OUTPUT
+
+  test:
+    if: ${{needs.check-changes.outputs.services}} != "[]"
+    needs: check-changes
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        service: ${{ fromJson(needs.check-changes.outputs.services) }}
+      fail-fast: false
+    env:
+      IFRS_CONFIG_YAML: ./services/ifrs/dev_config.yaml
+      IRS_CONFIG_YAML: ./services/irs/dev_config.yaml
+      PCS_CONFIG_YAML: ./services/pcs/dev_config.yaml
+    steps:
+      - name: Checkout ${{matrix.service}}
+        uses: actions/checkout@v4
+        with:
+          filter: tree:0
+          fetch-depth: 0
+
+      - name: Update pip and pyopenssl
+        id: pip-update
+        run: python -m pip install --upgrade pip && pip install --upgrade pyopenssl
+
+      - name: Install global and local dependencies for ${{matrix.service}}
+        id: install-dependencies
+        run: pip install --no-deps -r ./lock/requirements-dev.txt && pip install --no-deps ./services/${{matrix.service}}
+
+      - name: Run tests for ${{matrix.service}}
+        id: run-tests
+        run: pytest ./services/${{matrix.service}}

.pre-commit-config.yaml

@@ -48,13 +48,13 @@ repos:
       - id: no-commit-to-branch
         args: [--branch, dev, --branch, int, --branch, main]
   - repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.5
+    rev: v0.4.2
     hooks:
       - id: ruff
         args: [--fix, --exit-non-zero-on-fix]
       - id: ruff-format
   - repo: https://github.com/pre-commit/mirrors-mypy
-    rev: v1.9.0
+    rev: v1.10.0
     hooks:
       - id: mypy
         args: [--no-warn-unused-ignores]

.pyproject_generation/pyproject_custom.toml

@@ -3,10 +3,10 @@ name = "fsb"
 version = "0.1.0"
 description = "File Services Backend - monorepo housing file services"
 dependencies = [
-    "typer >= 0.9.0",
-    "ghga-service-commons[api] >= 3.0.0, <4",
-    "ghga-event-schemas >= 3.0.0, <4",
-    "hexkit[akafka,s3,mongodb] >= 2, <3",
+    "typer >= 0.12",
+    "ghga-service-commons[api] >= 3.1",
+    "ghga-event-schemas >= 3.1",
+    "hexkit[akafka,s3,mongodb] >= 3.0",
     "crypt4gh>=1.6",
 ]
 

.readme_generation/design.md

@@ -2,6 +2,6 @@
 Mention anything that deviates from the standard triple hexagonal architecture and
 the corresponding structure. -->
 
-This is a Python-based service following the Triple Hexagonal Architecture pattern.
+This is monorepo containing Python-based services following the Triple Hexagonal Architecture pattern.
 It uses protocol/provider pairs and dependency injection mechanisms provided by the
 [hexkit](https://github.com/ghga-de/hexkit) library.

lock/requirements-dev-template.in

@@ -3,32 +3,31 @@
 pytest>=7.4.0
 pytest-asyncio>=0.23.0
 pytest-cov>=4.1.0
-pytest-profiling>=1.7.0
-snakeviz>=2.2.0
-logot>=1.2.0
+snakeviz>=2.2
+logot>=1.3
 
-pre-commit>=3.6.0
+pre-commit>=3.7
 
-mypy>=1.8.0
-mypy-extensions>=1.0.0
+mypy>=1.10
+mypy-extensions>=1.0
 
-ruff>=0.3.0
+ruff>=0.4
 
-click>=8.1.0
-typer>=0.9.0
+click>=8.1
+typer>=0.12
 
-httpx>=0.27.0
-pytest-httpx>=0.30.0
+httpx>=0.27
+pytest-httpx>=0.30
 
 urllib3>=1.26.18
-requests>=2.31.0
+requests>=2.31
 
-stringcase>=1.2.0
-jsonschema2md>=1.0.0
-setuptools>=69.1.0
+stringcase>=1.2
+jsonschema2md>=1.1
+setuptools>=69.5
 
 # required since switch to pyproject.toml and pip-tools
 tomli>=2.0.1
 tomli_w>=1.0.0
 
-uv>=0.1.21
+uv>=0.1.39