IFRS: prototype migration framework and object_size migration (GSI-1271)

services/ifrs/README.md

@@ -272,6 +272,36 @@ The service requires the following configuration parameters:
   ```
 
 
+- **`db_version_collection`** *(string, required)*: The name of the collection containing DB version information for this service.
+
+
+  Examples:
+
+  ```json
+  "ifrsDbVersions"
+  ```
+
+
+- **`migration_wait_sec`** *(integer, required)*: The number of seconds to wait before checking the DB version again.
+
+
+  Examples:
+
+  ```json
+  5
+  ```
+
+
+  ```json
+  30
+  ```
+
+
+  ```json
+  180
+  ```
+
+
 ## Definitions
 
 

services/ifrs/config_schema.json

@@ -313,6 +313,24 @@
       ],
       "title": "Db Name",
       "type": "string"
+    },
+    "db_version_collection": {
+      "description": "The name of the collection containing DB version information for this service",
+      "examples": [
+        "ifrsDbVersions"
+      ],
+      "title": "Db Version Collection",
+      "type": "string"
+    },
+    "migration_wait_sec": {
+      "description": "The number of seconds to wait before checking the DB version again",
+      "examples": [
+        5,
+        30,
+        180
+      ],
+      "title": "Migration Wait Sec",
+      "type": "integer"
     }
   },
   "required": [
@@ -329,7 +347,9 @@
     "files_to_stage_topic",
     "kafka_servers",
     "db_connection_str",
-    "db_name"
+    "db_name",
+    "db_version_collection",
+    "migration_wait_sec"
   ],
   "title": "ModSettings",
   "type": "object"

services/ifrs/dev_config.yaml

@@ -18,3 +18,5 @@ file_staged_event_topic: internal-file-registry
 file_staged_event_type: file_staged_for_download
 file_deleted_event_topic: internal-file-registry
 file_deleted_event_type: file_deleted
+db_version_collection: ifrsDbVersions
+migration_wait_sec: 10

services/ifrs/example_config.yaml

@@ -1,5 +1,6 @@
 db_connection_str: '**********'
 db_name: dev
+db_version_collection: ifrsDbVersions
 file_deleted_event_topic: internal-file-registry
 file_deleted_event_type: file_deleted
 file_registered_event_topic: internal-file-registry
@@ -21,6 +22,7 @@ kafka_ssl_password: ''
 log_format: null
 log_level: INFO
 log_traceback: true
+migration_wait_sec: 10
 object_storages:
   test:
     bucket: permanent

services/ifrs/src/ifrs/config.py

@@ -19,15 +19,15 @@
 from hexkit.config import config_from_yaml
 from hexkit.log import LoggingConfig
 from hexkit.providers.akafka import KafkaConfig
-from hexkit.providers.mongodb import MongoDbConfig
 
 from ifrs.adapters.inbound.event_sub import OutboxSubTranslatorConfig
 from ifrs.adapters.outbound.event_pub import EventPubTranslatorConfig
+from ifrs.migration_logic import MigrationConfig
 
 
 @config_from_yaml(prefix="ifrs")
 class Config(
-    MongoDbConfig,
+    MigrationConfig,
     KafkaConfig,
     OutboxSubTranslatorConfig,
     EventPubTranslatorConfig,

services/ifrs/src/ifrs/main.py

@@ -19,12 +19,14 @@
 
 from ifrs.config import Config
 from ifrs.inject import prepare_outbox_subscriber
+from ifrs.migrations import run_db_migrations
 
 
 async def consume_events(run_forever: bool = True):
     """Run an event consumer listening to the specified topics."""
     config = Config()
     configure_logging(config=config)
+    await run_db_migrations(config=config)
 
     async with prepare_outbox_subscriber(config=config) as outbox_subscriber:
         await outbox_subscriber.run(forever=run_forever)

services/ifrs/src/ifrs/migration_logic/__init__.py

@@ -0,0 +1,34 @@
+# Copyright 2021 - 2024 Universität Tübingen, DKFZ, EMBL, and Universität zu Köln
+# for the German Human Genome-Phenome Archive (GHGA)
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Temporary database migration logic, some of which will move to a library"""
+
+from ._manager import (
+    MigrationConfig,
+    MigrationManager,
+    MigrationMap,
+    MigrationStepError,
+)
+from ._utils import Document, MigrationDefinition, Reversible, validate_doc
+
+__all__ = [
+    "Document",
+    "MigrationConfig",
+    "MigrationDefinition",
+    "MigrationManager",
+    "MigrationMap",
+    "MigrationStepError",
+    "Reversible",
+    "validate_doc",
+]