[DOP-22129] Allow using etl-entities 2.5.0

MobileTeleSystems · Jan 27, 2025 · e05a326 · e05a326
1 parent 6143423
commit e05a326
Show file tree

Hide file tree

Showing 4 changed files with 118 additions and 80 deletions.
diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml
@@ -25,7 +25,7 @@ jobs:
         include:
           - python-version: '3.7'
             pydantic-version: '1'
-            os: ubuntu-latest
+            os: ubuntu-22.04
 
           - python-version: '3.13'
             pydantic-version: '2'

diff --git a/conftest.py b/conftest.py
@@ -1,8 +1,11 @@
 import os
 import secrets
 from collections import namedtuple
-from datetime import date, datetime
+from datetime import date, datetime, timezone
+from pathlib import Path
+from unittest.mock import Mock
 
+import etl_entities
 import pytest
 from etl_entities.hwm import (
     ColumnDateHWM,
@@ -13,6 +16,7 @@
 )
 from horizon.client.auth import LoginPassword
 from horizon.commons.schemas.v1 import NamespaceCreateRequestV1
+from packaging.version import Version
 
 from horizon_hwm_store import HorizonHWMStore
 
@@ -25,93 +29,126 @@
 HORIZON_PASSWORD = os.environ.get("HORIZON_PASSWORD")
 HORIZON_NAMESPACE = os.environ.get("HORIZON_NAMESPACE")
 
-
-@pytest.fixture(
-    params=[
-        (
-            ColumnIntHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                # no source
-                expression=secrets.token_hex(5),
-                value=10,
-            ),
-            15,
+HWMS_WITH_VALUE = [
+    (
+        ColumnIntHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",  # noqa: WPS204
+            # no source
+            expression=secrets.token_hex(5),
+            value=10,
         ),
-        (
-            ColumnIntHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                source=secrets.token_hex(5),
-                expression=secrets.token_hex(5),
-                value=10,
-            ),
-            15,
+        15,
+    ),
+    (
+        ColumnIntHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            source=secrets.token_hex(5),
+            expression=secrets.token_hex(5),
+            value=10,
         ),
-        (
-            ColumnDateHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                source=secrets.token_hex(5),
-                expression=secrets.token_hex(5),
-                value=date(year=2023, month=8, day=15),
-            ),
-            date(year=2023, month=9, day=15),  # + 1 month
+        15,
+    ),
+    (
+        ColumnDateHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            source=secrets.token_hex(5),
+            expression=secrets.token_hex(5),
+            value=date(year=2023, month=8, day=15),
         ),
-        (
-            ColumnDateTimeHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                source=secrets.token_hex(5),
-                expression=secrets.token_hex(5),
-                value=datetime(year=2023, month=8, day=15, hour=11, minute=22, second=33),
-            ),
-            datetime(year=2023, month=8, day=15, hour=11, minute=23, second=33),  # + 1 minute
+        date(year=2023, month=9, day=15),  # + 1 month
+    ),
+    (
+        ColumnDateTimeHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            source=secrets.token_hex(5),
+            expression=secrets.token_hex(5),
+            value=datetime(year=2023, month=8, day=15, hour=11, minute=22, second=33),
         ),
-        (
-            FileListHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                # no directory
-                value=["/some/file1", "/another/file2"],
-            ),
-            ["/some/file1", "/another/file2", "/more/file3"],
+        datetime(year=2023, month=8, day=15, hour=11, minute=23, second=33),  # + 1 minute
+    ),
+    (
+        FileListHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            # no directory
+            value=["/some/file1", "/another/file2"],
         ),
-        (
-            FileListHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                directory="/absolute/path",
-                value=["/absolute/path/file1", "/absolute/path/file2"],
-            ),
-            ["/absolute/path/file1", "/absolute/path/file2", "/absolute/path/file3"],
+        ["/some/file1", "/another/file2", "/more/file3"],
+    ),
+    (
+        FileListHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            directory="/absolute/path",
+            value=["/absolute/path/file1", "/absolute/path/file2"],
         ),
-        (
-            KeyValueIntHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                # no topic
-                expression="offset",
-                value={
-                    0: 100,
-                    1: 123,
-                },
-            ),
-            {
-                0: 110,
-                1: 150,
+        ["/absolute/path/file1", "/absolute/path/file2", "/absolute/path/file3"],
+    ),
+    (
+        KeyValueIntHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            # no topic
+            expression="offset",
+            value={
+                0: 100,
+                1: 123,
             },
         ),
-        (
-            KeyValueIntHWM(
-                name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
-                topic="topic_name",
-                expression="offset",
-                value={
-                    0: 100,
-                    1: 123,
-                },
-            ),
-            {
-                0: 110,
-                1: 150,
+        {
+            0: 110,
+            1: 150,
+        },
+    ),
+    (
+        KeyValueIntHWM(
+            name=f"{secrets.token_hex(5)}.{secrets.token_hex(5)}",
+            topic="topic_name",
+            expression="offset",
+            value={
+                0: 100,
+                1: 123,
             },
         ),
-    ],
-)
+        {
+            0: 110,
+            1: 150,
+        },
+    ),
+]
+
+if Version(etl_entities.__version__) >= Version("2.5.0"):
+    from etl_entities.hwm import FileModifiedTimeHWM
+
+    def file_with_mtime(mtime: datetime) -> Path:
+        result = Mock(spec=Path)
+        result.exists.return_value = True
+        result.is_file.return_value = True
+        result_stat = Mock(spec=os.stat_result)
+        result_stat.st_mtime = mtime.timestamp()
+        result.stat.return_value = result_stat
+        return result
+
+    HWMS_WITH_VALUE.extend(
+        [
+            (
+                FileModifiedTimeHWM(
+                    name=secrets.token_hex(5),
+                    # no directory
+                    value=datetime(2025, 1, 1, 11, 22, 33, 456789, tzinfo=timezone.utc),
+                ),
+                file_with_mtime(datetime(2025, 1, 1, 22, 33, 44, 567890, tzinfo=timezone.utc)),
+            ),
+            (
+                FileModifiedTimeHWM(
+                    name=secrets.token_hex(5),
+                    directory="/absolute/path",
+                    value=datetime(2025, 1, 1, 11, 22, 33, 456789, tzinfo=timezone.utc),
+                ),
+                file_with_mtime(datetime(2025, 1, 1, 22, 33, 44, 567890, tzinfo=timezone.utc)),
+            ),
+        ],
+    )
+
+
+@pytest.fixture(params=HWMS_WITH_VALUE)
 def hwm_new_value(request):
     return request.param
 

diff --git a/docs/changelog/next_release/40.feature.rst b/docs/changelog/next_release/40.feature.rst
@@ -0,0 +1 @@
+Allow using `etl-entities==2.5.0 <https://github.com/MobileTeleSystems/etl-entities/releases/tag/2.5.0>`_.
diff --git a/requirements.txt b/requirements.txt
@@ -1,2 +1,2 @@
 data-horizon[client-sync]>=1.0.0,<1.1
-etl-entities>=2.1.0,<2.5.0
+etl-entities>=2.1.0,<2.6.0
Original file line number	Diff line number	Diff line change
		@@ -0,0 +1 @@
		Allow using `etl-entities==2.5.0 <https://github.com/MobileTeleSystems/etl-entities/releases/tag/2.5.0>`_.