Add example config and template test config (#13)

Add prerequisites configuration files

Add command line interface local, report, test groups

Add integration tests for local multiprocess mode

Add gRPC arbiter

Add tests and configs

CLI Local and distributed launch

Add docstings, add metrics to Prometheus

Add distributed experiment example

Add README.md and docs

Add tox, liner, code formatter, type checker

Add loop to member, change communicator interface

Make batcher available for members

Fix grpc based communicator according to changed interfaces

Fix PartyMaster/MemberImpl and launch

Fix local communicator interfaces

Co-authored-by: zakharova-anastasiia <nastyazakharova.nz@gmail.com>

.github/workflows/ci.yml

@@ -0,0 +1,27 @@
+name: CI pipeline
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    branches: [ "main" ]
+
+jobs:
+  run-tests:
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: ['3.9', '3.10', '3.11']
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Python ${{ matrix.python-version }}
+      uses: actions/setup-python@v4
+      with:
+        python-version: ${{ matrix.python-version }}
+    - name: Install dependencies
+      run: |
+        python -m pip install --upgrade pip
+        python -m pip install tox tox-gh-actions
+    - name: Test with tox
+      run: tox

.gitignore

@@ -159,6 +159,7 @@ cython_debug/
 #  option (not recommended) you can uncomment the following to ignore the entire idea folder.
 .idea/
 mlruns
+main_grps.py
 .docker
 data
 rsync_repo

.pyre_configuration

@@ -0,0 +1,34 @@
+{
+  "site_package_search_strategy": "pep561",
+  "source_directories": [
+    "./stalactite"
+  ],
+  "ignore_all_errors": [
+    ".*/generated_code/.*"
+  ],
+  "search_path": [
+    ".",
+    {
+      "site-package": "tenseal"
+    },
+    {
+      "site-package": "grpc"
+    },
+    {
+      "site-package": "mlflow"
+    },
+    {
+      "site-package": "sklearn"
+    },
+    {
+      "site-package": "docker"
+    },
+    {
+      "site-package": "scipy"
+    },
+    {
+      "site-package": "datasets"
+    }
+  ]
+}
+

configs/config-test.yml

@@ -8,7 +8,6 @@ common:
   world_size: 2
   batch_size: 1000
   experiment_label: test-experiment-0
-  rendezvous_timeout: 3600
   reports_export_folder: "." # todo: Path to export tests logs and reports
 
 prerequisites:
@@ -27,29 +26,24 @@ grpc_server:
   host: '0.0.0.0'
   port: '50051'
   max_message_size: -1
-  server_thread_pool_size: 10
 
 master:
   run_mlflow: True
   run_prometheus: True
   logging_level: 'debug'
   disconnect_idle_client_time: 120.
-  time_between_idle_connections_checks: 3.
 
 member:
   logging_level: 'debug'
   heartbeat_interval: 2.
-  task_requesting_pings_interval: 0.1
-  sent_task_timout: 3600
 
 grpc_arbiter:
-  use_arbiter: True
   container_host: 'node3.bdcl'
   host: '0.0.0.0'
   port: '50052'
   max_message_size: -1
   server_threadpool_max_workers: 10
-  grpc_operations_timeout: 300
+  use_arbiter: True
   ts_algorithm: CKKS
   ts_poly_modulus_degree: 8192
   ts_coeff_mod_bit_sizes:

configs/config.yml

@@ -6,7 +6,6 @@ common:
   batch_size: 1000
   experiment_label: experiment-vm
   reports_export_folder: "/mnt/ess_storage/DN_1/storage/qa-system-research/zakharova/vfl-benchmark-test-mounts/reports"
-  rendezvous_timeout: 3600
 
 prerequisites:
   mlflow_host: 'node3.bdcl'
@@ -24,31 +23,25 @@ grpc_server:
   host: '0.0.0.0'
   port: '50051'
   max_message_size: -1
-  server_thread_pool_size: 10
 
 master:
   container_host: 'node3.bdcl'
   run_mlflow: True
   run_prometheus: True
   logging_level: 'debug'
   disconnect_idle_client_time: 120.
-  time_between_idle_connections_checks: 3.
-
 
 member:
   logging_level: 'debug'
   heartbeat_interval: 2.
-  task_requesting_pings_interval: 0.1
-  sent_task_timout: 3600
 
 grpc_arbiter:
-  use_arbiter: True
   container_host: 'node3.bdcl'
   host: '0.0.0.0'
   port: '50052'
   max_message_size: -1
   server_threadpool_max_workers: 10
-  grpc_operations_timeout: 300
+  use_arbiter: True
   ts_algorithm: CKKS
   ts_poly_modulus_degree: 8192
   ts_coeff_mod_bit_sizes:

pyproject.toml

@@ -27,10 +27,9 @@ mlflow = "2.9.2" #
 pydantic = "^2.5.3"
 tenseal = "^0.3.14"
 
-
 [tool.poetry.group.dev.dependencies]
 grpcio-tools = "^1.59.3"
-
+tox = "*"
 
 [tool.poetry.group.test.dependencies]
 pytest = "^7.4.3"
@@ -47,6 +46,75 @@ log_cli_level = "DEBUG"
 log_cli_format = "%(asctime)s [%(levelname)8s] %(message)s (%(filename)s:%(lineno)s)"
 log_cli_date_format = "%Y-%m-%d %H:%M:%S"
 
+[tool.isort]
+profile = "black"
+force_single_line = false
+use_parentheses = true
+force_grid_wrap = 0
+multi_line_output = 3
+include_trailing_comma = true
+extend_skip_glob = ["*/generated_code/*"]
+
+[tool.pylint.format]
+max-line-length = "120"
+ignore = "E203,W0511"
+ignore-paths = ["stalactite/communications/grpc_utils/generated_code/*"]
+
+
+[tool.black]
+line-length = 120
+target-version = ['py39', 'py310', 'py311']
+exclude = '''
+/(
+  stalactite/communications/grpc_utils/generated_code/*
+)/
+'''
+
+
+[tool.tox]
+legacy_tox_ini = """
+    [tox]
+    envlist = py{39, 310, 311},format,lint,type
+
+    [gh-actions]
+    python =
+        3.9: py39
+        3.10: py310
+        3.11: py311
+
+    [testenv]
+    description = tests
+    allowlist_externals = poetry
+    commands =
+        poetry install
+        poetry run pytest tests -v --ignore=tests/distributed_grpc
+
+    [testenv:format]
+    description = run code formatters
+    skip_install = true
+    deps =
+        black
+        isort
+    commands =
+        isort stalactite
+        black stalactite
+
+    [testenv:lint]
+    description = run linters
+    skip_install = true
+    deps =
+        pylint
+    commands =
+        pylint stalactite
+
+    [testenv:type]
+    description = run type checks
+    deps =
+        pyre-check
+    commands =
+        pyre restart
+"""
+
 [build-system]
 requires = ["poetry-core"]
 build-backend = "poetry.core.masonry.api"