Get gc organisations data from S3 (#2056)

We now have a cached version of the gc-organisations datafile in S3 (both in Prod and Staging). This PR updates admin to use the file in S3.

app/__init__.py

@@ -86,7 +86,7 @@
 from app.notify_client.template_folder_api_client import template_folder_api_client
 from app.notify_client.template_statistics_api_client import template_statistics_client
 from app.notify_client.user_api_client import user_api_client
-from app.salesforce import salesforce_account
+from app.s3_client.s3_gc_organisations_client import get_gc_organisations
 from app.scanfiles.scanfiles_api_client import scanfiles_api_client
 from app.tou import EVENTS_KEY, show_tou_prompt
 from app.utils import documentation_url, id_safe
@@ -228,11 +228,9 @@ def get_locale():
     application.jinja_env.globals["events_key"] = EVENTS_KEY
     application.jinja_env.globals["now"] = datetime.utcnow
 
-    # Initialize Salesforce Account list
+    # Initialize the GC Organisation list
     if application.config["FF_SALESFORCE_CONTACT"]:
-        application.config["CRM_ORG_LIST"] = salesforce_account.get_accounts(
-            application.config["CRM_ORG_LIST_URL"], application.config["CRM_GITHUB_PERSONAL_ACCESS_TOKEN"], application.logger
-        )
+        application.config["CRM_ORG_LIST"] = get_gc_organisations(application)
 
     # Specify packages to be traced by MonkeyType. This can be overriden
     # via the MONKEYTYPE_TRACE_MODULES environment variable. e.g:

app/config.py

@@ -48,8 +48,6 @@ class Config(object):
 
     CHECK_PROXY_HEADER = False
     CONTACT_EMAIL = os.environ.get("CONTACT_EMAIL", "assistance+notification@cds-snc.ca")
-    CRM_GITHUB_PERSONAL_ACCESS_TOKEN = os.getenv("CRM_GITHUB_PERSONAL_ACCESS_TOKEN")
-    CRM_ORG_LIST_URL = os.getenv("CRM_ORG_LIST_URL")
     CSV_MAX_ROWS = env.int("CSV_MAX_ROWS", 50_000)
     CSV_MAX_ROWS_BULK_SEND = env.int("CSV_MAX_ROWS_BULK_SEND", 100_000)
     CSV_UPLOAD_BUCKET_NAME = os.getenv("CSV_UPLOAD_BUCKET_NAME", "notification-alpha-canada-ca-csv-upload")
@@ -87,6 +85,8 @@ class Config(object):
     GC_ARTICLES_API = os.environ.get("GC_ARTICLES_API", "articles.alpha.canada.ca/notification-gc-notify")
     GC_ARTICLES_API_AUTH_PASSWORD = os.environ.get("GC_ARTICLES_API_AUTH_PASSWORD")
     GC_ARTICLES_API_AUTH_USERNAME = os.environ.get("GC_ARTICLES_API_AUTH_USERNAME")
+    GC_ORGANISATIONS_BUCKET_NAME = os.environ.get("GC_ORGANISATIONS_BUCKET_NAME")
+    GC_ORGANISATIONS_FILENAME = os.getenv("GC_ORGANISATIONS_FILENAME", "all.json")
     GOOGLE_ANALYTICS_ID = os.getenv("GOOGLE_ANALYTICS_ID", "UA-102484926-14")
     GOOGLE_TAG_MANAGER_ID = os.getenv("GOOGLE_TAG_MANAGER_ID", "GTM-KRKRZQV")
     HC_EN_SERVICE_ID = os.getenv("HC_EN_SERVICE_ID")
@@ -154,7 +154,6 @@ def get_sensitive_config(cls) -> list[str]:
         return [
             "ADMIN_CLIENT_SECRET",
             "ANTIVIRUS_API_KEY",
-            "CRM_GITHUB_PERSONAL_ACCESS_TOKEN",
             "DANGEROUS_SALT",
             "DEBUG_KEY",
             "GC_ARTICLES_API_AUTH_PASSWORD",
@@ -196,8 +195,6 @@ class Test(Development):
     ANTIVIRUS_API_KEY = "test-antivirus-secret"
     API_HOST_NAME = os.environ.get("API_HOST_NAME", "http://localhost:6011")
     ASSET_DOMAIN = "static.example.com"
-    CRM_ORG_LIST_URL = "test-domain-dot-com"
-    CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "not-a-real-token"
     DANGEROUS_SALT = os.environ.get("DANGEROUS_SALT", "dev-notify-salt")
     DEBUG = True
     DEBUG_KEY = "debug"
@@ -212,7 +209,7 @@ class Test(Development):
     FF_SALESFORCE_CONTACT = False
     SYSTEM_STATUS_URL = "https://localhost:3000"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
-
+    GC_ORGANISATIONS_BUCKET_NAME = "test-gc-organisations"
     FF_RTL = True
     FF_ANNUAL_LIMIT = True
 
@@ -223,8 +220,6 @@ class ProductionFF(Config):
     ANTIVIRUS_API_KEY = "test-antivirus-secret"
     API_HOST_NAME = os.environ.get("API_HOST_NAME", "http://localhost:6011")
     ASSET_DOMAIN = "static.example.com"
-    CRM_ORG_LIST_URL = "test-domain-dot-com"
-    CRM_GITHUB_PERSONAL_ACCESS_TOKEN = "not-a-real-token"
     DANGEROUS_SALT = os.environ.get("DANGEROUS_SALT", "dev-notify-salt")
     DEBUG = True
     DEBUG_KEY = "debug"
@@ -239,7 +234,7 @@ class ProductionFF(Config):
     FF_SALESFORCE_CONTACT = False
     SYSTEM_STATUS_URL = "https://localhost:3000"
     NO_BRANDING_ID = "0af93cf1-2c49-485f-878f-f3e662e651ef"
-
+    GC_ORGANISATIONS_BUCKET_NAME = "dev-gc-organisations"
     FF_RTL = False
     FF_ANNUAL_LIMIT = False
 

app/s3_client/s3_gc_organisations_client.py

@@ -0,0 +1,43 @@
+import json
+
+import botocore
+from unidecode import unidecode
+
+from app.s3_client.s3_logo_client import get_s3_object
+
+
+def get_gc_organisations_from_s3(current_app):
+    bucket = current_app.config["GC_ORGANISATIONS_BUCKET_NAME"]
+    filename = current_app.config["GC_ORGANISATIONS_FILENAME"]
+    try:
+        key = get_s3_object(bucket, filename)
+        data_str = key.get()["Body"].read().decode("utf-8")
+        org_data = json.loads(data_str)
+        return org_data
+    except (botocore.exceptions.ClientError, botocore.exceptions.ParamValidationError, ValueError):
+        current_app.logger.error("Unable to download s3 file {}/{}".format(bucket, filename))
+        return []
+
+
+def parse_gc_organisations_data(org_data):
+    if not org_data:
+        return {"all": [], "names": {}}
+
+    org_name_data = {
+        "en": [item["name_eng"] for item in org_data],
+        "fr": [item["name_fra"] for item in org_data],
+    }
+    # unidecode is needed so that names starting with accents like é
+    # are sorted correctly
+    sorted_org_name_data = {
+        "en": sorted(org_name_data["en"], key=unidecode),
+        "fr": sorted(org_name_data["fr"], key=unidecode),
+    }
+    parsed_org_data = {"all": org_data, "names": sorted_org_name_data}
+    return parsed_org_data
+
+
+def get_gc_organisations(current_app):
+    org_data = get_gc_organisations_from_s3(current_app)
+    parsed_org_data = parse_gc_organisations_data(org_data)
+    return parsed_org_data

tests/app/s3_client/test_s3_gc_organisations_client.py

@@ -0,0 +1,27 @@
+from app.s3_client.s3_gc_organisations_client import parse_gc_organisations_data
+
+
+def test_parse_gc_organisations_data_sorts_alphabetically(mocker, app_):
+    test_data = [
+        {
+            "id": "1234",
+            "name_eng": "CDS",
+            "name_fra": "SNC",
+        },
+        {
+            "id": "2234",
+            "name_eng": "TBS",
+            "name_fra": "SCT",
+        },
+        {"id": "3456", "name_eng": "ABC", "name_fra": "ÉASDF"},
+    ]
+    parsed_data = parse_gc_organisations_data(test_data)
+    assert parsed_data["all"] == test_data
+    assert parsed_data["names"] == {"en": ["ABC", "CDS", "TBS"], "fr": ["ÉASDF", "SCT", "SNC"]}
+
+
+def test_parse_gc_organisations_data_returns_empty_dict(mocker, app_):
+    test_data = None
+    parsed_data = parse_gc_organisations_data(test_data)
+    assert parsed_data["all"] == []
+    assert parsed_data["names"] == {}