Release VZ 1.43.0 - Vista Verde Dr

atd-cr3-api/server.py

@@ -39,6 +39,8 @@
 AWS_S3_CR3_LOCATION = os.getenv("AWS_S3_CR3_LOCATION", "")
 AWS_S3_BUCKET = os.getenv("AWS_S3_BUCKET", "")
 
+ADMIN_ROLE_NAME = "vz-admin"
+
 
 def get_api_token():
     """
@@ -327,8 +329,14 @@ def user_list_users():
     user_dict = current_user._get_current_object()
     page = request.args.get("page")
     per_page = request.args.get("per_page")
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
-        endpoint = f"https://{AUTH0_DOMAIN}/api/v2/users?page=" + page + "&per_page=" + per_page + "&include_totals=true"
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
+        endpoint = (
+            f"https://{AUTH0_DOMAIN}/api/v2/users?page="
+            + page
+            + "&per_page="
+            + per_page
+            + "&include_totals=true"
+        )
         headers = {"Authorization": f"Bearer {get_api_token()}"}
         response = requests.get(endpoint, headers=headers).json()
         return jsonify(response)
@@ -342,7 +350,7 @@ def user_list_users():
 @requires_auth
 def user_get_user(id):
     user_dict = current_user._get_current_object()
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
         endpoint = f"https://{AUTH0_DOMAIN}/api/v2/users/" + id
         headers = {"Authorization": f"Bearer {get_api_token()}"}
         response = requests.get(endpoint, headers=headers).json()
@@ -357,7 +365,7 @@ def user_get_user(id):
 @requires_auth
 def user_create_user():
     user_dict = current_user._get_current_object()
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
         json_data = request.json
         endpoint = f"https://{AUTH0_DOMAIN}/api/v2/users"
         headers = {"Authorization": f"Bearer {get_api_token()}"}
@@ -373,7 +381,7 @@ def user_create_user():
 @requires_auth
 def user_update_user(id):
     user_dict = current_user._get_current_object()
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
         json_data = request.json
         endpoint = f"https://{AUTH0_DOMAIN}/api/v2/users/" + id
         headers = {"Authorization": f"Bearer {get_api_token()}"}
@@ -389,7 +397,7 @@ def user_update_user(id):
 @requires_auth
 def user_unblock_user(id):
     user_dict = current_user._get_current_object()
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
         endpoint = f"https://{AUTH0_DOMAIN}/api/v2/user_blocks/" + id
         headers = {"Authorization": f"Bearer {get_api_token()}"}
         response = requests.delete(endpoint, headers=headers)
@@ -404,7 +412,7 @@ def user_unblock_user(id):
 @requires_auth
 def user_delete_user(id):
     user_dict = current_user._get_current_object()
-    if isValidUser(user_dict) and hasUserRole("admin", user_dict):
+    if isValidUser(user_dict) and hasUserRole(ADMIN_ROLE_NAME, user_dict):
         endpoint = f"https://{AUTH0_DOMAIN}/api/v2/users/" + id
         headers = {"Authorization": f"Bearer {get_api_token()}"}
         response = requests.delete(endpoint, headers=headers)

atd-etl/socrata_export/process/helpers_socrata.py

@@ -14,7 +14,7 @@
 import json
 from copy import deepcopy
 from process.config import ATD_ETL_CONFIG
-from datetime import date, timedelta
+from datetime import date, timedelta, datetime
 
 # Dict to translate modes to correlate with atd__mode_category_lkp table
 mode_categories = {
@@ -88,9 +88,9 @@ def flatten_hasura_response(records):
                                     )
                                 else:
                                     # Create key at top-level
-                                    formatted_record[
-                                        third_level_key
-                                    ] = third_level_value
+                                    formatted_record[third_level_key] = (
+                                        third_level_value
+                                    )
                         # Copy non-nested key-values to top-level (if value is not null)
                         # Null records can create unwanted columns at top level of record
                         elif second_level_value is not None:
@@ -315,7 +315,7 @@ def format_person_data(data, formatter_config):
     return formatted_records
 
 
-def get_date_limit():
+def two_weeks_ago():
     """
     Returns a string with the date two weeks ago in iso format: yyyy-mm-dd
     :return str:
@@ -327,13 +327,13 @@ def get_date_limit():
         return (d - timedelta(days=14)).strftime("%Y-%m-%d")
 
 
-def get_initial_date_limit():
+def ten_years_ago():
     """
-    Returns a string with the date ten years ago in iso format: yyyy-mm-dd
+    Returns a string with the date on January 1st, ten years ago in format: yyyy-mm-dd.
     :return str:
     """
-    d = date.today()
-    return d.replace(year=d.year - 10).strftime("%Y-%m-%d")
+    year = datetime.now().year
+    return f"{year - 10}-01-01"
 
 
 def is_no_time_constraint():

atd-etl/socrata_export/socrata_export.py

@@ -69,8 +69,7 @@
 # For each config, get records from Hasura and upsert to Socrata until res is []
 for config in query_configs:
     print(f'Starting {config["table"]} table...')
-    print(f'Truncating {config["table"]} table...')
-    client.replace(config["dataset_uid"], [])
+    is_truncate_complete = False
     records = None
     offset = 0
     limit = 1000
@@ -82,8 +81,8 @@
         query = config["template"].substitute(
             limit=limit,
             offset=offset,
-            date_limit=get_date_limit(),
-            initial_date_limit=get_initial_date_limit(),
+            date_limit=two_weeks_ago(),
+            initial_date_limit=ten_years_ago(),
         )
         offset += limit
         data = run_hasura_query(query)
@@ -96,6 +95,12 @@
         # Format records
         records = config["formatter"](data, config["formatter_config"])
 
+        # truncate the existing data set once we have records
+        if records and not is_truncate_complete:
+            print(f'Truncating {config["table"]} table...')
+            client.replace(config["dataset_uid"], [])
+            is_truncate_complete = True
+
         # Upsert records to Socrata
         client.upsert(config["dataset_uid"], records)
         total_records += len(records)

atd-vzd/metadata/databases/default/tables/public_atd__coordination_partners_lkp.yaml

@@ -24,3 +24,10 @@ select_permissions:
         - id
       filter: {}
     comment: ""
+  - role: vz-admin
+    permission:
+      columns:
+        - coord_partner_desc
+        - id
+      filter: {}
+    comment: ""

atd-vzd/metadata/databases/default/tables/public_atd__mode_category_lkp.yaml

@@ -18,3 +18,11 @@ select_permissions:
         - id
       filter: {}
     comment: ""
+  - role: vz-admin
+    permission:
+      columns:
+        - atd_mode_category_desc
+        - atd_mode_category_mode_name
+        - id
+      filter: {}
+    comment: ""

atd-vzd/metadata/databases/default/tables/public_atd__recommendation_status_lkp.yaml

@@ -24,3 +24,10 @@ select_permissions:
         - rec_status_desc
       filter: {}
     comment: ""
+  - role: vz-admin
+    permission:
+      columns:
+        - id
+        - rec_status_desc
+      filter: {}
+    comment: ""

atd-vzd/metadata/databases/default/tables/public_atd_apd_blueform.yaml

@@ -19,6 +19,23 @@ insert_permissions:
         - longitude
         - position
         - speed_mgmt_points
+  - role: vz-admin
+    permission:
+      check: {}
+      columns:
+        - address
+        - case_id
+        - date
+        - est_comp_cost
+        - est_comp_cost_crash_based
+        - est_econ_cost
+        - form_id
+        - hour
+        - latitude
+        - location_id
+        - longitude
+        - position
+        - speed_mgmt_points
 select_permissions:
   - role: editor
     permission:
@@ -56,6 +73,24 @@ select_permissions:
         - speed_mgmt_points
       filter: {}
       allow_aggregations: true
+  - role: vz-admin
+    permission:
+      columns:
+        - address
+        - case_id
+        - date
+        - est_comp_cost
+        - est_comp_cost_crash_based
+        - est_econ_cost
+        - form_id
+        - hour
+        - latitude
+        - location_id
+        - longitude
+        - position
+        - speed_mgmt_points
+      filter: {}
+      allow_aggregations: true
 update_permissions:
   - role: editor
     permission:
@@ -75,3 +110,21 @@ update_permissions:
         - speed_mgmt_points
       filter: {}
       check: null
+  - role: vz-admin
+    permission:
+      columns:
+        - address
+        - case_id
+        - date
+        - est_comp_cost
+        - est_comp_cost_crash_based
+        - est_econ_cost
+        - form_id
+        - hour
+        - latitude
+        - location_id
+        - longitude
+        - position
+        - speed_mgmt_points
+      filter: {}
+      check: null

atd-vzd/metadata/databases/default/tables/public_atd_location_crash_and_cost_totals.yaml

@@ -24,3 +24,14 @@ select_permissions:
         - total_crashes
         - total_est_comp_cost
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - cr3_est_comp_cost
+        - cr3_total_crashes
+        - location_id
+        - noncr3_est_comp_cost
+        - noncr3_total_crashes
+        - total_crashes
+        - total_est_comp_cost
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__asmp_level_lkp.yaml

@@ -14,3 +14,9 @@ select_permissions:
         - asmp_level_desc
         - asmp_level_id
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - asmp_level_desc
+        - asmp_level_id
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__city_lkp.yaml

@@ -18,3 +18,11 @@ select_permissions:
         - eff_beg_date
         - eff_end_date
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - city_id
+        - city_desc
+        - eff_beg_date
+        - eff_end_date
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__collsn_lkp.yaml

@@ -1,15 +1,6 @@
 table:
   name: atd_txdot__collsn_lkp
   schema: public
-insert_permissions:
-  - role: editor
-    permission:
-      check: {}
-      columns:
-        - collsn_id
-        - collsn_desc
-        - eff_beg_date
-        - eff_end_date
 select_permissions:
   - role: editor
     permission:
@@ -27,13 +18,11 @@ select_permissions:
         - eff_beg_date
         - eff_end_date
       filter: {}
-update_permissions:
-  - role: editor
+  - role: vz-admin
     permission:
       columns:
         - collsn_id
         - collsn_desc
         - eff_beg_date
         - eff_end_date
       filter: {}
-      check: null

atd-vzd/metadata/databases/default/tables/public_atd_txdot__contrib_factr_lkp.yaml

@@ -20,3 +20,12 @@ select_permissions:
         - eff_end_date
         - other
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - contrib_factr_id
+        - contrib_factr_desc
+        - eff_beg_date
+        - eff_end_date
+        - other
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__est_comp_cost.yaml

@@ -14,3 +14,9 @@ select_permissions:
         - est_comp_cost_amount
         - est_comp_cost_id
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - est_comp_cost_amount
+        - est_comp_cost_id
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__est_comp_cost_crash_based.yaml

@@ -16,3 +16,10 @@ select_permissions:
         - est_comp_cost_amount
         - est_comp_cost_desc
       filter: {}
+  - role: vz-admin
+    permission:
+      columns:
+        - est_comp_cost_id
+        - est_comp_cost_amount
+        - est_comp_cost_desc
+      filter: {}

atd-vzd/metadata/databases/default/tables/public_atd_txdot__ethnicity_lkp.yaml

@@ -1,13 +1,6 @@
 table:
   name: atd_txdot__ethnicity_lkp
   schema: public
-insert_permissions:
-  - role: editor
-    permission:
-      check: {}
-      columns:
-        - ethnicity_desc
-        - ethnicity_id
 select_permissions:
   - role: editor
     permission:
@@ -21,15 +14,9 @@ select_permissions:
         - ethnicity_desc
         - ethnicity_id
       filter: {}
-update_permissions:
-  - role: editor
+  - role: vz-admin
     permission:
       columns:
         - ethnicity_desc
         - ethnicity_id
       filter: {}
-      check: null
-delete_permissions:
-  - role: editor
-    permission:
-      filter: {}