From f1b37536769463206ce48a7cfb9637dc8c3486ce Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Thu, 7 Mar 2024 13:41:00 -0600 Subject: [PATCH 01/11] Add person names to protected people fields --- atd-etl/cris_import/lib/mappings.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/atd-etl/cris_import/lib/mappings.py b/atd-etl/cris_import/lib/mappings.py index f13cce0fb..42a7f75da 100644 --- a/atd-etl/cris_import/lib/mappings.py +++ b/atd-etl/cris_import/lib/mappings.py @@ -74,6 +74,10 @@ def no_override_columns(): "prsn_occpnt_pos_id", # key column (but why?) "injury_severity", "prsn_age", + "prsn_name_honorific", + "prsn_last_name", + "prsn_first_name", + "prsn_mid_name", }, "atd_txdot_primaryperson": { "crash_id", # key column @@ -83,6 +87,10 @@ def no_override_columns(): "prsn_occpnt_pos_id", # key column (but why?) "injury_severity", "prsn_age", + "prsn_name_honorific", + "prsn_last_name", + "prsn_first_name", + "prsn_mid_name", }, "atd_txdot_charges": { "crash_id", # key column From 296da6e9ce93b555f49cbb52ed6f1399e432a7cc Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 09:40:23 -0500 Subject: [PATCH 02/11] Align important fields between VZE & cris_import.py --- atd-etl/cris_import/lib/mappings.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/atd-etl/cris_import/lib/mappings.py b/atd-etl/cris_import/lib/mappings.py index 42a7f75da..e05d026c9 100644 --- a/atd-etl/cris_import/lib/mappings.py +++ b/atd-etl/cris_import/lib/mappings.py @@ -57,14 +57,19 @@ def no_override_columns(): "address_confirmed_secondary", "private_dr_fl", "road_constr_zone_fl", + "case_id", + "sus_serious_injry_cnt", + "intrsct_relat_id", + "rpt_outside_city_limit_fl", }, "atd_txdot_units": { "crash_id", # key column "unit_nbr", # key column "travel_direction", "movement_id", - "death_cnt", "sus_serious_injry_cnt", + "unit_desc_id", + "veh_body_styl_id", }, "atd_txdot_person": { "crash_id", # key column @@ -78,6 +83,8 @@ def no_override_columns(): "prsn_last_name", "prsn_first_name", "prsn_mid_name", + "prsn_gndr_id", + "prsn_ethnicity_id", }, "atd_txdot_primaryperson": { "crash_id", # key column @@ -91,6 +98,8 @@ def no_override_columns(): "prsn_last_name", "prsn_first_name", "prsn_mid_name", + "prsn_gndr_id", + "prsn_ethnicity_id", }, "atd_txdot_charges": { "crash_id", # key column From 47d7597f91da49dd141ae10571aa709468b7c2c3 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:16:58 -0500 Subject: [PATCH 03/11] add comment about function import naming --- atd-etl/cris_import/cris_import.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/atd-etl/cris_import/cris_import.py b/atd-etl/cris_import/cris_import.py index b56abf23f..ab448c2fd 100755 --- a/atd-etl/cris_import/cris_import.py +++ b/atd-etl/cris_import/cris_import.py @@ -673,6 +673,9 @@ def align_records(map_state): all_changed_columns = ", ".join(important_changed_columns["changed_columns"] + changed_columns["changed_columns"]) # insert_change_template() is used with minimal changes from previous version of the ETL to better ensure conflict system compatibility + # the line immediately below is actually called `insert_crash_change_template`. As Rose + # observed today, non-crashes are not being handled correctly by the system. I think + # changes to those types of records may be just getting dropped. mutation = insert_change_template(new_record_dict=source, differences=all_changed_columns, crash_id=str(source["crash_id"])) if not dry_run: print("Making a mutation for " + str(source["crash_id"])) From c00c65906d3a30ae5b0ae735bda4603f55f818dd Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:18:12 -0500 Subject: [PATCH 04/11] add file to hold open the dev extract folder --- atd-etl/cris_import/development_extracts/.gitignore | 2 ++ 1 file changed, 2 insertions(+) create mode 100644 atd-etl/cris_import/development_extracts/.gitignore diff --git a/atd-etl/cris_import/development_extracts/.gitignore b/atd-etl/cris_import/development_extracts/.gitignore new file mode 100644 index 000000000..d77864a99 --- /dev/null +++ b/atd-etl/cris_import/development_extracts/.gitignore @@ -0,0 +1,2 @@ +*.zip + From 0e5aa01bfb396d6e3fa497d30782e4d489841977 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:19:42 -0500 Subject: [PATCH 05/11] remove explicit table name --- atd-etl/cris_import/lib/mappings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/atd-etl/cris_import/lib/mappings.py b/atd-etl/cris_import/lib/mappings.py index e05d026c9..15869326f 100644 --- a/atd-etl/cris_import/lib/mappings.py +++ b/atd-etl/cris_import/lib/mappings.py @@ -52,7 +52,7 @@ def no_override_columns(): "atd_fatality_count", "apd_confirmed_death_count", "tot_injry_cnt", - "atd_txdot_crashes.sus_serious_injry_cnt", + "sus_serious_injry_cnt", "address_confirmed_primary", "address_confirmed_secondary", "private_dr_fl", From eca6c1ac3a3735f863365b6ac1f72c70d2ff3edd Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:20:47 -0500 Subject: [PATCH 06/11] refine comment --- atd-etl/cris_import/cris_import.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/atd-etl/cris_import/cris_import.py b/atd-etl/cris_import/cris_import.py index ab448c2fd..e246a00a8 100755 --- a/atd-etl/cris_import/cris_import.py +++ b/atd-etl/cris_import/cris_import.py @@ -642,6 +642,10 @@ def align_records(map_state): important_changed_columns = util.get_changed_columns(pg, important_column_aggregators, output_map, table, linkage_clauses, record_key_sql, map_state["import_schema"]) + # print(f"Changed_columns: {changed_columns}") + # print(f"Important_changed_columns: {important_changed_columns}") + # input() + if len(important_changed_columns['changed_columns']) > 0: # This execution branch leads to the conflict resolution system in VZ @@ -673,15 +677,15 @@ def align_records(map_state): all_changed_columns = ", ".join(important_changed_columns["changed_columns"] + changed_columns["changed_columns"]) # insert_change_template() is used with minimal changes from previous version of the ETL to better ensure conflict system compatibility - # the line immediately below is actually called `insert_crash_change_template`. As Rose - # observed today, non-crashes are not being handled correctly by the system. I think - # changes to those types of records may be just getting dropped. + # the line immediately below is actually called `insert_crash_change_template`. mutation = insert_change_template(new_record_dict=source, differences=all_changed_columns, crash_id=str(source["crash_id"])) if not dry_run: print("Making a mutation for " + str(source["crash_id"])) graphql.make_hasura_request(query=mutation, endpoint=GRAPHQL_ENDPOINT, admin_secret=GRAPHQL_ENDPOINT_KEY) else: - # This execution branch leads to forming an update statement and executing it + # Thi) + # print(f"Mutation: {mutation}") + # input(s execution branch leads to forming an update statement and executing it if len(changed_columns["changed_columns"]) == 0: print(update_statement) From 8b9cc560e4d3e3718a941d4fc80d4fc450f01aa7 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:22:07 -0500 Subject: [PATCH 07/11] removing debug print statement --- atd-etl/cris_import/cris_import.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/atd-etl/cris_import/cris_import.py b/atd-etl/cris_import/cris_import.py index e246a00a8..9141ceee7 100755 --- a/atd-etl/cris_import/cris_import.py +++ b/atd-etl/cris_import/cris_import.py @@ -641,11 +641,6 @@ def align_records(map_state): # Do the same thing, but this time using the SQL clauses formed from "important" columns. important_changed_columns = util.get_changed_columns(pg, important_column_aggregators, output_map, table, linkage_clauses, record_key_sql, map_state["import_schema"]) - - # print(f"Changed_columns: {changed_columns}") - # print(f"Important_changed_columns: {important_changed_columns}") - # input() - if len(important_changed_columns['changed_columns']) > 0: # This execution branch leads to the conflict resolution system in VZ @@ -677,16 +672,12 @@ def align_records(map_state): all_changed_columns = ", ".join(important_changed_columns["changed_columns"] + changed_columns["changed_columns"]) # insert_change_template() is used with minimal changes from previous version of the ETL to better ensure conflict system compatibility - # the line immediately below is actually called `insert_crash_change_template`. mutation = insert_change_template(new_record_dict=source, differences=all_changed_columns, crash_id=str(source["crash_id"])) if not dry_run: print("Making a mutation for " + str(source["crash_id"])) graphql.make_hasura_request(query=mutation, endpoint=GRAPHQL_ENDPOINT, admin_secret=GRAPHQL_ENDPOINT_KEY) else: - # Thi) - # print(f"Mutation: {mutation}") - # input(s execution branch leads to forming an update statement and executing it - + # This execution branch leads to forming an update statement and executing it if len(changed_columns["changed_columns"]) == 0: print(update_statement) raise "No changed columns? Why are we forming an update? This is a bug." From d2086132bc643520d08986c97d2af163cc00b06d Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:28:40 -0500 Subject: [PATCH 08/11] bump version numbers --- atd-vze/package.json | 4 ++-- atd-vzv/package.json | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/atd-vze/package.json b/atd-vze/package.json index 379005e00..f8bf1634a 100644 --- a/atd-vze/package.json +++ b/atd-vze/package.json @@ -1,6 +1,6 @@ { "name": "atd-vz-data", - "version": "1.42.0", + "version": "1.42.1", "homepage": "./", "description": "ATD Vision Zero Editor", "author": "ATD Data & Technology Services", @@ -102,4 +102,4 @@ "node": ">=8.10", "npm": ">=6" } -} +} \ No newline at end of file diff --git a/atd-vzv/package.json b/atd-vzv/package.json index 6e1e69309..cd0a86657 100644 --- a/atd-vzv/package.json +++ b/atd-vzv/package.json @@ -1,6 +1,6 @@ { "name": "atd-vzv", - "version": "1.42.0", + "version": "1.42.1", "homepage": "/viewer", "description": "ATD Vision Zero Viewer", "author": "ATD Data & Technology Services", @@ -96,4 +96,4 @@ "devDependencies": { "react-axe": "^3.5.3" } -} +} \ No newline at end of file From 080b32950f7bcdc91caa60e802384785608b270b Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:30:00 -0500 Subject: [PATCH 09/11] override my linter which removed the EOF LFs --- atd-vze/package.json | 3 ++- atd-vzv/package.json | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/atd-vze/package.json b/atd-vze/package.json index f8bf1634a..d671dd182 100644 --- a/atd-vze/package.json +++ b/atd-vze/package.json @@ -102,4 +102,5 @@ "node": ">=8.10", "npm": ">=6" } -} \ No newline at end of file +} + diff --git a/atd-vzv/package.json b/atd-vzv/package.json index cd0a86657..1205da0f1 100644 --- a/atd-vzv/package.json +++ b/atd-vzv/package.json @@ -96,4 +96,5 @@ "devDependencies": { "react-axe": "^3.5.3" } -} \ No newline at end of file +} + From 9c40884aa3d0d17671c728cb10ff07ba9f2790c7 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Mon, 11 Mar 2024 12:44:03 -0500 Subject: [PATCH 10/11] vim adds the line-feed automatically, so I had added two. Correcting JSON EOF linefeed. --- atd-vze/package.json | 1 - atd-vzv/package.json | 1 - 2 files changed, 2 deletions(-) diff --git a/atd-vze/package.json b/atd-vze/package.json index d671dd182..2ae0cd585 100644 --- a/atd-vze/package.json +++ b/atd-vze/package.json @@ -103,4 +103,3 @@ "npm": ">=6" } } - diff --git a/atd-vzv/package.json b/atd-vzv/package.json index 1205da0f1..9ea327cbe 100644 --- a/atd-vzv/package.json +++ b/atd-vzv/package.json @@ -97,4 +97,3 @@ "react-axe": "^3.5.3" } } - From 5da0f73ef10ee81145411ba16bcb2028f45ae746 Mon Sep 17 00:00:00 2001 From: Frank Hereford Date: Tue, 19 Mar 2024 13:47:38 -0500 Subject: [PATCH 11/11] add delete permissions for vz-admin --- .../databases/default/tables/public_atd_txdot_changes.yaml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/atd-vzd/metadata/databases/default/tables/public_atd_txdot_changes.yaml b/atd-vzd/metadata/databases/default/tables/public_atd_txdot_changes.yaml index 585725ca6..a861a6fe3 100644 --- a/atd-vzd/metadata/databases/default/tables/public_atd_txdot_changes.yaml +++ b/atd-vzd/metadata/databases/default/tables/public_atd_txdot_changes.yaml @@ -57,3 +57,7 @@ select_permissions: - crash_date - record_uqid filter: {} +delete_permissions: + - role: vz-admin + permission: + filter: {}