diff --git a/dbt/dbt_project.yml b/dbt/dbt_project.yml index c9ade8c2d..b8e590f87 100644 --- a/dbt/dbt_project.yml +++ b/dbt/dbt_project.yml @@ -29,7 +29,7 @@ vars: # year, since errors in past data cannot usually be amended once records are # closed. Set as an integer for compatibility with comparison operators and # SQL's BETWEEN - data_test_iasworld_year_start: 2024 + data_test_iasworld_year_start: 2025 # End year for iasWorld data tests. Typically set to a date in the future, # but can also be use to select specific time frames for testing @@ -43,6 +43,8 @@ models: +write_compression: zstd +format: parquet +ha: true + ccao: + +schema: ccao census: +schema: census default: diff --git a/dbt/macros/pre_stage_filters.sql b/dbt/macros/pre_stage_filters.sql index cc03e5e09..a39502a55 100644 --- a/dbt/macros/pre_stage_filters.sql +++ b/dbt/macros/pre_stage_filters.sql @@ -1,21 +1,27 @@ -- Macros to simplify the filter conditions that produce pre-mailed and --- pre-certified values. +-- pre-certified values in the `default.vw_pin_value` view. -- -- Note that the `cur = 'Y'` filter is not necessary in the -- default.vw_pin_value query that is the main consumer of these macros, since --- that query already filters rows where `cur = 'Y'`; however, we leave the --- filter in here so that it might make the macro more generally useful in --- other instances where we may not filter queries the same way -{% macro pre_stage_filters(tablename, stage_name) %} +-- that query already filters rows where `cur = 'Y'`. +-- +-- A PIN is in the pre-mailed stage for a year if none of its representations +-- in `asmt_all` in that year have procnames (i.e. stage names). We can check +-- for this status by checking the cardinality of the `stages.procnames` array, +-- where "cardinality" is the Trino SQL equivalent of the length of the array +{% macro pre_mailed_filters(tablename) %} {{ tablename }}.procname is null and {{ tablename }}.cur = 'Y' - and not contains(stages.procnames, '{{ stage_name }}') -{% endmacro %} - -{% macro pre_mailed_filters(tablename) %} - {{- pre_stage_filters(tablename, "CCAOVALUE") -}} + and cardinality(stages.procnames) = 0 {% endmacro %} +-- A PIN is the pre-certified stage for a year if it has at least one +-- representation in `asmt_all` with a procname (i.e. stage name), but none +-- of its representations in `asmt_all` have the procname that corresponds to +-- the Assessor Certified stage {% macro pre_certified_filters(tablename) %} - {{- pre_stage_filters(tablename, "CCAOFINAL") -}} + {{ tablename }}.procname is null + and {{ tablename }}.cur = 'Y' + and cardinality(stages.procnames) > 0 + and not contains(stages.procnames, 'CCAOFINAL') {% endmacro %} diff --git a/dbt/macros/tests/test_all.sql b/dbt/macros/tests/test_all.sql index 009197e62..59ab583ea 100644 --- a/dbt/macros/tests/test_all.sql +++ b/dbt/macros/tests/test_all.sql @@ -5,6 +5,5 @@ {% do test_generate_schema_name() %} {% do test_generate_alias_name() %} {% do test_format_additional_select_columns() %} - {% do test_pre_stage_filters() %} {% do test_insert_hyphens() %} {% endmacro %} diff --git a/dbt/macros/tests/test_pre_stage_filters.sql b/dbt/macros/tests/test_pre_stage_filters.sql deleted file mode 100644 index 6c831da86..000000000 --- a/dbt/macros/tests/test_pre_stage_filters.sql +++ /dev/null @@ -1,47 +0,0 @@ -{% macro test_pre_stage_filters() %} - {% do test_pre_stage_filters_params() %} - {% do test_pre_mailed_filters() %} - {% do test_pre_certified_filters() %} -{% endmacro %} - -{% macro expected_pre_mailed_filters() %} - asmt.procname is null - and asmt.cur = 'Y' - and not contains(stages.procnames, 'CCAOVALUE') -{% endmacro %} - -{% macro expected_pre_certified_filters() %} - asmt.procname is null - and asmt.cur = 'Y' - and not contains(stages.procnames, 'CCAOFINAL') -{% endmacro %} - -{% macro test_pre_stage_filters_params() %} - {{ - assert_equals( - "test_pre_stage_filters_params", - pre_stage_filters("asmt", "CCAOVALUE"), - expected_pre_mailed_filters(), - ) - }} -{% endmacro %} - -{% macro test_pre_mailed_filters() %} - {{ - assert_equals( - "test_pre_mailed_filters_params", - pre_mailed_filters("asmt"), - expected_pre_mailed_filters(), - ) - }} -{% endmacro %} - -{% macro test_pre_certified_filters() %} - {{ - assert_equals( - "test_pre_certified_filters_params", - pre_certified_filters("asmt"), - expected_pre_certified_filters(), - ) - }} -{% endmacro %} diff --git a/dbt/models/ccao/ccao.vw_time_util.sql b/dbt/models/ccao/ccao.vw_time_util.sql new file mode 100644 index 000000000..b1cad4e83 --- /dev/null +++ b/dbt/models/ccao/ccao.vw_time_util.sql @@ -0,0 +1,4 @@ +-- View that supplies dynamic datetime values in a way that we can mock for +-- unit tests. See: +-- https://discourse.getdbt.com/t/dynamic-dates-in-unit-tests/16883/2 +SELECT CURRENT_DATE AS date_today diff --git a/dbt/models/ccao/docs.md b/dbt/models/ccao/docs.md index f27f5697a..06532831d 100644 --- a/dbt/models/ccao/docs.md +++ b/dbt/models/ccao/docs.md @@ -97,3 +97,12 @@ Collected yearly from Valuations via spreadsheets. **Primary Key**: `pin`, `year` {% enddocs %} + +# vw_time_util + +{% docs view_vw_time_util %} +View that supplies dynamic datetime values in a way that we can mock for unit +tests. + +See: +{% enddocs %} diff --git a/dbt/models/ccao/schema.yml b/dbt/models/ccao/schema.yml index 334ce87df..150273cb4 100644 --- a/dbt/models/ccao/schema.yml +++ b/dbt/models/ccao/schema.yml @@ -1,3 +1,7 @@ +models: + - name: ccao.vw_time_util + description: '{{ doc("view_vw_time_util") }}' + sources: - name: ccao tags: diff --git a/dbt/models/default/default.vw_pin_history.sql b/dbt/models/default/default.vw_pin_history.sql index 054fc86f7..dd95a67fd 100644 --- a/dbt/models/default/default.vw_pin_history.sql +++ b/dbt/models/default/default.vw_pin_history.sql @@ -6,6 +6,12 @@ SELECT leg.user1 AS township_code, town.township_name, REGEXP_REPLACE(par.nbhd, '([^0-9])', '') AS nbhd, + -- Add pre-mailed values to cover us for the period after the rollover but + -- before the year's assessment cycle starts + vwpv.pre_mailed_class, + vwpv.pre_mailed_bldg, + vwpv.pre_mailed_land, + vwpv.pre_mailed_tot, vwpv.mailed_class, vwpv.mailed_bldg, vwpv.mailed_land, diff --git a/dbt/models/default/default.vw_pin_value.sql b/dbt/models/default/default.vw_pin_value.sql index 5664c829d..e9cd312be 100644 --- a/dbt/models/default/default.vw_pin_value.sql +++ b/dbt/models/default/default.vw_pin_value.sql @@ -16,10 +16,25 @@ WITH stages AS ( SELECT parid, taxyr, - ARRAY_AGG(procname) AS procnames + -- Force an empty array representing the procnames for any PIN/year + -- combo that has no procnames yet, so that our `CONTAINS()` check + -- in subsequent queries that join to this stage do not have to worry + -- about side effects from null comparisons + ARRAY_REMOVE( + ARRAY_AGG( + CASE + WHEN procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') + THEN procname + -- Can't use null to indicate missing data here, since a + -- null comparison in the outer `ARRAY_REMOVE` call would + -- always cast the array to null + ELSE '' + END + ), + '' + ) AS procnames FROM {{ source('iasworld', 'asmt_all') }} - WHERE procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE') - AND rolltype != 'RR' + WHERE rolltype != 'RR' AND deactivat IS NULL AND valclass IS NULL GROUP BY parid, taxyr @@ -401,7 +416,10 @@ stage_values AS ( -- stages, it is most likely a provisional value for a PIN -- that has not mailed yet CARDINALITY(stages.procnames) != 0 - OR asmt.taxyr = DATE_FORMAT(NOW(), '%Y') + OR asmt.taxyr = DATE_FORMAT( + (SELECT date_today FROM {{ ref("ccao.vw_time_util") }}), + '%Y' + ) ) ) ) diff --git a/dbt/models/default/schema/default.vw_pin_history.yml b/dbt/models/default/schema/default.vw_pin_history.yml index 15c0e2c64..c697864b3 100644 --- a/dbt/models/default/schema/default.vw_pin_history.yml +++ b/dbt/models/default/schema/default.vw_pin_history.yml @@ -53,6 +53,14 @@ models: description: '{{ doc("shared_column_mailed_tot") }}' - name: pin description: '{{ doc("shared_column_pin") }}' + - name: pre_mailed_bldg + description: '{{ doc("shared_column_pre_mailed_bldg") }}' + - name: pre_mailed_class + description: '{{ doc("shared_column_pre_mailed_class") }}' + - name: pre_mailed_land + description: '{{ doc("shared_column_pre_mailed_land") }}' + - name: pre_mailed_tot + description: '{{ doc("shared_column_pre_mailed_tot") }}' - name: township_code description: '{{ doc("shared_column_township_code") }}' - name: township_name diff --git a/dbt/models/default/schema/default.vw_pin_value.yml b/dbt/models/default/schema/default.vw_pin_value.yml index ad02bda65..485cd0031 100644 --- a/dbt/models/default/schema/default.vw_pin_value.yml +++ b/dbt/models/default/schema/default.vw_pin_value.yml @@ -118,6 +118,9 @@ models: - `3` = Board certified - name: year description: '{{ doc("shared_column_year") }}' + data_tests: + - not_null: + name: default_vw_pin_value_stage_num_not_null data_tests: - not_null: @@ -125,7 +128,7 @@ models: column_name: board_class config: where: CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1 - error_if: ">1260" + error_if: ">1261" - not_null: name: default_vw_pin_value_board_tot_mv_not_null column_name: board_tot_mv @@ -133,13 +136,13 @@ models: where: | CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1 AND year >= '2020' - error_if: ">1260" + error_if: ">1261" - not_null: name: default_vw_pin_value_board_tot_not_null column_name: board_tot config: where: CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1 - error_if: ">1260" + error_if: ">1261" - not_null: name: default_vw_pin_value_certified_class_not_null column_name: certified_class @@ -226,12 +229,58 @@ unit_tests: # provided to ensure proper joins when creating the dummy table rows: - {parid: "123", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10, class: "2.1-1)A"} + # The input tables below are not important, and are only included for the + # sake of completing the joins that construct the view - input: source("iasworld", "aprval") rows: - {parid: "123", taxyr: "2024", procname: "CCAOVALUE", reascd: "28"} - input: ref("ccao.aprval_reascd") rows: - {reascd: "28"} + - input: ref("ccao.vw_time_util") + rows: + - {date_today: "2024-01-01"} expect: rows: - {mailed_class: "211A"} + - name: default_vw_pin_value_stage_name_matches_procname + description: stage_name should match the set of procnames for a PIN + model: default.vw_pin_value + given: + - input: source("iasworld", "asmt_all") + rows: + # `cur` and `procname` are the important fields that determine the + # stage, everything else in these inputs is just for joins + - {parid: "pre-mailed", taxyr: "2024", cur: "Y", procname: null, rolltype: "RP", valasm3: 10} + - {parid: "mailed", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10} + # Pre-certified needs an extra row so that the history filter can see + # that there has already been a mailed value + - {parid: "pre-certified", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10} + - {parid: "pre-certified", taxyr: "2024", cur: "Y", procname: null, rolltype: "RP", valasm3: 10} + - {parid: "ccao-certified", taxyr: "2024", procname: "CCAOFINAL", rolltype: "RP", valasm3: 10} + - {parid: "bor-certified", taxyr: "2024", procname: "BORVALUE", rolltype: "RP", valasm3: 10} + # Reason codes are not important, and we only include them to complete + # the joins that are required to construct the view + - input: source("iasworld", "aprval") + rows: + - {parid: "pre-mailed", taxyr: "2024", procname: null, reascd: "28"} + - {parid: "mailed", taxyr: "2024", procname: "CCAOVALUE", reascd: "28"} + - {parid: "pre-certified", taxyr: "2024", procname: null, reascd: "28"} + - {parid: "ccao-certified", taxyr: "2024", procname: "CCAOFINAL", reascd: "28"} + - {parid: "bor-certified", taxyr: "2024", procname: "BORVALUE", reascd: "28"} + - input: ref("ccao.aprval_reascd") + rows: + - {reascd: "28"} + - input: ref("ccao.vw_time_util") + rows: + # This mock is important, since otherwise the view's dynamic + # reference to the current year could become out of date with this + # test + - {date_today: "2024-01-01"} + expect: + rows: + - {stage_name: "PRE-MAILED"} + - {stage_name: "MAILED"} + - {stage_name: "ASSESSOR PRE-CERTIFIED"} + - {stage_name: "ASSESSOR CERTIFIED"} + - {stage_name: "BOARD CERTIFIED"} diff --git a/dbt/models/reporting/reporting.vw_top_5_muni.sql b/dbt/models/reporting/reporting.vw_top_5_muni.sql index 1b9c4a4c8..4d5bcd569 100644 --- a/dbt/models/reporting/reporting.vw_top_5_muni.sql +++ b/dbt/models/reporting/reporting.vw_top_5_muni.sql @@ -63,8 +63,11 @@ most_recent_values AS ( pin, year, oneyr_pri_board_tot AS prior_bor_av, - COALESCE(certified_tot, mailed_tot) AS ccao_av, + COALESCE(certified_tot, mailed_tot, pre_mailed_tot) AS ccao_av, CASE + WHEN + (certified_tot IS NULL AND mailed_tot IS NULL) + THEN 'pre-mailed' WHEN certified_tot IS NULL THEN 'mailed' ELSE 'certified' END AS ccao_stage_used, diff --git a/dbt/models/reporting/schema.yml b/dbt/models/reporting/schema.yml index a2d0e44d7..949f31ac9 100644 --- a/dbt/models/reporting/schema.yml +++ b/dbt/models/reporting/schema.yml @@ -344,6 +344,16 @@ models: - name: reporting.vw_top_5_muni description: '{{ doc("view_vw_top_5_muni") }}' + columns: + - name: ccao_stage_used + data_tests: + - accepted_values: + name: reporting_vw_top_5_muni_ccao_stage_used_certified_in_prior_years + values: + - certified + config: + where: > + CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} data_tests: # It would be nice to move hyphen checks to unit tests, but the fixture # definitions would be extremely complicated, so for now we leave them