Fixup tests and views for the 2025 rollover

dbt/dbt_project.yml

@@ -29,7 +29,7 @@ vars:
   # year, since errors in past data cannot usually be amended once records are
   # closed. Set as an integer for compatibility with comparison operators and
   # SQL's BETWEEN
-  data_test_iasworld_year_start: 2024
+  data_test_iasworld_year_start: 2025
 
   # End year for iasWorld data tests. Typically set to a date in the future,
   # but can also be use to select specific time frames for testing
@@ -43,6 +43,8 @@ models:
     +write_compression: zstd
     +format: parquet
     +ha: true
+    ccao:
+      +schema: ccao
     census:
       +schema: census
     default:

dbt/macros/pre_stage_filters.sql

@@ -1,21 +1,27 @@
 -- Macros to simplify the filter conditions that produce pre-mailed and
--- pre-certified values.
+-- pre-certified values in the `default.vw_pin_value` view.
 --
 -- Note that the `cur = 'Y'` filter is not necessary in the
 -- default.vw_pin_value query that is the main consumer of these macros, since
--- that query already filters rows where `cur = 'Y'`; however, we leave the
--- filter in here so that it might make the macro more generally useful in
--- other instances where we may not filter queries the same way
-{% macro pre_stage_filters(tablename, stage_name) %}
+-- that query already filters rows where `cur = 'Y'`.
+--
+-- A PIN is in the pre-mailed stage for a year if none of its representations
+-- in `asmt_all` in that year have procnames (i.e. stage names). We can check
+-- for this status by checking the cardinality of the `stages.procnames` array,
+-- where "cardinality" is the Trino SQL equivalent of the length of the array
+{% macro pre_mailed_filters(tablename) %}
     {{ tablename }}.procname is null
     and {{ tablename }}.cur = 'Y'
-    and not contains(stages.procnames, '{{ stage_name }}')
-{% endmacro %}
-
-{% macro pre_mailed_filters(tablename) %}
-    {{- pre_stage_filters(tablename, "CCAOVALUE") -}}
+    and cardinality(stages.procnames) = 0
 {% endmacro %}
 
+-- A PIN is the pre-certified stage for a year if it has at least one
+-- representation in `asmt_all` with a procname (i.e. stage name), but none
+-- of its representations in `asmt_all` have the procname that corresponds to
+-- the Assessor Certified stage
 {% macro pre_certified_filters(tablename) %}
-    {{- pre_stage_filters(tablename, "CCAOFINAL") -}}
+    {{ tablename }}.procname is null
+    and {{ tablename }}.cur = 'Y'
+    and cardinality(stages.procnames) > 0
+    and not contains(stages.procnames, 'CCAOFINAL')
 {% endmacro %}

dbt/macros/tests/test_all.sql

@@ -5,6 +5,5 @@
     {% do test_generate_schema_name() %}
     {% do test_generate_alias_name() %}
     {% do test_format_additional_select_columns() %}
-    {% do test_pre_stage_filters() %}
     {% do test_insert_hyphens() %}
 {% endmacro %}

dbt/models/ccao/ccao.vw_time_util.sql

@@ -0,0 +1,4 @@
+-- View that supplies dynamic datetime values in a way that we can mock for
+-- unit tests. See:
+-- https://discourse.getdbt.com/t/dynamic-dates-in-unit-tests/16883/2
+SELECT CURRENT_DATE AS date_today

dbt/models/ccao/docs.md

@@ -97,3 +97,12 @@ Collected yearly from Valuations via spreadsheets.
 
 **Primary Key**: `pin`, `year`
 {% enddocs %}
+
+# vw_time_util
+
+{% docs view_vw_time_util %}
+View that supplies dynamic datetime values in a way that we can mock for unit
+tests.
+
+See: <https://discourse.getdbt.com/t/dynamic-dates-in-unit-tests/16883/2?>
+{% enddocs %}

dbt/models/ccao/schema.yml

@@ -1,3 +1,7 @@
+models:
+  - name: ccao.vw_time_util
+    description: '{{ doc("view_vw_time_util") }}'
+
 sources:
   - name: ccao
     tags:

dbt/models/default/default.vw_pin_history.sql

@@ -6,6 +6,12 @@ SELECT
     leg.user1 AS township_code,
     town.township_name,
     REGEXP_REPLACE(par.nbhd, '([^0-9])', '') AS nbhd,
+    -- Add pre-mailed values to cover us for the period after the rollover but
+    -- before the year's assessment cycle starts
+    vwpv.pre_mailed_class,
+    vwpv.pre_mailed_bldg,
+    vwpv.pre_mailed_land,
+    vwpv.pre_mailed_tot,
     vwpv.mailed_class,
     vwpv.mailed_bldg,
     vwpv.mailed_land,

dbt/models/default/default.vw_pin_value.sql

@@ -16,10 +16,25 @@ WITH stages AS (
     SELECT
         parid,
         taxyr,
-        ARRAY_AGG(procname) AS procnames
+        -- Force an empty array representing the procnames for any PIN/year
+        -- combo that has no procnames yet, so that our `CONTAINS()` check
+        -- in subsequent queries that join to this stage do not have to worry
+        -- about side effects from null comparisons
+        ARRAY_REMOVE(
+            ARRAY_AGG(
+                CASE
+                    WHEN procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE')
+                        THEN procname
+                    -- Can't use null to indicate missing data here, since a
+                    -- null comparison in the outer `ARRAY_REMOVE` call would
+                    -- always cast the array to null
+                    ELSE ''
+                END
+            ),
+            ''
+        ) AS procnames
     FROM {{ source('iasworld', 'asmt_all') }}
-    WHERE procname IN ('CCAOVALUE', 'CCAOFINAL', 'BORVALUE')
-        AND rolltype != 'RR'
+    WHERE rolltype != 'RR'
         AND deactivat IS NULL
         AND valclass IS NULL
     GROUP BY parid, taxyr
@@ -401,7 +416,10 @@ stage_values AS (
                 -- stages, it is most likely a provisional value for a PIN
                 -- that has not mailed yet
                 CARDINALITY(stages.procnames) != 0
-                OR asmt.taxyr = DATE_FORMAT(NOW(), '%Y')
+                OR asmt.taxyr = DATE_FORMAT(
+                    (SELECT date_today FROM {{ ref("ccao.vw_time_util") }}),
+                    '%Y'
+                )
             )
         )
     )

dbt/models/default/schema/default.vw_pin_history.yml

@@ -53,6 +53,14 @@ models:
         description: '{{ doc("shared_column_mailed_tot") }}'
       - name: pin
         description: '{{ doc("shared_column_pin") }}'
+      - name: pre_mailed_bldg
+        description: '{{ doc("shared_column_pre_mailed_bldg") }}'
+      - name: pre_mailed_class
+        description: '{{ doc("shared_column_pre_mailed_class") }}'
+      - name: pre_mailed_land
+        description: '{{ doc("shared_column_pre_mailed_land") }}'
+      - name: pre_mailed_tot
+        description: '{{ doc("shared_column_pre_mailed_tot") }}'
       - name: township_code
         description: '{{ doc("shared_column_township_code") }}'
       - name: township_name

dbt/models/default/schema/default.vw_pin_value.yml

@@ -118,28 +118,31 @@ models:
           - `3` = Board certified
       - name: year
         description: '{{ doc("shared_column_year") }}'
+        data_tests:
+          - not_null:
+              name: default_vw_pin_value_stage_num_not_null
 
     data_tests:
     - not_null:
         name: default_vw_pin_value_board_class_not_null
         column_name: board_class
         config:
           where: CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1
-          error_if: ">1260"
+          error_if: ">1261"
     - not_null:
         name: default_vw_pin_value_board_tot_mv_not_null
         column_name: board_tot_mv
         config:
           where: |
             CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1 AND
             year >= '2020'
-          error_if: ">1260"
+          error_if: ">1261"
     - not_null:
         name: default_vw_pin_value_board_tot_not_null
         column_name: board_tot
         config:
           where: CAST(year AS int) < {{ var('data_test_iasworld_year_start') }} - 1
-          error_if: ">1260"
+          error_if: ">1261"
     - not_null:
         name: default_vw_pin_value_certified_class_not_null
         column_name: certified_class
@@ -226,12 +229,58 @@ unit_tests:
         # provided to ensure proper joins when creating the dummy table
         rows:
           - {parid: "123", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10, class: "2.1-1)A"}
+      # The input tables below are not important, and are only included for the
+      # sake of completing the joins that construct the view
       - input: source("iasworld", "aprval")
         rows:
           - {parid: "123", taxyr: "2024", procname: "CCAOVALUE", reascd: "28"}
       - input: ref("ccao.aprval_reascd")
         rows:
           - {reascd: "28"}
+      - input: ref("ccao.vw_time_util")
+        rows:
+          - {date_today: "2024-01-01"}
     expect:
       rows:
         - {mailed_class: "211A"}
+  - name: default_vw_pin_value_stage_name_matches_procname
+    description: stage_name should match the set of procnames for a PIN
+    model: default.vw_pin_value
+    given:
+      - input: source("iasworld", "asmt_all")
+        rows:
+          # `cur` and `procname` are the important fields that determine the
+          # stage, everything else in these inputs is just for joins
+          - {parid: "pre-mailed", taxyr: "2024", cur: "Y", procname: null, rolltype: "RP", valasm3: 10}
+          - {parid: "mailed", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10}
+          # Pre-certified needs an extra row so that the history filter can see
+          # that there has already been a mailed value
+          - {parid: "pre-certified", taxyr: "2024", procname: "CCAOVALUE", rolltype: "RP", valasm3: 10}
+          - {parid: "pre-certified", taxyr: "2024", cur: "Y", procname: null, rolltype: "RP", valasm3: 10}
+          - {parid: "ccao-certified", taxyr: "2024", procname: "CCAOFINAL", rolltype: "RP", valasm3: 10}
+          - {parid: "bor-certified", taxyr: "2024", procname: "BORVALUE", rolltype: "RP", valasm3: 10}
+      # Reason codes are not important, and we only include them to complete
+      # the joins that are required to construct the view
+      - input: source("iasworld", "aprval")
+        rows:
+          - {parid: "pre-mailed", taxyr: "2024", procname: null, reascd: "28"}
+          - {parid: "mailed", taxyr: "2024", procname: "CCAOVALUE", reascd: "28"}
+          - {parid: "pre-certified", taxyr: "2024", procname: null, reascd: "28"}
+          - {parid: "ccao-certified", taxyr: "2024", procname: "CCAOFINAL", reascd: "28"}
+          - {parid: "bor-certified", taxyr: "2024", procname: "BORVALUE", reascd: "28"}
+      - input: ref("ccao.aprval_reascd")
+        rows:
+          - {reascd: "28"}
+      - input: ref("ccao.vw_time_util")
+        rows:
+          # This mock is important, since otherwise the view's dynamic
+          # reference to the current year could become out of date with this
+          # test
+          - {date_today: "2024-01-01"}
+    expect:
+      rows:
+        - {stage_name: "PRE-MAILED"}
+        - {stage_name: "MAILED"}
+        - {stage_name: "ASSESSOR PRE-CERTIFIED"}
+        - {stage_name: "ASSESSOR CERTIFIED"}
+        - {stage_name: "BOARD CERTIFIED"}

dbt/models/reporting/reporting.vw_top_5_muni.sql

@@ -63,8 +63,11 @@ most_recent_values AS (
         pin,
         year,
         oneyr_pri_board_tot AS prior_bor_av,
-        COALESCE(certified_tot, mailed_tot) AS ccao_av,
+        COALESCE(certified_tot, mailed_tot, pre_mailed_tot) AS ccao_av,
         CASE
+            WHEN
+                (certified_tot IS NULL AND mailed_tot IS NULL)
+                THEN 'pre-mailed'
             WHEN certified_tot IS NULL THEN 'mailed'
             ELSE 'certified'
         END AS ccao_stage_used,

dbt/models/reporting/schema.yml

@@ -344,6 +344,16 @@ models:
 
   - name: reporting.vw_top_5_muni
     description: '{{ doc("view_vw_top_5_muni") }}'
+    columns:
+      - name: ccao_stage_used
+        data_tests:
+          - accepted_values:
+              name: reporting_vw_top_5_muni_ccao_stage_used_certified_in_prior_years
+              values:
+                - certified
+              config:
+                where: >
+                  CAST(year AS int) < {{ var('data_test_iasworld_year_start') }}
     data_tests:
       # It would be nice to move hyphen checks to unit tests, but the fixture
       # definitions would be extremely complicated, so for now we leave them