Add raw price threshold for sales val

glue/flagging_script_glue/flagging.py

@@ -21,6 +21,7 @@ def go(
     iso_forest_cols: list,
     dev_bounds: tuple,
     condos: bool,
+    raw_price_thresh: int,
 ):
     """
     This function runs all of our other functions in the correct sequence.
@@ -50,7 +51,9 @@ def go(
     print("string_processing() done")
     df = iso_forest(df, groups, iso_forest_cols)
     print("iso_forest() done")
-    df = outlier_taxonomy(df, dev_bounds, groups, condos=condos)
+    df = outlier_taxonomy(
+        df, dev_bounds, groups, condos=condos, raw_price_thresh=raw_price_thresh
+    )
     print("outlier_taxonomy() done\nfinished")
 
     return df
@@ -69,7 +72,9 @@ def create_group_string(groups: tuple, sep: str) -> str:
     return sep.join(groups)
 
 
-def outlier_taxonomy(df: pd.DataFrame, permut: tuple, groups: tuple, condos: bool):
+def outlier_taxonomy(
+    df: pd.DataFrame, permut: tuple, groups: tuple, condos: bool, raw_price_thresh: int
+):
     """
     Creates columns having to do with our chosen outlier taxonomy.
     Ex: Family sale, Home flip sale, Non-person sale, High price (raw and or sqft), etc.
@@ -84,7 +89,7 @@ def outlier_taxonomy(df: pd.DataFrame, permut: tuple, groups: tuple, condos: boo
 
     df = check_days(df, SHORT_TERM_OWNER_THRESHOLD)
     df = pricing_info(df, permut, groups, condos=condos)
-    df = outlier_type(df, condos=condos)
+    df = outlier_type(df, condos=condos, raw_price_thresh=raw_price_thresh)
 
     return df
 
@@ -740,7 +745,7 @@ def z_normalize_groupby(s: pd.Series):
     return zscore(s, nan_policy="omit")
 
 
-def outlier_type(df: pd.DataFrame, condos: bool) -> pd.DataFrame:
+def outlier_type(df: pd.DataFrame, condos: bool, raw_price_thresh: int) -> pd.DataFrame:
     """
     This function create indicator columns for each distinct outlier type between price
     and characteristic outliers. These columns are prefixed with 'sv_ind_'.
@@ -807,6 +812,10 @@ def outlier_type(df: pd.DataFrame, condos: bool) -> pd.DataFrame:
             "sv_ind_price_low_price_sqft",
         ]
 
+    # Implement raw threshold, unlog  price
+    price_conditions.append((10 ** df["meta_sale_price"]) > raw_price_thresh)
+    price_labels.append("sv_ind_raw_price_threshold")
+
     combined_conditions = price_conditions + char_conditions
     combined_labels = price_labels + char_labels
 

glue/sales_val_flagging.py

@@ -144,7 +144,8 @@ def classify_outliers(df, stat_groups: list, min_threshold):
     2. Implement our group threshold requirement. In the statistical flagging process, if
     the group a sale belongs too is below N=30 then we want to manually set these flags to
     non-outlier status, even if they were flagged in the mansueto script. This requirement
-    is bypasses for ptax outliers - we don't care about group threshold in this case.
+    is bypasses for ptax outliers and raw price threshold outliers - we don't care about
+    group threshold in this case.
 
     Inputs:
         df: The data right after we perform the flagging script (go()), when the exploded
@@ -178,6 +179,7 @@ def classify_outliers(df, stat_groups: list, min_threshold):
         "sv_ind_ptax_flag_w_high_price_sqft": "High price per square foot",
         "sv_ind_price_low_price_sqft": "Low price per square foot",
         "sv_ind_ptax_flag_w_low_price_sqft": "Low price per square foot",
+        "sv_ind_raw_price_threshold": "Raw price threshold",
         "sv_ind_ptax_flag": "PTAX-203 Exclusion",
         "sv_ind_char_short_term_owner": "Short-term owner",
         "sv_ind_char_family_sale": "Family Sale",
@@ -199,6 +201,11 @@ def classify_outliers(df, stat_groups: list, min_threshold):
 
     Note: This doesn't apply for sales that also have a ptax outlier status.
           In this case, we still assign the price outlier status.
+
+          We also don't apply this threshold with sv_raw_price_threshold,
+          since this is design to be a safeguard that catches very high price
+          sales that may have slipped through the cracks due to the group
+          threshold requirement
     """
     group_thresh_price_fix = [
         "sv_ind_price_high_price",
@@ -237,12 +244,14 @@ def fill_outlier_reasons(row):
     # Drop the _merge column
     df = df.drop(columns=["_merge"])
 
-    # Assign outlier status
+    # Assign outlier status, these are the outlier types
+    # that assign a sale as an outlier
     values_to_check = {
         "High price",
         "Low price",
         "High price per square foot",
         "Low price per square foot",
+        "Raw price threshold",
     }
 
     df["sv_is_outlier"] = np.where(

manual_flagging/yaml/inputs.yaml

@@ -163,3 +163,7 @@ ptax_sd: [1, 1]
 
 # Flags are only applied if there are at least this many sales in the group
 min_groups_threshold: 30
+
+# This is the raw price threshold that is used to set sales to outlier status
+# regardless of group size
+raw_price_thresh: 1500000