{"status":"ok","backtest":{"cutoff":"2024-01-01","score_version":"1.1.0","outcome":"the door pulled a work permit (any type but demolition) or sold at arm's length (price > $1,000) after the cutoff","outcome_window":{"after":"2024-01-01","until":null,"permits_through":"2026-08-03","deeds_through":"2024-10-31"},"excluded_built_after_cutoff":1,"n_doors":541,"n_positive":156,"base_rate":0.2884,"model":{"top_decile_lift":{"value":1.3241,"ci95":[0.9,1.7186]},"top_decile_hit_rate":{"value":0.3818,"ci95":[0.2545,0.5091]},"top20_hit_rate":{"value":0.35,"ci95":[0.15,0.6]},"spearman_rho":{"value":0.0176,"ci95":[-0.0612,0.1031]}},"baseline_home_age":{"top_decile_lift":{"value":0.8197,"ci95":[0.473,1.1943]},"top_decile_hit_rate":{"value":0.2364,"ci95":[0.1282,0.3546]},"top20_hit_rate":{"value":0.2104,"ci95":[0.0657,0.3629]},"spearman_rho":{"value":0.0097,"ci95":[-0.0772,0.0942]}},"model_minus_baseline_lift":{"value":0.5044,"ci95":[-0.0377,0.9942]},"score_1_0_0":{"version":"1.0.0","model":{"top_decile_lift":{"value":0.9458,"ci95":[0.5786,1.3476]},"top_decile_hit_rate":{"value":0.2727,"ci95":[0.1636,0.4]},"top20_hit_rate":{"value":0.35,"ci95":[0.1,0.55]},"spearman_rho":{"value":0.0038,"ci95":[-0.0825,0.0845]}},"model_minus_1_0_0_lift":{"value":0.3783,"ci95":[-0.1553,0.8607]},"verdict":"1.0.0 top-decile lift 0.95 (95% CI 0.58 to 1.35) is not above 1: it does not beat random."},"excluding_doors_sold_after_cutoff":{"n_doors":521,"base_rate":0.2687,"top_decile_lift":1.2639,"top_decile_hit_rate":0.3396,"top20_hit_rate":0.35,"spearman_rho":0.0135},"door_list_only":{"n_doors":541,"base_rate":0.2884,"top_decile_lift":1.3241,"top_decile_hit_rate":0.3818,"top20_hit_rate":0.35,"spearman_rho":0.0176},"bootstrap":{"resamples":1000,"seed":20240101,"method":"percentile"},"verdict":"Top-decile lift 1.32 (95% CI 0.90 to 1.72) is above 1 but the interval includes 1: suggestive, not proven. Home-age baseline lift 0.82; score minus baseline +0.50 (95% CI -0.04 to +0.99).","notes":["Features use only data dated on or before the cutoff; outcomes use only data after it.","MOD-IV is a current snapshot: for doors sold after the cutoff the owner fields (deed, price, absentee) are hidden as unknown, and the sensitivity block drops those doors.","The absentee flag is a fact of the 2026 MOD-IV snapshot with no date of its own, so it is hidden for any cutoff before the snapshot date: nobody is on the landlord list at the cutoff and door_list_only is every door.","Doors whose MOD-IV year built is after the cutoff did not stand then and are left out.","Vision signals count only when their imagery date is on or before the cutoff; missing ones are neutral.","door_list_only scores the owner-occupied doors a rep would knock (the landlord list is ranked separately).","Permits feed both a feature (before) and the outcome (after): repeat renovation is a real pattern, which is why the home-age baseline is reported alongside.","Doors with a vision-based reason at the cutoff: 255 of 541."]},"calibration":{"score_version":"1.1.0","outcome":"the door pulled a work permit (any type but demolition) or sold at arm's length (price > $1,000) after the cutoff","method":"L2 logistic regression (numpy, damped Newton) of the outcome on per-door log factors, standardised; the coefficient on each raw log factor is that factor's exponent: score 1.1.0 = prod(factor ** exponent)","quality_gate":{"validated":[],"metrics":{"pool":{"precision":0.782608695652174,"n_pred_yes":46},"solar":{"precision":1.0,"n_pred_yes":4},"trampoline_playset":{"precision":null,"n_pred_yes":0},"pool_added":{"precision":0.34782608695652173,"n_pred_yes":23},"deck_patio":{"precision":1.0,"n_pred_yes":16},"lawn_maintained":{"precision":1.0,"n_pred_yes":41},"tree_overhang_roof":{"precision":0.489,"n_pred_yes":88},"roof_condition_poor":{"precision":null,"n_pred_yes":0},"long_driveway":{"precision":0.333,"n_pred_yes":21}}},"train_window":{"n_doors":541,"n_positive":140,"n_positive_with_sale":33,"base_rate":0.2588,"owner_hidden_in_fit":64,"excluded_built_after_cutoff":1},"cv":{"folds":5,"seed":20220101,"metric":"mean held-out log loss (lower is better)","grid":[{"l2":0.01,"log_loss":0.58246,"top_decile_lift":1.6862},{"l2":0.1,"log_loss":0.58243,"top_decile_lift":1.6862},{"l2":0.3,"log_loss":0.58236,"top_decile_lift":1.6862},{"l2":1.0,"log_loss":0.58213,"top_decile_lift":1.6862},{"l2":3.0,"log_loss":0.58151,"top_decile_lift":1.6862},{"l2":10.0,"log_loss":0.57978,"top_decile_lift":1.616},{"l2":30.0,"log_loss":0.57692,"top_decile_lift":1.616},{"l2":100.0,"log_loss":0.57391,"top_decile_lift":1.4755},{"l2":300.0,"log_loss":0.57288,"top_decile_lift":1.3349},{"l2":1000.0,"log_loss":0.57283,"top_decile_lift":1.1944}],"intercept_only_log_loss":0.57298,"chosen_l2":1000.0},"intercept":-1.0871,"exponents":{"new_mover":-0.0851,"tenure":-0.1448,"absentee":0.0,"permits":0.2567,"home_age":0.2669,"pool_added":0.0,"nbr_permit_share":-0.0295,"nbr_recent_sale_share":-0.5092,"v":0.0559,"density":0.0286},"coefficients":{"new_mover":{"exponent":-0.0851,"ci95":[-0.2228,0.0613],"per_sd":-0.0101,"doors_with_factor":51,"ci_includes_0":true},"tenure":{"exponent":-0.1448,"ci95":[-0.5494,0.2857],"per_sd":-0.0062,"doors_with_factor":151,"ci_includes_0":true},"permits":{"exponent":0.2567,"ci95":[0.0199,0.5265],"per_sd":0.0197,"doors_with_factor":40,"ci_includes_0":false},"home_age":{"exponent":0.2669,"ci95":[-0.5257,1.0163],"per_sd":0.0057,"doors_with_factor":512,"ci_includes_0":true},"nbr_permit_share":{"exponent":-0.0295,"ci95":[-0.45,0.363],"per_sd":-0.0013,"doors_with_factor":251,"ci_includes_0":true},"nbr_recent_sale_share":{"exponent":-0.5092,"ci95":[-1.0961,0.0886],"per_sd":-0.0152,"doors_with_factor":268,"ci_includes_0":true},"v":{"exponent":0.0559,"ci95":[-0.0566,0.1682],"per_sd":0.0094,"doors_with_factor":529,"ci_includes_0":true},"density":{"exponent":0.0286,"ci95":[-0.1906,0.2319],"per_sd":0.0025,"doors_with_factor":541,"ci_includes_0":true},"absentee":{"exponent":0.0,"doors_with_factor":0},"pool_added":{"exponent":0.0,"doors_with_factor":0}},"coefficient_bootstrap":{"resamples":1000,"seed":20220101,"method":"percentile over doors"},"test_window":{"window":{"outcome_until":null,"permits_through":"2026-08-03","deeds_through":"2024-10-31"},"excluded_built_after_cutoff":1,"n_doors":541,"n_positive":156,"base_rate":0.2884,"v1_1":{"top_decile_lift":{"value":1.3241,"ci95":[0.9,1.7186]},"top_decile_hit_rate":{"value":0.3818,"ci95":[0.2545,0.5091]},"top20_hit_rate":{"value":0.35,"ci95":[0.15,0.6]},"spearman_rho":{"value":0.0176,"ci95":[-0.0612,0.1031]}},"v1_0":{"top_decile_lift":{"value":0.9458,"ci95":[0.5786,1.3476]},"top_decile_hit_rate":{"value":0.2727,"ci95":[0.1636,0.4]},"top20_hit_rate":{"value":0.35,"ci95":[0.1,0.55]},"spearman_rho":{"value":0.0038,"ci95":[-0.0825,0.0845]}},"baseline_home_age":{"top_decile_lift":{"value":0.8197,"ci95":[0.473,1.1943]},"top_decile_hit_rate":{"value":0.2364,"ci95":[0.1282,0.3546]},"top20_hit_rate":{"value":0.2104,"ci95":[0.0657,0.3629]},"spearman_rho":{"value":0.0097,"ci95":[-0.0772,0.0942]}},"v1_1_minus_v1_0_lift":{"value":0.3783,"ci95":[-0.1553,0.8607]},"v1_1_minus_home_age_lift":{"value":0.5044,"ci95":[-0.0377,0.9942]},"excluding_doors_sold_after_cutoff":{"n_doors":521,"v1_1":{"base_rate":0.2687,"top_decile_lift":1.2639,"top_decile_hit_rate":0.3396,"top20_hit_rate":0.35,"spearman_rho":0.0135},"v1_0":{"base_rate":0.2687,"top_decile_lift":0.983,"top_decile_hit_rate":0.2642,"top20_hit_rate":0.35,"spearman_rho":0.0155}},"bootstrap":{"resamples":1000,"seed":20240101,"method":"percentile, paired over doors"}},"test_looks":[{"validated":[],"exponents":{"new_mover":-0.1026,"tenure":1.3697,"absentee":0.4016,"permits":1.4177,"home_age":1.8037,"pool_added":0.0,"nbr_permit_share":0.1905,"nbr_recent_sale_share":-4.0639,"v":0.6832,"density":0.1813},"v1_1_lift":{"value":1.0671,"ci95":[0.6236,1.4259]},"v1_0_lift":{"value":0.8787,"ci95":[0.5354,1.3052]},"v1_1_lift_not_sold":1.0532,"v1_0_lift_not_sold":0.983,"met_rule":true,"look":1},{"validated":[],"exponents":{"new_mover":-0.0973,"tenure":1.3703,"absentee":0.3981,"permits":1.4094,"home_age":2.498,"pool_added":0.0,"nbr_permit_share":0.1929,"nbr_recent_sale_share":-4.0701,"v":0.3397,"density":0.1674},"v1_1_lift":{"value":1.1926,"ci95":[0.739,1.5719]},"v1_0_lift":{"value":1.0043,"ci95":[0.6045,1.3572]},"v1_1_lift_not_sold":1.1937,"v1_0_lift_not_sold":1.0532,"met_rule":true,"look":2},{"status":"run","look":3,"validated":[],"exponents":{"new_mover":-0.0851,"tenure":-0.1448,"absentee":1.0,"permits":0.2567,"home_age":0.2669,"pool_added":0.0,"nbr_permit_share":-0.0295,"nbr_recent_sale_share":-0.5092,"v":0.0559,"density":0.0286},"v1_1_lift":{"value":1.2295,"ci95":[0.8771,1.7117]},"v1_0_lift":{"value":0.9458,"ci95":[0.6034,1.3785]},"v1_1_lift_not_sold":1.1937,"v1_0_lift_not_sold":0.983,"met_rule":true},{"status":"run","look":4,"validated":[],"exponents":{"new_mover":-0.0851,"tenure":-0.1448,"absentee":1.0,"permits":0.2567,"home_age":0.2669,"pool_added":0.0,"nbr_permit_share":-0.0295,"nbr_recent_sale_share":-0.5092,"v":0.0559,"density":0.0286},"v1_1_lift":{"value":1.3241,"ci95":[0.9,1.7186]},"v1_0_lift":{"value":0.9458,"ci95":[0.5786,1.3476]},"v1_1_lift_not_sold":1.2639,"v1_0_lift_not_sold":0.983,"met_rule":true}],"verdict":"1.1.0 top-decile lift 1.32 (95% CI 0.90 to 1.72) is above 1 but the interval includes 1: suggestive, not proven. 1.0.0 top-decile lift 0.95 (95% CI 0.58 to 1.35) is not above 1: it does not beat random. 1.1.0 minus 1.0.0: +0.38 (95% CI -0.16 to +0.86). 1.1.0 met the pre-declared rule and becomes the deployed default. The test window has been scored 4 times (each look declared before it ran); 4 met the rule, and 1.1.0 needs every one.","limitations":[]},"rerun":"python -m otn.backtest"}