{"status":"ok","report":{"blind":{"census_meta":{"disputed_adjudicated_count":116,"inter_rater_cell_agreement":0.9106002554278416,"labeler":"claude-opus-blind-census","method":"two independent labellers per crop, images only (no access to signals, db, eval or code); a third labeller adjudicated every disagreement","other_signals":{"inter_rater_cell_agreement":0.8966666666666666,"labeler":"claude-opus-blind-census","model_vs_labels":{"deck_patio":{"fn":47,"fp":0,"label_cant_see_or_missing":29,"n_pred_yes":16,"positives":63,"precision":1.0,"precision_ci95":[0.806,1.0],"recall":0.254,"recall_ci95":[0.163,0.373],"tn":28,"tp":16},"lawn_maintained":{"fn":56,"fp":0,"label_cant_see_or_missing":19,"n_pred_yes":41,"positives":97,"precision":1.0,"precision_ci95":[0.914,1.0],"recall":0.423,"recall_ci95":[0.329,0.522],"tn":4,"tp":41},"long_driveway":{"fn":5,"fp":14,"label_cant_see_or_missing":9,"n_pred_yes":21,"positives":12,"precision":0.333,"precision_ci95":[0.172,0.546],"recall":0.583,"recall_ci95":[0.32,0.807],"tn":85,"tp":7},"roof_condition_poor":{"fn":0,"fp":0,"label_cant_see_or_missing":5,"n_pred_yes":0,"positives":0,"precision":null,"precision_ci95":[null,null],"recall":null,"recall_ci95":[null,null],"tn":115,"tp":0},"tree_overhang_roof":{"fn":2,"fp":45,"label_cant_see_or_missing":0,"n_pred_yes":88,"positives":45,"precision":0.489,"precision_ci95":[0.387,0.591],"recall":0.956,"recall_ci95":[0.852,0.988],"tn":30,"tp":43}},"per_signal_agreement":{"deck_patio":0.825,"lawn_maintained":0.9166666666666666,"long_driveway":0.875,"roof_condition_poor":0.9833333333333333,"tree_overhang_roof":0.8833333333333333},"year":2023},"year":2023},"doors":542,"labeler":"claude-opus-blind-census + claude-opus-blind","labels":1626,"labels_before_model_output":null,"method":"attested","sample_vs_census":{"cant_see_rate":null,"excluded":0,"f1":null,"fn":0,"fp":0,"n":0,"positives":0,"precision":null,"precision_ci":[0.0,1.0],"recall":null,"recall_ci":[0.0,1.0],"tn":0,"tp":0},"timestamp_blind_before_model_output":true,"timestamp_blind_doors":20},"cant_see_rate_all_signals":{"deck_patio":0.27490774907749077,"lawn_maintained":0.5719557195571956,"long_driveway":0.2822878228782288,"pool":0.03690036900369004,"roof_condition_poor":0.2767527675276753,"solar":0.0,"trampoline_playset":0.016605166051660517,"tree_overhang_roof":0.03505535055350553},"change":{"pool_added":{"cant_see_rate":0.033073929961089495,"excluded":0,"f1":0.5,"fn":1,"fp":15,"fp_split":{"label_2020_yes":9,"label_2023_no":6},"n":514,"on_pool_yes_doors":{"cant_see_rate":0.06976744186046512,"excluded":0,"f1":0.6153846153846153,"fn":1,"fp":9,"n":43,"positives":9,"precision":0.47058823529411764,"precision_ci":[0.26164801196088106,0.6903710268845051],"recall":0.8888888888888888,"recall_ci":[0.5649937852319399,0.9801096286728695],"tn":25,"tp":8},"pool_yes_doors":43,"pool_yes_doors_with_prev_label":43,"positives":9,"precision":0.34782608695652173,"precision_ci":[0.18811054636982238,0.5511002309310836],"predicted_yes":23,"recall":0.8888888888888888,"recall_ci":[0.5649937852319399,0.9801096286728695],"tn":490,"tp":8,"truth_doors":514}},"cost":{"crops":1562,"crops_escalated_share":0.9670710571923743,"doors":542,"escalation_share":0.3274647887323944,"ledger":[{"calls":46,"in_tokens":730377,"model":"claude-opus-5-5","out_tokens":33566,"usd":3.592828},{"calls":217,"in_tokens":479787,"model":"claude-haiku-5-5","out_tokens":46673,"usd":0.071315},{"calls":208,"in_tokens":289526,"model":"claude-sonnet-5-5","out_tokens":18374,"usd":0.762792},{"calls":45,"in_tokens":99495,"model":"claude-haiku-5-5","out_tokens":10042,"usd":0.01497},{"calls":21,"in_tokens":27154,"model":"claude-sonnet-5-5","out_tokens":1566,"usd":0.069968},{"calls":2885,"in_tokens":6378735,"model":"claude-haiku-5-5","out_tokens":612699,"usd":0.944223},{"calls":2790,"in_tokens":3587573,"model":"claude-sonnet-5-5","out_tokens":201719,"usd":9.192336}],"ledger_total_usd":14.648433,"per_crop_usd":0.003517477080665813,"per_door_usd":0.010137083394833948,"projected_per_door_usd":0.01055243124199744,"signals_usd":5.494299,"survey_years":3},"dry_run":false,"eval_year":2023,"findings":["Probe offparcel_road_2 drew 'yes' for lawn_maintained.","Probe offparcel_road_4 drew 'yes' for lawn_maintained.","pool_added precision is 0.35 (8 of 23). Of the 15 false 'added' calls, 6 are doors with no pool in the 2023 labels (a 2023 false pool) and 9 already had a pool in the 2020 labels (the model missed the 2020 pool; 2020 pool recall 1.000 on 2 labelled pools in the seeded random sample). Treat pool_added as a lead, not evidence.","Escalation is high: 97% of crops sent at least one signal to Sonnet (32.7% of signal answers came from Sonnet), so Sonnet dominates cost. The 0.6 threshold was not tuned against the blind labels (they are the test set); tuning it needs a held-out split.","solar: only 5 blind positives, so its recall CI is wide [0.38, 0.96]; treat the point estimate as indicative.","trampoline_playset: recall 0.00 on 13 blind positives; the model rarely sees it."],"generated_at":"2026-10-23T15:25:54.147423+00:00","models":{"escalate_below":0.6,"escalation":"claude-sonnet-5-5"},"notes":[],"osm":{"blind_pools_also_in_osm":0,"in_territory_pool_labels":0,"recall_set":{"cant_see_rate":0.009900990099009901,"found":85,"missed":[],"n":101,"pools":101,"recall":0.8415841584158416,"recall_ci":[0.7580624179833655,0.9000732838754386],"skipped_no_imagery":[],"uncached":0}},"prev_year":{"signals":{"pool":{"cant_see_rate":0.0,"excluded":0,"f1":0.8,"fn":0,"fp":1,"n":40,"positives":2,"precision":0.6666666666666666,"precision_ci":[0.2076549551264879,0.9385096847238394],"recall":1.0,"recall_ci":[0.34237195288961925,1.0],"tn":37,"tp":2}},"year":2020},"probes":{"answers":304,"by_probe":{"blank_grey_128":{"answers":8,"cant_see":8,"yes":[]},"blank_grey_200":{"answers":8,"cant_see":8,"yes":[]},"blank_grey_60":{"answers":8,"cant_see":5,"yes":[]},"noise_1":{"answers":8,"cant_see":8,"yes":[]},"noise_2":{"answers":8,"cant_see":8,"yes":[]},"noise_3":{"answers":8,"cant_see":8,"yes":[]},"offparcel_field_1":{"answers":8,"cant_see":3,"yes":[]},"offparcel_field_2":{"answers":8,"cant_see":3,"yes":[]},"offparcel_field_3":{"answers":8,"cant_see":5,"yes":[]},"offparcel_field_4":{"answers":8,"cant_see":5,"yes":[]},"offparcel_field_5":{"answers":8,"cant_see":3,"yes":[]},"offparcel_field_6":{"answers":8,"cant_see":3,"yes":[]},"offparcel_field_7":{"answers":8,"cant_see":1,"yes":[]},"offparcel_field_8":{"answers":8,"cant_see":5,"yes":[]},"offparcel_parking_1":{"answers":8,"cant_see":4,"yes":[]},"offparcel_parking_10":{"answers":8,"cant_see":3,"yes":[]},"offparcel_parking_2":{"answers":8,"cant_see":4,"yes":[]},"offparcel_parking_3":{"answers":8,"cant_see":3,"yes":[]},"offparcel_parking_4":{"answers":8,"cant_see":2,"yes":[]},"offparcel_parking_5":{"answers":8,"cant_see":3,"yes":[]},"offparcel_parking_6":{"answers":8,"cant_see":5,"yes":[]},"offparcel_parking_7":{"answers":8,"cant_see":3,"yes":[]},"offparcel_parking_8":{"answers":8,"cant_see":3,"yes":[]},"offparcel_parking_9":{"answers":8,"cant_see":2,"yes":[]},"offparcel_road_1":{"answers":8,"cant_see":2,"yes":[]},"offparcel_road_2":{"answers":8,"cant_see":3,"yes":["lawn_maintained"]},"offparcel_road_3":{"answers":8,"cant_see":5,"yes":[]},"offparcel_road_4":{"answers":8,"cant_see":1,"yes":["lawn_maintained"]},"offparcel_road_5":{"answers":8,"cant_see":3,"yes":[]},"offparcel_road_6":{"answers":8,"cant_see":4,"yes":[]},"offparcel_road_7":{"answers":8,"cant_see":3,"yes":[]},"offparcel_road_8":{"answers":8,"cant_see":4,"yes":[]},"offparcel_water_10":{"answers":8,"cant_see":5,"yes":[]},"offparcel_water_2":{"answers":8,"cant_see":8,"yes":[]},"offparcel_water_3":{"answers":8,"cant_see":6,"yes":[]},"offparcel_water_5":{"answers":8,"cant_see":6,"yes":[]},"offparcel_water_7":{"answers":8,"cant_see":5,"yes":[]},"offparcel_water_9":{"answers":8,"cant_see":6,"yes":[]}},"hallucination_ci":[0.0018060048117778827,0.02366683576597379],"hallucination_rate":0.006578947368421052,"missing":[],"offparcel_probes":32,"probe_hallucination_ci":[0.014553902448536798,0.17285766881068843],"probe_hallucination_rate":0.05263157894736842,"probes":38,"probes_with_any_yes":2,"yes_answers":2},"prompt_version":"v1","signals":{"deck_patio":{"labeler":"claude-opus-blind-census","missing_predictions":0,"vs_blind":{"cant_see_rate":0.17582417582417584,"excluded":29,"f1":0.4050632911392405,"fn":47,"fp":0,"n":91,"positives":63,"precision":1.0,"precision_ci":[0.8063865817272802,1.0],"recall":0.25396825396825395,"recall_ci":[0.16280274583098153,0.37341422084845166],"tn":28,"tp":16}},"lawn_maintained":{"labeler":"claude-opus-blind-census","missing_predictions":0,"vs_blind":{"cant_see_rate":0.45544554455445546,"excluded":19,"f1":0.5942028985507246,"fn":56,"fp":0,"n":101,"positives":97,"precision":1.0,"precision_ci":[0.9143295511310926,1.0],"recall":0.422680412371134,"recall_ci":[0.3291646600667981,0.5220872042431691],"tn":4,"tp":41}},"long_driveway":{"labeler":"claude-opus-blind-census","missing_predictions":0,"vs_blind":{"cant_see_rate":0.15315315315315314,"excluded":9,"f1":0.4242424242424242,"fn":5,"fp":14,"n":111,"positives":12,"precision":0.3333333333333333,"precision_ci":[0.17194536432683688,0.5462692434278971],"recall":0.5833333333333334,"recall_ci":[0.3195073356553728,0.8067425380947535],"tn":85,"tp":7}},"pool":{"labeler":"claude-opus-blind-census + claude-opus-blind","missing_predictions":0,"vs_blind":{"cant_see_rate":0.021400778210116732,"excluded":28,"f1":0.8089887640449438,"fn":7,"fp":10,"n":514,"positives":43,"precision":0.782608695652174,"precision_ci":[0.6442603416081054,0.8773922538141925],"recall":0.8372093023255814,"recall_ci":[0.7002730466408176,0.9188347549714246],"tn":461,"tp":36},"vs_sample":{"cant_see_rate":0.10526315789473684,"excluded":1,"f1":1.0,"fn":0,"fp":0,"n":19,"positives":3,"precision":1.0,"precision_ci":[0.43849391955098227,1.0],"recall":1.0,"recall_ci":[0.43849391955098227,1.0],"tn":16,"tp":3}},"roof_condition_poor":{"labeler":"claude-opus-blind-census","missing_predictions":0,"vs_blind":{"cant_see_rate":0.23478260869565218,"excluded":5,"f1":null,"fn":0,"fp":0,"n":115,"positives":0,"precision":null,"precision_ci":[0.0,1.0],"recall":null,"recall_ci":[0.0,1.0],"tn":115,"tp":0}},"solar":{"labeler":"claude-opus-blind-census + claude-opus-blind","missing_predictions":0,"vs_blind":{"cant_see_rate":0.0,"excluded":11,"f1":0.888888888888889,"fn":1,"fp":0,"n":531,"positives":5,"precision":1.0,"precision_ci":[0.5100999795960008,1.0],"recall":0.8,"recall_ci":[0.3755282641185388,0.9637768390302125],"tn":526,"tp":4},"vs_sample":{"cant_see_rate":0.0,"excluded":1,"f1":null,"fn":0,"fp":0,"n":19,"positives":0,"precision":null,"precision_ci":[0.0,1.0],"recall":null,"recall_ci":[0.0,1.0],"tn":19,"tp":0}},"trampoline_playset":{"labeler":"claude-opus-blind-census + claude-opus-blind","missing_predictions":0,"vs_blind":{"cant_see_rate":0.004319654427645789,"excluded":79,"f1":0.0,"fn":13,"fp":0,"n":463,"positives":13,"precision":null,"precision_ci":[0.0,1.0],"recall":0.0,"recall_ci":[0.0,0.22810184305529166],"tn":450,"tp":0},"vs_sample":{"cant_see_rate":0.0,"excluded":1,"f1":0.0,"fn":1,"fp":0,"n":19,"positives":1,"precision":null,"precision_ci":[0.0,1.0],"recall":0.0,"recall_ci":[0.0,0.7934567085261071],"tn":18,"tp":0}},"tree_overhang_roof":{"labeler":"claude-opus-blind-census","missing_predictions":0,"vs_blind":{"cant_see_rate":0.025,"excluded":0,"f1":0.6466165413533835,"fn":2,"fp":45,"n":120,"positives":45,"precision":0.48863636363636365,"precision_ci":[0.38687686309205005,0.5913465128073245],"recall":0.9555555555555556,"recall_ci":[0.8517222081611939,0.9877261309595557],"tn":30,"tp":43}}},"target_detail":{"SC-003 hallucination rate <= 5% (probes with any yes)":{"min_offparcel":30,"missing":0,"offparcel_probes":32,"probes":38,"probes_with_any_yes":2,"value":0.05263157894736842},"SC-003 pool precision >= 0.85":{"n":46,"value":0.782608695652174},"SC-003 pool recall >= 0.80":{"n":43,"value":0.8372093023255814},"SC-004 cost per door <= $0.02 (all survey years)":{"n":542,"value":0.01055243124199744}},"targets":{"SC-003 hallucination rate <= 5% (probes with any yes)":false,"SC-003 pool precision >= 0.85":false,"SC-003 pool recall >= 0.80":true,"SC-004 cost per door <= $0.02 (all survey years)":true}},"ledger":{"rows":[{"step":"vision","model":"claude-sonnet-5-5","calls":2790,"in_tokens":3587573,"out_tokens":201719,"usd":9.192336},{"step":"discover","model":"claude-opus-5-5","calls":46,"in_tokens":730377,"out_tokens":33566,"usd":3.592828},{"step":"vision","model":"claude-haiku-5-5","calls":2885,"in_tokens":6378735,"out_tokens":612699,"usd":0.944223},{"step":"osm-recall","model":"claude-sonnet-5-5","calls":208,"in_tokens":289526,"out_tokens":18374,"usd":0.762792},{"step":"osm-recall","model":"claude-haiku-5-5","calls":217,"in_tokens":479787,"out_tokens":46673,"usd":0.071315},{"step":"probe","model":"claude-sonnet-5-5","calls":21,"in_tokens":27154,"out_tokens":1566,"usd":0.069968},{"step":"probe","model":"claude-haiku-5-5","calls":45,"in_tokens":99495,"out_tokens":10042,"usd":0.01497}],"total_usd":14.648433,"doors":542,"per_door_usd":0.027027,"cap_usd":15.0,"source":"eval/report.json","matches_spend_csv":true},"rerun":"make eval"}