..

analysis/generate_findings.py

@@ -0,0 +1,220 @@
+#!/usr/bin/env python3
+"""Turn the cheaper-twin index into publishable FINDINGS.
+
+Each finding is the single artifact the whole engine renders four ways (growth/README.md):
+an SEO page, an OG/unfurl card, a video storyboard, and a ≤3-filter deep-link CTA into the live map.
+This script curates the raw 415 twin pairs into a reviewed set of findings and emits:
+
+  analysis/out/findings/<slug>.json:  one machine-readable finding per page/video
+  analysis/out/findings_review.md:    human-readable sheet for the founder to eyeball + fix names
+
+Place labels come from analysis/place_names.json (APPROXIMATE, verify before publishing).
+
+Run:  source .venv/bin/activate && python analysis/generate_findings.py
+"""
+from __future__ import annotations
+
+import json
+import urllib.parse
+from pathlib import Path
+
+import polars as pl
+
+OUT = Path("analysis/out")
+FIND = OUT / "findings"
+SITE = "https://perfect-postcode.co.uk"
+ATTRIB = "Contains HM Land Registry data © Crown copyright and database right. Licensed under the Open Government Licence v3.0."
+SOURCES = "HM Land Registry · EPC (DLUHC) · Ofsted · DfT · ONS · Police.uk"
+TYPE_SINGULAR = {
+    "Flats/Maisonettes": "flat",
+    "Terraced": "terraced house",
+    "Semi-Detached": "semi-detached house",
+    "Detached": "detached house",
+}
+
+NAMES = json.loads(Path("analysis/place_names.json").read_text())
+
+
+def label(sector: str) -> dict:
+    outward = sector.split(" ")[0]
+    name = NAMES.get(outward)
+    return {"sector": sector, "name": name, "label": f"{name} ({sector})" if name else sector, "named": bool(name)}
+
+
+def sector_slug(sector: str) -> str:
+    return sector.lower().replace(" ", "-")
+
+
+def map_query(t: dict) -> str:
+    """A ≤3-filter deep link that frames the value: centre between the pair, cap £/sqm near the
+    cheaper twin, require a good secondary catchment. Reproducible by non-payers (DEMO_MAX_FILTERS=3)."""
+    mid_lat = round((t["pricey_lat"] + t["twin_lat"]) / 2, 5)
+    mid_lon = round((t["pricey_lon"] + t["twin_lon"]) / 2, 5)
+    psqm_cap = int(round(t["twin_psqm"] * 1.05, -2))
+    enc_psqm = urllib.parse.quote("Est. price per sqm", safe="")
+    enc_school = urllib.parse.quote("Good+ secondary school catchments", safe="")
+    # Two plain numeric filters (within the 3-filter demo cap): frames value + good schools and
+    # uses the well-understood filter=NAME:MIN:MAX form so the OG-card filter-name guard passes.
+    parts = [
+        f"lat={mid_lat}",
+        f"lon={mid_lon}",
+        "zoom=12.5",
+        f"filter={enc_psqm}:0:{psqm_cap}",
+        f"filter={enc_school}:1:11",
+    ]
+    return "&".join(parts)
+
+
+def twin_finding(t: dict) -> dict:
+    p, w = label(t["pricey_sector"]), label(t["twin_sector"])
+    typ = TYPE_SINGULAR.get(t["dominant_type"], "home")
+    slug = f"cheaper-twin/{sector_slug(t['pricey_sector'])}-vs-{sector_slug(t['twin_sector'])}"
+    q = map_query(t)
+    headline_name = f"{p['name']} vs {w['name']}" if p["named"] and w["named"] else f"{t['pricey_sector']} vs {t['twin_sector']}"
+    return {
+        "slug": slug,
+        "type": "cheaper_twin",
+        "page_path": f"/{slug}",
+        "title": f"{headline_name}: the same {typ}, about {t['gap_pct']:.0f}% cheaper per m²",
+        "hook": f"£{t['gap_on_90sqm']:,} less for an equivalent {typ}: same station, similar schools, ~{t['dist_km']}km apart",
+        "shocking_number": f"{t['gap_pct']:.0f}%",
+        "pricey": {**p, "est_psqm": t["pricey_psqm"], "n": t["pricey_n"]},
+        "twin": {**w, "est_psqm": t["twin_psqm"], "n": t["twin_n"]},
+        "stats": {
+            "gap_pct": t["gap_pct"],
+            "gap_per_sqm": t["gap_per_sqm"],
+            "gap_on_90sqm": t["gap_on_90sqm"],
+            "gap_on_avg_home": t["gap_on_avg_home"],
+            "dominant_type": t["dominant_type"],
+            "build_year": t["build_year"],
+            "good_secondary_catchments": t["good_secondary"],
+            "station_km": t["station_km"],
+            "distance_km": t["dist_km"],
+        },
+        "map_query": q,
+        "map_url": f"{SITE}/?{q}",
+        "og_image": f"{SITE}/api/screenshot?og=1&{q}",
+        "methodology": (
+            "Postcode sectors (e.g. N10 3) compared on estimated £/m² of floor space. A pair is only "
+            "called a 'twin' when the two sectors share the dominant property type, build era (±30y), "
+            "good-school catchment provision, station access, deprivation/tenure, education, age and "
+            "home size, so the price gap reflects a name premium, not a different kind of area. "
+            "Estimates, not valuations; aggregated to sector, never address-level."
+        ),
+        "needs_name_check": not (p["named"] and w["named"]),
+        "attribution": ATTRIB,
+        "sources": SOURCES,
+    }
+
+
+def curate(tw: pl.DataFrame) -> list[dict]:
+    rows = tw.to_dicts()
+    london = ("E", "EC", "WC", "W", "SW", "SE", "N", "NW")
+
+    def area(s):
+        import re
+
+        return re.match(r"^[A-Z]+", s).group(0)
+
+    family = [r for r in rows if r["dominant_type"] in ("Terraced", "Semi-Detached", "Detached")]
+    prime = sorted(rows, key=lambda r: r["gap_on_90sqm"], reverse=True)
+    regional_family = sorted(
+        [r for r in family if area(r["pricey_sector"]) not in london],
+        key=lambda r: r["gap_on_90sqm"],
+        reverse=True,
+    )
+    london_family = sorted(
+        [r for r in family if area(r["pricey_sector"]) in london],
+        key=lambda r: r["gap_on_90sqm"],
+        reverse=True,
+    )
+    # A spread: biggest national name premiums (PR) + relatable family-home twins (buyer video).
+    picks, seen = [], set()
+    for r in prime[:6] + regional_family[:8] + london_family[:6]:
+        key = r["pricey_sector"] + "|" + r["twin_sector"]
+        if key not in seen:
+            seen.add(key)
+            picks.append(r)
+    return picks
+
+
+def national_findings(idx: pl.DataFrame, facts: dict) -> list[dict]:
+    named = idx.with_columns(
+        pl.col("Sector").str.extract(r"^([A-Z]+[0-9][A-Z0-9]?)", 1).alias("outward")
+    )
+    best = facts["best_value_sector"]
+    dear = facts["dearest_sector"]
+    return [
+        {
+            "slug": "square-metres-per-100k",
+            "type": "national_table",
+            "page_path": "/square-metres-per-100k",
+            "title": "How many square metres £100,000 buys across England",
+            "shocking_number": f"{best['sqm_per_100k']:.0f} m² vs {dear['sqm_per_100k']:.0f} m²",
+            "hook": (
+                f"£100k buys ~{best['sqm_per_100k']:.0f} m² of floor space in {label(best['sector'])['label']} "
+                f"but only ~{dear['sqm_per_100k']:.0f} m² in {label(dear['sector'])['label']}"
+            ),
+            "stats": {"best": best, "dearest": dear, "n_sectors": facts["n_sectors"]},
+            "map_query": "zoom=6&filter=" + urllib.parse.quote("Est. price per sqm", safe="") + ":0:4000",
+            "methodology": "100000 ÷ median estimated £/m², per England postcode sector with sufficient sales.",
+            "needs_name_check": not (label(best["sector"])["named"] and label(dear["sector"])["named"]),
+            "attribution": ATTRIB,
+            "sources": SOURCES,
+        }
+    ]
+
+
+def main():
+    FIND.mkdir(parents=True, exist_ok=True)
+    tw = pl.read_parquet(OUT / "cheaper_twins.parquet")
+    idx = pl.read_parquet(OUT / "sector_index.parquet")
+    facts = json.loads((OUT / "national_facts.json").read_text())
+
+    findings = [twin_finding(r) for r in curate(tw)] + national_findings(idx, facts)
+
+    for f in findings:
+        (FIND / (f["slug"].replace("/", "__") + ".json")).write_text(json.dumps(f, indent=2, default=str))
+
+    # Human review sheet
+    lines = [
+        "# Findings: review before publishing",
+        "",
+        f"{len(findings)} findings generated from analysis/out/cheaper_twins.parquet.",
+        "**Check the place names** (⚠ = unnamed sector, needs a label in analysis/place_names.json) "
+        "and spot-check a couple of numbers. Then these feed the page batch + video factory.",
+        "",
+        "| ⚠ | Title | Hook number | Page path | Deep link |",
+        "|---|-------|-------------|-----------|-----------|",
+    ]
+    for f in findings:
+        warn = "⚠" if f.get("needs_name_check") else ""
+        lines.append(
+            f"| {warn} | {f['title']} | {f.get('shocking_number','')} | `{f['page_path']}` | "
+            f"[map]({f.get('map_url', SITE + '/?' + f['map_query'])}) |"
+        )
+    lines += ["", "## Per-finding detail", ""]
+    for f in findings:
+        lines.append(f"### {f['title']}")
+        lines.append(f"- **Type:** {f['type']}  ·  **Page:** `{f['page_path']}`")
+        lines.append(f"- **Hook:** {f['hook']}")
+        if f["type"] == "cheaper_twin":
+            lines.append(
+                f"- **{f['pricey']['label']}** £{f['pricey']['est_psqm']:,}/m² (n={f['pricey']['n']:,})  →  "
+                f"**{f['twin']['label']}** £{f['twin']['est_psqm']:,}/m² (n={f['twin']['n']:,})  ·  "
+                f"gap {f['stats']['gap_pct']}%  ·  {f['stats']['dominant_type']}, ~{f['stats']['build_year']}"
+            )
+        lines.append(f"- **OG card / deep link:** `{f['map_query']}`")
+        lines.append("")
+    (OUT / "findings_review.md").write_text("\n".join(lines))
+
+    n_named = sum(1 for f in findings if not f.get("needs_name_check"))
+    print(f"Wrote {len(findings)} findings to {FIND}/ ({n_named} fully named, {len(findings)-n_named} need a name check)")
+    print(f"Review sheet: {OUT / 'findings_review.md'}")
+    for f in findings[:14]:
+        flag = "  ⚠needs-name" if f.get("needs_name_check") else ""
+        print(f"  [{f['shocking_number']:>14}] {f['title']}{flag}")
+
+
+if __name__ == "__main__":
+    main()