220 lines
9.5 KiB
Python
220 lines
9.5 KiB
Python
#!/usr/bin/env python3
|
|
"""Turn the cheaper-twin index into publishable FINDINGS.
|
|
|
|
Each finding is the single artifact the whole engine renders four ways (growth/README.md):
|
|
an SEO page, an OG/unfurl card, a video storyboard, and a ≤3-filter deep-link CTA into the live map.
|
|
This script curates the raw 415 twin pairs into a reviewed set of findings and emits:
|
|
|
|
analysis/out/findings/<slug>.json: one machine-readable finding per page/video
|
|
analysis/out/findings_review.md: human-readable sheet for the founder to eyeball + fix names
|
|
|
|
Place labels come from analysis/place_names.json (APPROXIMATE, verify before publishing).
|
|
|
|
Run: source .venv/bin/activate && python analysis/generate_findings.py
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.parse
|
|
from pathlib import Path
|
|
|
|
import polars as pl
|
|
|
|
OUT = Path("analysis/out")
|
|
FIND = OUT / "findings"
|
|
SITE = "https://perfect-postcode.co.uk"
|
|
ATTRIB = "Contains HM Land Registry data © Crown copyright and database right. Licensed under the Open Government Licence v3.0."
|
|
SOURCES = "HM Land Registry · EPC (DLUHC) · Ofsted · DfT · ONS · Police.uk"
|
|
TYPE_SINGULAR = {
|
|
"Flats/Maisonettes": "flat",
|
|
"Terraced": "terraced house",
|
|
"Semi-Detached": "semi-detached house",
|
|
"Detached": "detached house",
|
|
}
|
|
|
|
NAMES = json.loads(Path("analysis/place_names.json").read_text())
|
|
|
|
|
|
def label(sector: str) -> dict:
|
|
outward = sector.split(" ")[0]
|
|
name = NAMES.get(outward)
|
|
return {"sector": sector, "name": name, "label": f"{name} ({sector})" if name else sector, "named": bool(name)}
|
|
|
|
|
|
def sector_slug(sector: str) -> str:
|
|
return sector.lower().replace(" ", "-")
|
|
|
|
|
|
def map_query(t: dict) -> str:
|
|
"""A ≤3-filter deep link that frames the value: centre between the pair, cap £/sqm near the
|
|
cheaper twin, require a good secondary catchment. Reproducible by non-payers (DEMO_MAX_FILTERS=3)."""
|
|
mid_lat = round((t["pricey_lat"] + t["twin_lat"]) / 2, 5)
|
|
mid_lon = round((t["pricey_lon"] + t["twin_lon"]) / 2, 5)
|
|
psqm_cap = int(round(t["twin_psqm"] * 1.05, -2))
|
|
enc_psqm = urllib.parse.quote("Est. price per sqm", safe="")
|
|
enc_school = urllib.parse.quote("Good+ secondary school catchments", safe="")
|
|
# Two plain numeric filters (within the 3-filter demo cap): frames value + good schools and
|
|
# uses the well-understood filter=NAME:MIN:MAX form so the OG-card filter-name guard passes.
|
|
parts = [
|
|
f"lat={mid_lat}",
|
|
f"lon={mid_lon}",
|
|
"zoom=12.5",
|
|
f"filter={enc_psqm}:0:{psqm_cap}",
|
|
f"filter={enc_school}:1:11",
|
|
]
|
|
return "&".join(parts)
|
|
|
|
|
|
def twin_finding(t: dict) -> dict:
|
|
p, w = label(t["pricey_sector"]), label(t["twin_sector"])
|
|
typ = TYPE_SINGULAR.get(t["dominant_type"], "home")
|
|
slug = f"cheaper-twin/{sector_slug(t['pricey_sector'])}-vs-{sector_slug(t['twin_sector'])}"
|
|
q = map_query(t)
|
|
headline_name = f"{p['name']} vs {w['name']}" if p["named"] and w["named"] else f"{t['pricey_sector']} vs {t['twin_sector']}"
|
|
return {
|
|
"slug": slug,
|
|
"type": "cheaper_twin",
|
|
"page_path": f"/{slug}",
|
|
"title": f"{headline_name}: the same {typ}, about {t['gap_pct']:.0f}% cheaper per m²",
|
|
"hook": f"£{t['gap_on_90sqm']:,} less for an equivalent {typ}: same station, similar schools, ~{t['dist_km']}km apart",
|
|
"shocking_number": f"{t['gap_pct']:.0f}%",
|
|
"pricey": {**p, "est_psqm": t["pricey_psqm"], "n": t["pricey_n"]},
|
|
"twin": {**w, "est_psqm": t["twin_psqm"], "n": t["twin_n"]},
|
|
"stats": {
|
|
"gap_pct": t["gap_pct"],
|
|
"gap_per_sqm": t["gap_per_sqm"],
|
|
"gap_on_90sqm": t["gap_on_90sqm"],
|
|
"gap_on_avg_home": t["gap_on_avg_home"],
|
|
"dominant_type": t["dominant_type"],
|
|
"build_year": t["build_year"],
|
|
"good_secondary_catchments": t["good_secondary"],
|
|
"station_km": t["station_km"],
|
|
"distance_km": t["dist_km"],
|
|
},
|
|
"map_query": q,
|
|
"map_url": f"{SITE}/?{q}",
|
|
"og_image": f"{SITE}/api/screenshot?og=1&{q}",
|
|
"methodology": (
|
|
"Postcode sectors (e.g. N10 3) compared on estimated £/m² of floor space. A pair is only "
|
|
"called a 'twin' when the two sectors share the dominant property type, build era (±30y), "
|
|
"good-school catchment provision, station access, deprivation/tenure, education, age and "
|
|
"home size, so the price gap reflects a name premium, not a different kind of area. "
|
|
"Estimates, not valuations; aggregated to sector, never address-level."
|
|
),
|
|
"needs_name_check": not (p["named"] and w["named"]),
|
|
"attribution": ATTRIB,
|
|
"sources": SOURCES,
|
|
}
|
|
|
|
|
|
def curate(tw: pl.DataFrame) -> list[dict]:
|
|
rows = tw.to_dicts()
|
|
london = ("E", "EC", "WC", "W", "SW", "SE", "N", "NW")
|
|
|
|
def area(s):
|
|
import re
|
|
|
|
return re.match(r"^[A-Z]+", s).group(0)
|
|
|
|
family = [r for r in rows if r["dominant_type"] in ("Terraced", "Semi-Detached", "Detached")]
|
|
prime = sorted(rows, key=lambda r: r["gap_on_90sqm"], reverse=True)
|
|
regional_family = sorted(
|
|
[r for r in family if area(r["pricey_sector"]) not in london],
|
|
key=lambda r: r["gap_on_90sqm"],
|
|
reverse=True,
|
|
)
|
|
london_family = sorted(
|
|
[r for r in family if area(r["pricey_sector"]) in london],
|
|
key=lambda r: r["gap_on_90sqm"],
|
|
reverse=True,
|
|
)
|
|
# A spread: biggest national name premiums (PR) + relatable family-home twins (buyer video).
|
|
picks, seen = [], set()
|
|
for r in prime[:6] + regional_family[:8] + london_family[:6]:
|
|
key = r["pricey_sector"] + "|" + r["twin_sector"]
|
|
if key not in seen:
|
|
seen.add(key)
|
|
picks.append(r)
|
|
return picks
|
|
|
|
|
|
def national_findings(idx: pl.DataFrame, facts: dict) -> list[dict]:
|
|
named = idx.with_columns(
|
|
pl.col("Sector").str.extract(r"^([A-Z]+[0-9][A-Z0-9]?)", 1).alias("outward")
|
|
)
|
|
best = facts["best_value_sector"]
|
|
dear = facts["dearest_sector"]
|
|
return [
|
|
{
|
|
"slug": "square-metres-per-100k",
|
|
"type": "national_table",
|
|
"page_path": "/square-metres-per-100k",
|
|
"title": "How many square metres £100,000 buys across England",
|
|
"shocking_number": f"{best['sqm_per_100k']:.0f} m² vs {dear['sqm_per_100k']:.0f} m²",
|
|
"hook": (
|
|
f"£100k buys ~{best['sqm_per_100k']:.0f} m² of floor space in {label(best['sector'])['label']} "
|
|
f"but only ~{dear['sqm_per_100k']:.0f} m² in {label(dear['sector'])['label']}"
|
|
),
|
|
"stats": {"best": best, "dearest": dear, "n_sectors": facts["n_sectors"]},
|
|
"map_query": "zoom=6&filter=" + urllib.parse.quote("Est. price per sqm", safe="") + ":0:4000",
|
|
"methodology": "100000 ÷ median estimated £/m², per England postcode sector with sufficient sales.",
|
|
"needs_name_check": not (label(best["sector"])["named"] and label(dear["sector"])["named"]),
|
|
"attribution": ATTRIB,
|
|
"sources": SOURCES,
|
|
}
|
|
]
|
|
|
|
|
|
def main():
|
|
FIND.mkdir(parents=True, exist_ok=True)
|
|
tw = pl.read_parquet(OUT / "cheaper_twins.parquet")
|
|
idx = pl.read_parquet(OUT / "sector_index.parquet")
|
|
facts = json.loads((OUT / "national_facts.json").read_text())
|
|
|
|
findings = [twin_finding(r) for r in curate(tw)] + national_findings(idx, facts)
|
|
|
|
for f in findings:
|
|
(FIND / (f["slug"].replace("/", "__") + ".json")).write_text(json.dumps(f, indent=2, default=str))
|
|
|
|
# Human review sheet
|
|
lines = [
|
|
"# Findings: review before publishing",
|
|
"",
|
|
f"{len(findings)} findings generated from analysis/out/cheaper_twins.parquet.",
|
|
"**Check the place names** (⚠ = unnamed sector, needs a label in analysis/place_names.json) "
|
|
"and spot-check a couple of numbers. Then these feed the page batch + video factory.",
|
|
"",
|
|
"| ⚠ | Title | Hook number | Page path | Deep link |",
|
|
"|---|-------|-------------|-----------|-----------|",
|
|
]
|
|
for f in findings:
|
|
warn = "⚠" if f.get("needs_name_check") else ""
|
|
lines.append(
|
|
f"| {warn} | {f['title']} | {f.get('shocking_number','')} | `{f['page_path']}` | "
|
|
f"[map]({f.get('map_url', SITE + '/?' + f['map_query'])}) |"
|
|
)
|
|
lines += ["", "## Per-finding detail", ""]
|
|
for f in findings:
|
|
lines.append(f"### {f['title']}")
|
|
lines.append(f"- **Type:** {f['type']} · **Page:** `{f['page_path']}`")
|
|
lines.append(f"- **Hook:** {f['hook']}")
|
|
if f["type"] == "cheaper_twin":
|
|
lines.append(
|
|
f"- **{f['pricey']['label']}** £{f['pricey']['est_psqm']:,}/m² (n={f['pricey']['n']:,}) → "
|
|
f"**{f['twin']['label']}** £{f['twin']['est_psqm']:,}/m² (n={f['twin']['n']:,}) · "
|
|
f"gap {f['stats']['gap_pct']}% · {f['stats']['dominant_type']}, ~{f['stats']['build_year']}"
|
|
)
|
|
lines.append(f"- **OG card / deep link:** `{f['map_query']}`")
|
|
lines.append("")
|
|
(OUT / "findings_review.md").write_text("\n".join(lines))
|
|
|
|
n_named = sum(1 for f in findings if not f.get("needs_name_check"))
|
|
print(f"Wrote {len(findings)} findings to {FIND}/ ({n_named} fully named, {len(findings)-n_named} need a name check)")
|
|
print(f"Review sheet: {OUT / 'findings_review.md'}")
|
|
for f in findings[:14]:
|
|
flag = " ⚠needs-name" if f.get("needs_name_check") else ""
|
|
print(f" [{f['shocking_number']:>14}] {f['title']}{flag}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|