andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Fmt

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-15 21:22:28 +00:00
parent 479ef92236
commit c38d654ac7
44 changed files with 2526 additions and 701 deletions
Download patch file Download diff file Expand all files Collapse all files

22
analyses/bank_postcode_boundaries.ipynb
View file

@ -813,8 +813,14 @@
],
"source": [
"# Build area lookup from both sets\n",
"areas_before = {f[\"properties\"][\"postcode\"]: f[\"properties\"][\"area_sqm\"] for f in no_green[\"features\"]}\n",
"areas_after = {f[\"properties\"][\"postcode\"]: f[\"properties\"][\"area_sqm\"] for f in with_green[\"features\"]}\n",
"areas_before = {\n",
" f[\"properties\"][\"postcode\"]: f[\"properties\"][\"area_sqm\"]\n",
" for f in no_green[\"features\"]\n",
"}\n",
"areas_after = {\n",
" f[\"properties\"][\"postcode\"]: f[\"properties\"][\"area_sqm\"]\n",
" for f in with_green[\"features\"]\n",
"}\n",
"\n",
"# Compute percentage removed\n",
"diffs = []\n",
@ -1161,16 +1167,23 @@
"\n",
"colormap = cm.LinearColormap(\n",
" colors=[\"#ffffcc\", \"#fd8d3c\", \"#e31a1c\", \"#800026\"],\n",
" vmin=0, vmax=min(max_pct, 90),\n",
" vmin=0,\n",
" vmax=min(max_pct, 90),\n",
" caption=\"% area removed by greenspace\",\n",
")\n",
"\n",
"\n",
"# Show original boundaries, colored by how much was removed\n",
"def style_by_removal(feature):\n",
" pc = feature[\"properties\"][\"postcode\"]\n",
" pct = diff_lookup.get(pc, 0)\n",
" if pct <= 1:\n",
" return {\"fillColor\": \"#cccccc\", \"color\": \"#999\", \"weight\": 0.5, \"fillOpacity\": 0.15}\n",
" return {\n",
" \"fillColor\": \"#cccccc\",\n",
" \"color\": \"#999\",\n",
" \"weight\": 0.5,\n",
" \"fillOpacity\": 0.15,\n",
" }\n",
" return {\n",
" \"fillColor\": colormap(min(pct, 90)),\n",
" \"color\": \"white\",\n",
@ -1178,6 +1191,7 @@
" \"fillOpacity\": 0.6,\n",
" }\n",
"\n",
"\n",
"folium.GeoJson(\n",
" no_green,\n",
" name=\"Greenspace removal %\",\n",

125
analyses/price_model_evaluation.ipynb
View file

@ -54,25 +54,32 @@
" ape = np.abs(p - a) / a\n",
" err = p - a\n",
" return {\n",
" \"MdAPE (%)\": f\"{np.median(ape)*100:.1f}\",\n",
" \"% within 10%\": f\"{np.mean(ape <= 0.10)*100:.1f}\",\n",
" \"% within 20%\": f\"{np.mean(ape <= 0.20)*100:.1f}\",\n",
" \"% within 30%\": f\"{np.mean(ape <= 0.30)*100:.1f}\",\n",
" \"MdAPE (%)\": f\"{np.median(ape) * 100:.1f}\",\n",
" \"% within 10%\": f\"{np.mean(ape <= 0.10) * 100:.1f}\",\n",
" \"% within 20%\": f\"{np.mean(ape <= 0.20) * 100:.1f}\",\n",
" \"% within 30%\": f\"{np.mean(ape <= 0.30) * 100:.1f}\",\n",
" \"MAE (\\u00a3)\": f\"{np.mean(np.abs(err)):,.0f}\",\n",
" \"Mean signed error (\\u00a3)\": f\"{np.mean(err):+,.0f}\",\n",
" \"n\": f\"{len(a):,}\",\n",
" }\n",
"\n",
"\n",
"actual = backtest_df[\"actual_price\"].to_numpy().astype(np.float64)\n",
"metrics = {\n",
" \"Naive\": compute_metrics(actual, backtest_df[\"input_price\"].to_numpy().astype(np.float64)),\n",
" \"Index\": compute_metrics(actual, backtest_df[\"predicted\"].to_numpy().astype(np.float64)),\n",
" \"Naive\": compute_metrics(\n",
" actual, backtest_df[\"input_price\"].to_numpy().astype(np.float64)\n",
" ),\n",
" \"Index\": compute_metrics(\n",
" actual, backtest_df[\"predicted\"].to_numpy().astype(np.float64)\n",
" ),\n",
"}\n",
"\n",
"metrics_table = pl.DataFrame([\n",
" {\"Metric\": k, **{stage: v[k] for stage, v in metrics.items()}}\n",
" for k in list(metrics[\"Naive\"].keys())\n",
"])\n",
"metrics_table = pl.DataFrame(\n",
" [\n",
" {\"Metric\": k, **{stage: v[k] for stage, v in metrics.items()}}\n",
" for k in list(metrics[\"Naive\"].keys())\n",
" ]\n",
")\n",
"metrics_table"
]
},
@ -91,8 +98,7 @@
"source": [
"# National index (average across all sectors weighted by n_pairs)\n",
"national = (\n",
" index_df\n",
" .group_by(\"year\")\n",
" index_df.group_by(\"year\")\n",
" .agg(\n",
" (pl.col(\"log_index\") * pl.col(\"n_pairs\")).sum() / pl.col(\"n_pairs\").sum(),\n",
" )\n",
@ -107,14 +113,23 @@
"\n",
"# If not enough, pick some with high/low n_pairs\n",
"if len(sample_sectors) < 3:\n",
" sector_counts = index_df.group_by(\"sector\").agg(pl.col(\"n_pairs\").first()).sort(\"n_pairs\", descending=True)\n",
" sector_counts = (\n",
" index_df.group_by(\"sector\")\n",
" .agg(pl.col(\"n_pairs\").first())\n",
" .sort(\"n_pairs\", descending=True)\n",
" )\n",
" top = sector_counts.head(2)[\"sector\"].to_list()\n",
" bottom = sector_counts.filter(pl.col(\"n_pairs\") > 0).tail(2)[\"sector\"].to_list()\n",
" sample_sectors = list(set(sample_sectors + top + bottom))[:5]\n",
"\n",
"samples = index_df.filter(pl.col(\"sector\").is_in(sample_sectors))\n",
"\n",
"combined = pl.concat([national.select(\"sector\", \"year\", \"log_index\"), samples.select(\"sector\", \"year\", \"log_index\")])\n",
"combined = pl.concat(\n",
" [\n",
" national.select(\"sector\", \"year\", \"log_index\"),\n",
" samples.select(\"sector\", \"year\", \"log_index\"),\n",
" ]\n",
")\n",
"\n",
"# Normalize: index = 100 at base year (earliest available)\n",
"combined = combined.with_columns(\n",
@ -122,7 +137,10 @@
")\n",
"\n",
"fig = px.line(\n",
" combined.to_pandas(), x=\"year\", y=\"index_100\", color=\"sector\",\n",
" combined.to_pandas(),\n",
" x=\"year\",\n",
" y=\"index_100\",\n",
" color=\"sector\",\n",
" title=\"Repeat-Sales Price Index (base year = 100)\",\n",
" labels={\"index_100\": \"Index (base=100)\", \"year\": \"Year\"},\n",
")\n",
@ -155,8 +173,10 @@
"\n",
"fig.update_layout(\n",
" title=\"Absolute Percentage Error Distribution\",\n",
" xaxis_title=\"APE (%)\", yaxis_title=\"Count\",\n",
" barmode=\"overlay\", height=500,\n",
" xaxis_title=\"APE (%)\",\n",
" yaxis_title=\"Count\",\n",
" barmode=\"overlay\",\n",
" height=500,\n",
")\n",
"fig.show()"
]
@ -183,17 +203,27 @@
"pred = sample[\"predicted\"].to_numpy().astype(np.float64)\n",
"\n",
"fig = go.Figure()\n",
"fig.add_trace(go.Scattergl(\n",
" x=actual_sample, y=pred, mode=\"markers\",\n",
" marker=dict(size=2, opacity=0.3), name=\"Index\",\n",
"))\n",
"fig.add_trace(\n",
" go.Scattergl(\n",
" x=actual_sample,\n",
" y=pred,\n",
" mode=\"markers\",\n",
" marker=dict(size=2, opacity=0.3),\n",
" name=\"Index\",\n",
" )\n",
")\n",
"# 45-degree reference line\n",
"min_val = max(10_000, min(actual_sample.min(), np.nanmin(pred)))\n",
"max_val = min(5_000_000, max(actual_sample.max(), np.nanmax(pred)))\n",
"fig.add_trace(go.Scatter(\n",
" x=[min_val, max_val], y=[min_val, max_val],\n",
" mode=\"lines\", line=dict(color=\"red\", dash=\"dash\"), showlegend=False,\n",
"))\n",
"fig.add_trace(\n",
" go.Scatter(\n",
" x=[min_val, max_val],\n",
" y=[min_val, max_val],\n",
" mode=\"lines\",\n",
" line=dict(color=\"red\", dash=\"dash\"),\n",
" showlegend=False,\n",
" )\n",
")\n",
"fig.update_xaxes(type=\"log\", title_text=\"Actual (\\u00a3)\")\n",
"fig.update_yaxes(type=\"log\", title_text=\"Predicted (\\u00a3)\")\n",
"fig.update_layout(title=\"Predicted vs Actual Price (log scale, 10K sample)\", height=500)\n",
@ -234,12 +264,22 @@
" for name, arr in [(\"Naive\", naive), (\"Index\", pred)]:\n",
" ape = np.abs(arr[mask] - actual[mask]) / actual[mask]\n",
" valid = np.isfinite(ape)\n",
" rows.append({\"Price Band\": label, \"Method\": name, \"MdAPE (%)\": float(np.median(ape[valid]) * 100)})\n",
" rows.append(\n",
" {\n",
" \"Price Band\": label,\n",
" \"Method\": name,\n",
" \"MdAPE (%)\": float(np.median(ape[valid]) * 100),\n",
" }\n",
" )\n",
"\n",
"band_df = pl.DataFrame(rows)\n",
"fig = px.bar(\n",
" band_df.to_pandas(), x=\"Price Band\", y=\"MdAPE (%)\", color=\"Method\",\n",
" barmode=\"group\", title=\"MdAPE by Price Band\",\n",
" band_df.to_pandas(),\n",
" x=\"Price Band\",\n",
" y=\"MdAPE (%)\",\n",
" color=\"Method\",\n",
" barmode=\"group\",\n",
" title=\"MdAPE by Price Band\",\n",
" category_orders={\"Price Band\": [b[2] for b in bands]},\n",
")\n",
"fig.update_layout(height=450)\n",
@ -264,7 +304,9 @@
")\n",
"\n",
"# Top 20 areas by volume\n",
"top_areas = bt.group_by(\"area\").len().sort(\"len\", descending=True).head(20)[\"area\"].to_list()\n",
"top_areas = (\n",
" bt.group_by(\"area\").len().sort(\"len\", descending=True).head(20)[\"area\"].to_list()\n",
")\n",
"\n",
"actual_np = bt[\"actual_price\"].to_numpy().astype(np.float64)\n",
"pred_np = bt[\"predicted\"].to_numpy().astype(np.float64)\n",
@ -279,12 +321,18 @@
" p = arr[mask]\n",
" valid = np.isfinite(p) & (a > 0)\n",
" ape = np.abs(p[valid] - a[valid]) / a[valid]\n",
" rows.append({\"Area\": area, \"Method\": name, \"MdAPE (%)\": float(np.median(ape) * 100)})\n",
" rows.append(\n",
" {\"Area\": area, \"Method\": name, \"MdAPE (%)\": float(np.median(ape) * 100)}\n",
" )\n",
"\n",
"area_df = pl.DataFrame(rows)\n",
"fig = px.bar(\n",
" area_df.to_pandas(), x=\"Area\", y=\"MdAPE (%)\", color=\"Method\",\n",
" barmode=\"group\", title=\"MdAPE by Postcode Area (Top 20 by Volume)\",\n",
" area_df.to_pandas(),\n",
" x=\"Area\",\n",
" y=\"MdAPE (%)\",\n",
" color=\"Method\",\n",
" barmode=\"group\",\n",
" title=\"MdAPE by Postcode Area (Top 20 by Volume)\",\n",
" category_orders={\"Area\": top_areas},\n",
")\n",
"fig.update_layout(height=500)\n",
@ -324,11 +372,20 @@
" p = arr[mask]\n",
" valid = np.isfinite(p) & (a > 0)\n",
" ape = np.abs(p[valid] - a[valid]) / a[valid]\n",
" rows.append({\"Gap (years)\": gap, \"Method\": name, \"MdAPE (%)\": float(np.median(ape) * 100)})\n",
" rows.append(\n",
" {\n",
" \"Gap (years)\": gap,\n",
" \"Method\": name,\n",
" \"MdAPE (%)\": float(np.median(ape) * 100),\n",
" }\n",
" )\n",
"\n",
"gap_df = pl.DataFrame(rows)\n",
"fig = px.line(\n",
" gap_df.to_pandas(), x=\"Gap (years)\", y=\"MdAPE (%)\", color=\"Method\",\n",
" gap_df.to_pandas(),\n",
" x=\"Gap (years)\",\n",
" y=\"MdAPE (%)\",\n",
" color=\"Method\",\n",
" title=\"MdAPE by Holding Period (years between input and actual sale)\",\n",
" markers=True,\n",
")\n",

390
analyses/rightmove_buy.ipynb
View file

@ -52,7 +52,9 @@
"pl.Config.set_tbl_rows(20)\n",
"pl.Config.set_fmt_str_lengths(80)\n",
"\n",
"df = pl.read_parquet(\"/volumes/syncthing/Projects/property-map/property-data/rightmove_buy.parquet\")\n",
"df = pl.read_parquet(\n",
" \"/volumes/syncthing/Projects/property-map/property-data/rightmove_buy.parquet\"\n",
")\n",
"schema = df.schema\n",
"print(f\"Total rows: {len(df):,}\")\n",
"print(f\"Columns ({len(schema)}):\")\n",
@ -150,11 +152,13 @@
],
"source": [
"# Null counts\n",
"null_df = pl.DataFrame({\n",
" \"column\": df.columns,\n",
" \"nulls\": [df[c].null_count() for c in df.columns],\n",
" \"pct\": [f\"{df[c].null_count()/len(df)*100:.1f}%\" for c in df.columns],\n",
"})\n",
"null_df = pl.DataFrame(\n",
" {\n",
" \"column\": df.columns,\n",
" \"nulls\": [df[c].null_count() for c in df.columns],\n",
" \"pct\": [f\"{df[c].null_count() / len(df) * 100:.1f}%\" for c in df.columns],\n",
" }\n",
")\n",
"null_df.filter(pl.col(\"nulls\") > 0)"
]
},
@ -197,13 +201,17 @@
" \"price = 0\": len(df.filter(pl.col(\"price\") == 0)),\n",
" \"price > 50M\": len(df.filter(pl.col(\"price\") > 50_000_000)),\n",
" \"floorspace > 10,000 sqm\": len(df.filter(pl.col(\"floorspace_sqm\") > 10_000)),\n",
" \"latitude outside UK (< 49 or > 61)\": len(df.filter((pl.col(\"latitude\") < 49) | (pl.col(\"latitude\") > 61))),\n",
" \"longitude outside UK (< -8 or > 2)\": len(df.filter((pl.col(\"longitude\") < -8) | (pl.col(\"longitude\") > 2))),\n",
" \"latitude outside UK (< 49 or > 61)\": len(\n",
" df.filter((pl.col(\"latitude\") < 49) | (pl.col(\"latitude\") > 61))\n",
" ),\n",
" \"longitude outside UK (< -8 or > 2)\": len(\n",
" df.filter((pl.col(\"longitude\") < -8) | (pl.col(\"longitude\") > 2))\n",
" ),\n",
" \"house_share = true\": len(df.filter(pl.col(\"house_share\"))),\n",
"}\n",
"print(\"Data quality issues:\")\n",
"for desc, count in issues.items():\n",
" print(f\" {desc}: {count:,} ({count/len(df)*100:.2f}%)\")"
" print(f\" {desc}: {count:,} ({count / len(df) * 100:.2f}%)\")"
]
},
{
@ -230,7 +238,7 @@
" & (pl.col(\"longitude\") >= -8)\n",
" & (pl.col(\"longitude\") <= 2)\n",
")\n",
"print(f\"Clean rows: {len(clean):,} ({len(clean)/len(df)*100:.1f}% of original)\")"
"print(f\"Clean rows: {len(clean):,} ({len(clean) / len(df) * 100:.1f}% of original)\")"
]
},
{
@ -1126,8 +1134,12 @@
"# Price histogram (clipped to 2nd-98th percentile)\n",
"lo, hi = price.quantile(0.02), price.quantile(0.98)\n",
"clipped = clean.filter((pl.col(\"price\") >= lo) & (pl.col(\"price\") <= hi))\n",
"fig = px.histogram(clipped.to_pandas(), x=\"price\", nbins=80,\n",
" title=f\"Asking Price Distribution (£{lo:,.0f} - £{hi:,.0f}, 2nd-98th pctl)\")\n",
"fig = px.histogram(\n",
" clipped.to_pandas(),\n",
" x=\"price\",\n",
" nbins=80,\n",
" title=f\"Asking Price Distribution (£{lo:,.0f} - £{hi:,.0f}, 2nd-98th pctl)\",\n",
")\n",
"fig.update_layout(height=400, xaxis_title=\"Asking Price (£)\", yaxis_title=\"Count\")\n",
"fig.show()"
]
@ -439978,9 +439990,13 @@
],
"source": [
"# Price by property type\n",
"fig = px.box(clean.filter(pl.col(\"price\") <= 2_000_000).to_pandas(),\n",
" x=\"property_type\", y=\"price\", color=\"property_type\",\n",
" title=\"Price by Property Type (capped at £2M for readability)\")\n",
"fig = px.box(\n",
" clean.filter(pl.col(\"price\") <= 2_000_000).to_pandas(),\n",
" x=\"property_type\",\n",
" y=\"price\",\n",
" color=\"property_type\",\n",
" title=\"Price by Property Type (capped at £2M for readability)\",\n",
")\n",
"fig.update_layout(height=500, showlegend=False, yaxis_title=\"Price (£)\")\n",
"fig.show()"
]
@ -440079,9 +440095,7 @@
"source": [
"# Price qualifier breakdown\n",
"pq = clean[\"price_qualifier\"].value_counts().sort(\"count\", descending=True)\n",
"pq = pq.with_columns(\n",
" (pl.col(\"count\") / pl.col(\"count\").sum() * 100).alias(\"pct\")\n",
")\n",
"pq = pq.with_columns((pl.col(\"count\") / pl.col(\"count\").sum() * 100).alias(\"pct\"))\n",
"pq"
]
},
@ -440928,8 +440942,12 @@
"source": [
"# Property type distribution\n",
"type_counts = clean[\"property_type\"].value_counts().sort(\"count\", descending=True)\n",
"fig = px.pie(type_counts.to_pandas(), names=\"property_type\", values=\"count\",\n",
" title=\"Property Type Distribution\")\n",
"fig = px.pie(\n",
" type_counts.to_pandas(),\n",
" names=\"property_type\",\n",
" values=\"count\",\n",
" title=\"Property Type Distribution\",\n",
")\n",
"fig.update_layout(height=400)\n",
"fig.show()"
]
@ -441805,9 +441823,16 @@
],
"source": [
"# Top 20 sub-types\n",
"sub_counts = clean[\"property_sub_type\"].value_counts().sort(\"count\", descending=True).head(20)\n",
"fig = px.bar(sub_counts.to_pandas(), x=\"count\", y=\"property_sub_type\", orientation=\"h\",\n",
" title=\"Top 20 Property Sub-types\")\n",
"sub_counts = (\n",
" clean[\"property_sub_type\"].value_counts().sort(\"count\", descending=True).head(20)\n",
")\n",
"fig = px.bar(\n",
" sub_counts.to_pandas(),\n",
" x=\"count\",\n",
" y=\"property_sub_type\",\n",
" orientation=\"h\",\n",
" title=\"Top 20 Property Sub-types\",\n",
")\n",
"fig.update_layout(height=600, yaxis={\"categoryorder\": \"total ascending\"})\n",
"fig.show()"
]
@ -442643,9 +442668,15 @@
],
"source": [
"# Tenure split\n",
"tenure_counts = clean[\"tenure\"].drop_nulls().value_counts().sort(\"count\", descending=True)\n",
"fig = px.pie(tenure_counts.to_pandas(), names=\"tenure\", values=\"count\",\n",
" title=f\"Tenure Split ({clean['tenure'].null_count():,} unknown / {clean['tenure'].null_count()/len(clean)*100:.1f}% missing)\")\n",
"tenure_counts = (\n",
" clean[\"tenure\"].drop_nulls().value_counts().sort(\"count\", descending=True)\n",
")\n",
"fig = px.pie(\n",
" tenure_counts.to_pandas(),\n",
" names=\"tenure\",\n",
" values=\"count\",\n",
" title=f\"Tenure Split ({clean['tenure'].null_count():,} unknown / {clean['tenure'].null_count() / len(clean) * 100:.1f}% missing)\",\n",
")\n",
"fig.update_layout(height=400)\n",
"fig.show()"
]
@ -443546,8 +443577,14 @@
" .agg(pl.len().alias(\"count\"))\n",
" .sort(\"property_type\")\n",
")\n",
"fig = px.bar(tenure_by_type.to_pandas(), x=\"property_type\", y=\"count\", color=\"tenure\",\n",
" barmode=\"group\", title=\"Tenure by Property Type\")\n",
"fig = px.bar(\n",
" tenure_by_type.to_pandas(),\n",
" x=\"property_type\",\n",
" y=\"count\",\n",
" color=\"tenure\",\n",
" barmode=\"group\",\n",
" title=\"Tenure by Property Type\",\n",
")\n",
"fig.update_layout(height=400)\n",
"fig.show()"
]
@ -444412,9 +444449,12 @@
],
"source": [
"# Bedroom distribution\n",
"bed_counts = clean.filter(pl.col(\"bedrooms\") <= 10)[\"bedrooms\"].value_counts().sort(\"bedrooms\")\n",
"fig = px.bar(bed_counts.to_pandas(), x=\"bedrooms\", y=\"count\",\n",
" title=\"Bedroom Count Distribution\")\n",
"bed_counts = (\n",
" clean.filter(pl.col(\"bedrooms\") <= 10)[\"bedrooms\"].value_counts().sort(\"bedrooms\")\n",
")\n",
"fig = px.bar(\n",
" bed_counts.to_pandas(), x=\"bedrooms\", y=\"count\", title=\"Bedroom Count Distribution\"\n",
")\n",
"fig.update_layout(height=400)\n",
"fig.show()"
]
@ -445279,16 +445319,25 @@
")\n",
"\n",
"fig = go.Figure()\n",
"fig.add_trace(go.Bar(\n",
" x=price_by_beds[\"bedrooms\"], y=price_by_beds[\"median_price\"],\n",
" name=\"Median\", error_y=dict(type=\"data\",\n",
" symmetric=False,\n",
" array=(price_by_beds[\"p75\"] - price_by_beds[\"median_price\"]).to_list(),\n",
" arrayminus=(price_by_beds[\"median_price\"] - price_by_beds[\"p25\"]).to_list()\n",
"fig.add_trace(\n",
" go.Bar(\n",
" x=price_by_beds[\"bedrooms\"],\n",
" y=price_by_beds[\"median_price\"],\n",
" name=\"Median\",\n",
" error_y=dict(\n",
" type=\"data\",\n",
" symmetric=False,\n",
" array=(price_by_beds[\"p75\"] - price_by_beds[\"median_price\"]).to_list(),\n",
" arrayminus=(price_by_beds[\"median_price\"] - price_by_beds[\"p25\"]).to_list(),\n",
" ),\n",
" )\n",
"))\n",
"fig.update_layout(title=\"Median Price by Bedrooms (with IQR)\", height=400,\n",
" xaxis_title=\"Bedrooms\", yaxis_title=\"Price (£)\")\n",
")\n",
"fig.update_layout(\n",
" title=\"Median Price by Bedrooms (with IQR)\",\n",
" height=400,\n",
" xaxis_title=\"Bedrooms\",\n",
" yaxis_title=\"Price (£)\",\n",
")\n",
"fig.show()"
]
},
@ -446263,8 +446312,14 @@
" .agg(pl.len().alias(\"count\"))\n",
" .sort(\"property_type\", \"bedrooms\")\n",
")\n",
"fig = px.bar(beds_by_type.to_pandas(), x=\"bedrooms\", y=\"count\", color=\"property_type\",\n",
" barmode=\"group\", title=\"Bedroom Distribution by Property Type\")\n",
"fig = px.bar(\n",
" beds_by_type.to_pandas(),\n",
" x=\"bedrooms\",\n",
" y=\"count\",\n",
" color=\"property_type\",\n",
" barmode=\"group\",\n",
" title=\"Bedroom Distribution by Property Type\",\n",
")\n",
"fig.update_layout(height=450)\n",
"fig.show()"
]
@ -446323,19 +446378,26 @@
],
"source": [
"# Floorspace availability by property type\n",
"has_floor = clean.with_columns(pl.col(\"floorspace_sqm\").is_not_null().alias(\"has_floorspace\"))\n",
"floor_by_type = (\n",
" has_floor.group_by(\"property_type\", \"has_floorspace\")\n",
" .agg(pl.len().alias(\"count\"))\n",
"has_floor = clean.with_columns(\n",
" pl.col(\"floorspace_sqm\").is_not_null().alias(\"has_floorspace\")\n",
")\n",
"floor_by_type = has_floor.group_by(\"property_type\", \"has_floorspace\").agg(\n",
" pl.len().alias(\"count\")\n",
")\n",
"totals = floor_by_type.group_by(\"property_type\").agg(\n",
" pl.col(\"count\").sum().alias(\"total\")\n",
")\n",
"totals = floor_by_type.group_by(\"property_type\").agg(pl.col(\"count\").sum().alias(\"total\"))\n",
"floor_pct = (\n",
" floor_by_type.filter(pl.col(\"has_floorspace\"))\n",
" .join(totals, on=\"property_type\")\n",
" .with_columns((pl.col(\"count\") / pl.col(\"total\") * 100).alias(\"pct_with_floorspace\"))\n",
" .with_columns(\n",
" (pl.col(\"count\") / pl.col(\"total\") * 100).alias(\"pct_with_floorspace\")\n",
" )\n",
" .sort(\"pct_with_floorspace\", descending=True)\n",
")\n",
"print(f\"Overall floorspace availability: {clean['floorspace_sqm'].drop_nulls().len():,} / {len(clean):,} ({clean['floorspace_sqm'].drop_nulls().len()/len(clean)*100:.1f}%)\")\n",
"print(\n",
" f\"Overall floorspace availability: {clean['floorspace_sqm'].drop_nulls().len():,} / {len(clean):,} ({clean['floorspace_sqm'].drop_nulls().len() / len(clean) * 100:.1f}%)\"\n",
")\n",
"floor_pct.select(\"property_type\", \"count\", \"total\", \"pct_with_floorspace\")"
]
},
@ -447298,8 +447360,13 @@
")\n",
"print(f\"Properties with reasonable floorspace (10-1000 sqm): {len(with_floor):,}\")\n",
"\n",
"fig = px.histogram(with_floor.to_pandas(), x=\"floorspace_sqm\", nbins=80, color=\"property_type\",\n",
" title=\"Floorspace Distribution by Property Type\")\n",
"fig = px.histogram(\n",
" with_floor.to_pandas(),\n",
" x=\"floorspace_sqm\",\n",
" nbins=80,\n",
" color=\"property_type\",\n",
" title=\"Floorspace Distribution by Property Type\",\n",
")\n",
"fig.update_layout(height=450, xaxis_title=\"Floorspace (sqm)\", barmode=\"overlay\")\n",
"fig.update_traces(opacity=0.6)\n",
"fig.show()"
@ -448176,8 +448243,12 @@
"print(f\" P25: £{s.quantile(0.25):,.0f}/sqm\")\n",
"print(f\" P75: £{s.quantile(0.75):,.0f}/sqm\")\n",
"\n",
"fig = px.histogram(ppsqm.to_pandas(), x=\"price_per_sqm\", nbins=80,\n",
" title=\"Price per Square Metre Distribution\")\n",
"fig = px.histogram(\n",
" ppsqm.to_pandas(),\n",
" x=\"price_per_sqm\",\n",
" nbins=80,\n",
" title=\"Price per Square Metre Distribution\",\n",
")\n",
"fig.update_layout(height=400, xaxis_title=\"Price per sqm (£)\")\n",
"fig.show()"
]
@ -584906,8 +584977,13 @@
}
],
"source": [
"fig = px.box(ppsqm.to_pandas(), x=\"property_type\", y=\"price_per_sqm\", color=\"property_type\",\n",
" title=\"Price per sqm by Property Type\")\n",
"fig = px.box(\n",
" ppsqm.to_pandas(),\n",
" x=\"property_type\",\n",
" y=\"price_per_sqm\",\n",
" color=\"property_type\",\n",
" title=\"Price per sqm by Property Type\",\n",
")\n",
"fig.update_layout(height=450, showlegend=False, yaxis_title=\"£ per sqm\")\n",
"fig.show()"
]
@ -585865,9 +585941,15 @@
")\n",
"\n",
"top30 = outcode_stats.head(30)\n",
"fig = px.bar(top30.to_pandas(), x=\"count\", y=\"outcode\", orientation=\"h\",\n",
" color=\"median_price\", color_continuous_scale=\"Viridis\",\n",
" title=\"Top 30 Outcodes by Listing Volume\")\n",
"fig = px.bar(\n",
" top30.to_pandas(),\n",
" x=\"count\",\n",
" y=\"outcode\",\n",
" orientation=\"h\",\n",
" color=\"median_price\",\n",
" color_continuous_scale=\"Viridis\",\n",
" title=\"Top 30 Outcodes by Listing Volume\",\n",
")\n",
"fig.update_layout(height=700, yaxis={\"categoryorder\": \"total ascending\"})\n",
"fig.show()"
]
@ -587400,11 +587482,25 @@
],
"source": [
"# Most expensive outcodes (min 50 listings)\n",
"expensive = outcode_stats.filter(pl.col(\"count\") >= 50).sort(\"median_price\", descending=True).head(30)\n",
"fig = px.bar(expensive.to_pandas(), x=\"median_price\", y=\"outcode\", orientation=\"h\",\n",
" color=\"count\", color_continuous_scale=\"Blues\",\n",
" title=\"Top 30 Most Expensive Outcodes (min 50 listings, by median price)\")\n",
"fig.update_layout(height=700, yaxis={\"categoryorder\": \"total ascending\"}, xaxis_title=\"Median Price (£)\")\n",
"expensive = (\n",
" outcode_stats.filter(pl.col(\"count\") >= 50)\n",
" .sort(\"median_price\", descending=True)\n",
" .head(30)\n",
")\n",
"fig = px.bar(\n",
" expensive.to_pandas(),\n",
" x=\"median_price\",\n",
" y=\"outcode\",\n",
" orientation=\"h\",\n",
" color=\"count\",\n",
" color_continuous_scale=\"Blues\",\n",
" title=\"Top 30 Most Expensive Outcodes (min 50 listings, by median price)\",\n",
")\n",
"fig.update_layout(\n",
" height=700,\n",
" yaxis={\"categoryorder\": \"total ascending\"},\n",
" xaxis_title=\"Median Price (£)\",\n",
")\n",
"fig.show()"
]
},
@ -588914,10 +589010,20 @@
"source": [
"# Cheapest outcodes (min 50 listings)\n",
"cheapest = outcode_stats.filter(pl.col(\"count\") >= 50).sort(\"median_price\").head(30)\n",
"fig = px.bar(cheapest.to_pandas(), x=\"median_price\", y=\"outcode\", orientation=\"h\",\n",
" color=\"count\", color_continuous_scale=\"Blues\",\n",
" title=\"Top 30 Cheapest Outcodes (min 50 listings, by median price)\")\n",
"fig.update_layout(height=700, yaxis={\"categoryorder\": \"total descending\"}, xaxis_title=\"Median Price (£)\")\n",
"fig = px.bar(\n",
" cheapest.to_pandas(),\n",
" x=\"median_price\",\n",
" y=\"outcode\",\n",
" orientation=\"h\",\n",
" color=\"count\",\n",
" color_continuous_scale=\"Blues\",\n",
" title=\"Top 30 Cheapest Outcodes (min 50 listings, by median price)\",\n",
")\n",
"fig.update_layout(\n",
" height=700,\n",
" yaxis={\"categoryorder\": \"total descending\"},\n",
" xaxis_title=\"Median Price (£)\",\n",
")\n",
"fig.show()"
]
},
@ -589828,14 +589934,19 @@
"source": [
"# Geographic scatter of listings (sample for performance)\n",
"sample = clean.sample(n=min(20_000, len(clean)), seed=42)\n",
"fig = px.scatter_map(sample.to_pandas(),\n",
" lat=\"latitude\", lon=\"longitude\",\n",
" color=\"price\", size_max=4,\n",
" color_continuous_scale=\"Viridis\",\n",
" range_color=[100_000, 1_500_000],\n",
" zoom=5, center={\"lat\": 52.5, \"lon\": -1.5},\n",
" title=\"Listing Locations (20k sample, colored by price)\",\n",
" opacity=0.4)\n",
"fig = px.scatter_map(\n",
" sample.to_pandas(),\n",
" lat=\"latitude\",\n",
" lon=\"longitude\",\n",
" color=\"price\",\n",
" size_max=4,\n",
" color_continuous_scale=\"Viridis\",\n",
" range_color=[100_000, 1_500_000],\n",
" zoom=5,\n",
" center={\"lat\": 52.5, \"lon\": -1.5},\n",
" title=\"Listing Locations (20k sample, colored by price)\",\n",
" opacity=0.4,\n",
")\n",
"fig.update_layout(height=700)\n",
"fig.show()"
]
@ -589864,7 +589975,9 @@
"source": [
"# Parse dates and look at listing age\n",
"with_dates = clean.with_columns(\n",
" pl.col(\"first_visible_date\").str.to_datetime(\"%Y-%m-%dT%H:%M:%SZ\").alias(\"listed_at\"),\n",
" pl.col(\"first_visible_date\")\n",
" .str.to_datetime(\"%Y-%m-%dT%H:%M:%SZ\")\n",
" .alias(\"listed_at\"),\n",
")\n",
"\n",
"print(f\"Date range: {with_dates['listed_at'].min()} to {with_dates['listed_at'].max()}\")"
@ -590856,8 +590969,9 @@
" .sort(\"month\")\n",
")\n",
"\n",
"fig = px.bar(monthly.to_pandas(), x=\"month\", y=\"count\",\n",
" title=\"Listings by Month Listed\")\n",
"fig = px.bar(\n",
" monthly.to_pandas(), x=\"month\", y=\"count\", title=\"Listings by Month Listed\"\n",
")\n",
"fig.update_layout(height=400, xaxis_title=\"Month\", yaxis_title=\"Listings\")\n",
"fig.show()"
]
@ -590884,6 +590998,7 @@
"source": [
"# How old are current listings? (days since first visible)\n",
"import datetime\n",
"\n",
"now = datetime.datetime(2026, 2, 14)\n",
"with_age = with_dates.with_columns(\n",
" ((pl.lit(now) - pl.col(\"listed_at\")).dt.total_days()).alias(\"days_on_market\")\n",
@ -590896,7 +591011,7 @@
"print(f\" P25: {age.quantile(0.25):.0f} days\")\n",
"print(f\" P75: {age.quantile(0.75):.0f} days\")\n",
"print(f\" P95: {age.quantile(0.95):.0f} days\")\n",
"print(f\" Max: {age.max():.0f} days ({age.max()/365:.1f} years)\")"
"print(f\" Max: {age.max():.0f} days ({age.max() / 365:.1f} years)\")"
]
},
{
@ -591749,8 +591864,12 @@
"source": [
"# Days on market distribution (cap at 2 years for readability)\n",
"capped = with_age.filter(pl.col(\"days_on_market\") <= 730)\n",
"fig = px.histogram(capped.to_pandas(), x=\"days_on_market\", nbins=100,\n",
" title=\"Days on Market Distribution (capped at 2 years)\")\n",
"fig = px.histogram(\n",
" capped.to_pandas(),\n",
" x=\"days_on_market\",\n",
" nbins=100,\n",
" title=\"Days on Market Distribution (capped at 2 years)\",\n",
")\n",
"fig.update_layout(height=400, xaxis_title=\"Days on Market\", yaxis_title=\"Count\")\n",
"fig.show()"
]
@ -591883,11 +592002,13 @@
"# Explode features list and count most common\n",
"features_exploded = clean.select(\"features\").explode(\"features\").drop_nulls()\n",
"print(f\"Total feature entries: {len(features_exploded):,}\")\n",
"print(f\"Features per listing: {len(features_exploded)/len(clean):.1f} avg\")\n",
"print(f\"Features per listing: {len(features_exploded) / len(clean):.1f} avg\")\n",
"\n",
"# Most common features (lowercased for grouping)\n",
"feature_counts = (\n",
" features_exploded.with_columns(pl.col(\"features\").str.to_lowercase().str.strip_chars().alias(\"feature_lower\"))\n",
" features_exploded.with_columns(\n",
" pl.col(\"features\").str.to_lowercase().str.strip_chars().alias(\"feature_lower\")\n",
" )\n",
" .group_by(\"feature_lower\")\n",
" .agg(pl.len().alias(\"count\"))\n",
" .sort(\"count\", descending=True)\n",
@ -592794,16 +592915,64 @@
"all_features = features_exploded[\"features\"].to_list()\n",
"word_counter = Counter()\n",
"for feat in all_features:\n",
" words = re.findall(r'[a-z]+', feat.lower())\n",
" words = re.findall(r\"[a-z]+\", feat.lower())\n",
" word_counter.update(words)\n",
"\n",
"# Filter out very short/common words\n",
"stop_words = {'the', 'a', 'an', 'and', 'or', 'of', 'to', 'in', 'with', 'for', 'on', 'at', 'by', 'is', 'it', 'from', 'as', 'be', 'this', 'that', 'are', 'was', 'has', 'have', 'not', 'but', 'all', 'can', 'had', 'her', 'his', 'one', 'our', 'out', 'you', 'will'}\n",
"keywords = [(w, c) for w, c in word_counter.most_common(100) if w not in stop_words and len(w) > 2]\n",
"kw_df = pl.DataFrame({\"word\": [w for w,c in keywords[:40]], \"count\": [c for w,c in keywords[:40]]})\n",
"stop_words = {\n",
" \"the\",\n",
" \"a\",\n",
" \"an\",\n",
" \"and\",\n",
" \"or\",\n",
" \"of\",\n",
" \"to\",\n",
" \"in\",\n",
" \"with\",\n",
" \"for\",\n",
" \"on\",\n",
" \"at\",\n",
" \"by\",\n",
" \"is\",\n",
" \"it\",\n",
" \"from\",\n",
" \"as\",\n",
" \"be\",\n",
" \"this\",\n",
" \"that\",\n",
" \"are\",\n",
" \"was\",\n",
" \"has\",\n",
" \"have\",\n",
" \"not\",\n",
" \"but\",\n",
" \"all\",\n",
" \"can\",\n",
" \"had\",\n",
" \"her\",\n",
" \"his\",\n",
" \"one\",\n",
" \"our\",\n",
" \"out\",\n",
" \"you\",\n",
" \"will\",\n",
"}\n",
"keywords = [\n",
" (w, c)\n",
" for w, c in word_counter.most_common(100)\n",
" if w not in stop_words and len(w) > 2\n",
"]\n",
"kw_df = pl.DataFrame(\n",
" {\"word\": [w for w, c in keywords[:40]], \"count\": [c for w, c in keywords[:40]]}\n",
")\n",
"\n",
"fig = px.bar(kw_df.to_pandas(), x=\"count\", y=\"word\", orientation=\"h\",\n",
" title=\"Most Common Words in Feature Descriptions\")\n",
"fig = px.bar(\n",
" kw_df.to_pandas(),\n",
" x=\"count\",\n",
" y=\"word\",\n",
" orientation=\"h\",\n",
" title=\"Most Common Words in Feature Descriptions\",\n",
")\n",
"fig.update_layout(height=800, yaxis={\"categoryorder\": \"total ascending\"})\n",
"fig.show()"
]
@ -593767,9 +593936,14 @@
" & (pl.col(\"price\") < 3_000_000)\n",
").sample(n=min(15_000, len(with_floor)), seed=42)\n",
"\n",
"fig = px.scatter(scatter_df.to_pandas(), x=\"floorspace_sqm\", y=\"price\",\n",
" color=\"property_type\", opacity=0.3,\n",
" title=\"Price vs Floorspace (sample, capped at £3M / 500sqm)\")\n",
"fig = px.scatter(\n",
" scatter_df.to_pandas(),\n",
" x=\"floorspace_sqm\",\n",
" y=\"price\",\n",
" color=\"property_type\",\n",
" opacity=0.3,\n",
" title=\"Price vs Floorspace (sample, capped at £3M / 500sqm)\",\n",
")\n",
"fig.update_layout(height=600, xaxis_title=\"Floorspace (sqm)\", yaxis_title=\"Price (£)\")\n",
"fig.show()"
]
@ -594739,8 +594913,14 @@
" .agg(pl.col(\"price\").median().alias(\"median_price\"), pl.len().alias(\"count\"))\n",
" .sort(\"property_type\", \"bedrooms\")\n",
")\n",
"fig = px.line(bp.to_pandas(), x=\"bedrooms\", y=\"median_price\", color=\"property_type\",\n",
" markers=True, title=\"Median Price by Bedrooms and Property Type\")\n",
"fig = px.line(\n",
" bp.to_pandas(),\n",
" x=\"bedrooms\",\n",
" y=\"median_price\",\n",
" color=\"property_type\",\n",
" markers=True,\n",
" title=\"Median Price by Bedrooms and Property Type\",\n",
")\n",
"fig.update_layout(height=450, xaxis_title=\"Bedrooms\", yaxis_title=\"Median Price (£)\")\n",
"fig.show()"
]
@ -594789,18 +594969,28 @@
"print(f\"Total listings: {len(clean):,}\")\n",
"print(f\"Outcodes covered: {clean['outcode'].n_unique():,}\")\n",
"print(\"\")\n",
"print(f\"Price: median £{clean['price'].median():,.0f}, mean £{clean['price'].mean():,.0f}\")\n",
"print(f\"Bedrooms: median {clean['bedrooms'].median():.0f}, mean {clean['bedrooms'].mean():.1f}\")\n",
"print(\n",
" f\"Price: median £{clean['price'].median():,.0f}, mean £{clean['price'].mean():,.0f}\"\n",
")\n",
"print(\n",
" f\"Bedrooms: median {clean['bedrooms'].median():.0f}, mean {clean['bedrooms'].mean():.1f}\"\n",
")\n",
"print(\"\")\n",
"print(f\"Tenure known: {(len(clean) - clean['tenure'].null_count())/len(clean)*100:.1f}%\")\n",
"print(\n",
" f\"Tenure known: {(len(clean) - clean['tenure'].null_count()) / len(clean) * 100:.1f}%\"\n",
")\n",
"print(f\" Freehold: {len(clean.filter(pl.col('tenure') == 'Freehold')):,}\")\n",
"print(f\" Leasehold: {len(clean.filter(pl.col('tenure') == 'Leasehold')):,}\")\n",
"print(\"\")\n",
"print(f\"Floorspace available: {clean['floorspace_sqm'].drop_nulls().len()/len(clean)*100:.1f}%\")\n",
"print(\n",
" f\"Floorspace available: {clean['floorspace_sqm'].drop_nulls().len() / len(clean) * 100:.1f}%\"\n",
")\n",
"print(\"\")\n",
"print(\"Property types:\")\n",
"for row in clean['property_type'].value_counts().sort('count', descending=True).iter_rows():\n",
" print(f\" {row[0]}: {row[1]:,} ({row[1]/len(clean)*100:.1f}%)\")"
"for row in (\n",
" clean[\"property_type\"].value_counts().sort(\"count\", descending=True).iter_rows()\n",
"):\n",
" print(f\" {row[0]}: {row[1]:,} ({row[1] / len(clean) * 100:.1f}%)\")"
]
}
],

255
analyses/source_overlap.ipynb
View file

@ -52,6 +52,7 @@
"buy = pl.read_parquet(f\"{DATA}/online_listings_buy.parquet\")\n",
"rent = pl.read_parquet(f\"{DATA}/online_listings_rent.parquet\")\n",
"\n",
"\n",
"def tag_source(df: pl.DataFrame) -> pl.DataFrame:\n",
" return df.with_columns(\n",
" pl.when(pl.col(\"Listing URL\").str.contains(\"rightmove\"))\n",
@ -62,6 +63,7 @@
" .alias(\"source\")\n",
" )\n",
"\n",
"\n",
"buy = tag_source(buy)\n",
"rent = tag_source(rent)\n",
"\n",
@ -122,7 +124,7 @@
" print(f\"\\n=== {label} ===\")\n",
" for row in counts.iter_rows():\n",
" src, cnt = row\n",
" print(f\" {src}: {cnt:,} ({cnt/len(df)*100:.1f}%)\")\n",
" print(f\" {src}: {cnt:,} ({cnt / len(df) * 100:.1f}%)\")\n",
"\n",
"# Known dedup count from scraper logs\n",
"CROSS_DEDUP_BUY = 2_220\n",
@ -132,7 +134,7 @@
"print(f\"Home.co.uk scraped (before dedup): {hk_buy_total:,}\")\n",
"print(f\"Home.co.uk unique (after dedup): {hk_buy_unique:,}\")\n",
"print(f\"Cross-source duplicates removed: {CROSS_DEDUP_BUY:,}\")\n",
"print(f\"Overlap rate: {CROSS_DEDUP_BUY/hk_buy_total*100:.1f}%\")"
"print(f\"Overlap rate: {CROSS_DEDUP_BUY / hk_buy_total * 100:.1f}%\")"
]
},
{
@ -987,23 +989,29 @@
"# Venn-style summary\n",
"rm_buy = len(buy.filter(pl.col(\"source\") == \"Rightmove\"))\n",
"\n",
"fig = go.Figure(go.Sankey(\n",
" node=dict(\n",
" label=[\n",
" f\"Rightmove\\n{rm_buy:,}\",\n",
" f\"Home.co.uk\\n{hk_buy_total:,} scraped\",\n",
" f\"Merged BUY\\n{len(buy):,}\",\n",
" f\"Deduped\\n{CROSS_DEDUP_BUY:,}\",\n",
" ],\n",
" color=[\"#2563eb\", \"#10b981\", \"#6366f1\", \"#ef4444\"],\n",
" ),\n",
" link=dict(\n",
" source=[0, 1, 1],\n",
" target=[2, 2, 3],\n",
" value=[rm_buy, hk_buy_unique, CROSS_DEDUP_BUY],\n",
" color=[\"rgba(37,99,235,0.3)\", \"rgba(16,185,129,0.3)\", \"rgba(239,68,68,0.3)\"],\n",
" ),\n",
"))\n",
"fig = go.Figure(\n",
" go.Sankey(\n",
" node=dict(\n",
" label=[\n",
" f\"Rightmove\\n{rm_buy:,}\",\n",
" f\"Home.co.uk\\n{hk_buy_total:,} scraped\",\n",
" f\"Merged BUY\\n{len(buy):,}\",\n",
" f\"Deduped\\n{CROSS_DEDUP_BUY:,}\",\n",
" ],\n",
" color=[\"#2563eb\", \"#10b981\", \"#6366f1\", \"#ef4444\"],\n",
" ),\n",
" link=dict(\n",
" source=[0, 1, 1],\n",
" target=[2, 2, 3],\n",
" value=[rm_buy, hk_buy_unique, CROSS_DEDUP_BUY],\n",
" color=[\n",
" \"rgba(37,99,235,0.3)\",\n",
" \"rgba(16,185,129,0.3)\",\n",
" \"rgba(239,68,68,0.3)\",\n",
" ],\n",
" ),\n",
" )\n",
")\n",
"fig.update_layout(title=\"BUY Channel: Source Contribution Flow\", height=350)\n",
"fig.show()"
]
@ -1106,8 +1114,11 @@
"oc_comparison = (\n",
" hk_by_oc.join(rm_by_oc, on=\"outcode\", how=\"left\")\n",
" .with_columns(\n",
" (pl.col(\"hk_count\") / (pl.col(\"hk_count\") + pl.col(\"rm_count\").fill_null(0)) * 100)\n",
" .alias(\"hk_pct_of_total\")\n",
" (\n",
" pl.col(\"hk_count\")\n",
" / (pl.col(\"hk_count\") + pl.col(\"rm_count\").fill_null(0))\n",
" * 100\n",
" ).alias(\"hk_pct_of_total\")\n",
" )\n",
" .sort(\"hk_count\", descending=True)\n",
")\n",
@ -2215,18 +2226,28 @@
"source": [
"# Bar chart: home.co.uk vs Rightmove counts per outcode\n",
"fig = go.Figure()\n",
"fig.add_trace(go.Bar(\n",
" x=oc_comparison[\"outcode\"], y=oc_comparison[\"rm_count\"],\n",
" name=\"Rightmove\", marker_color=\"#2563eb\",\n",
"))\n",
"fig.add_trace(go.Bar(\n",
" x=oc_comparison[\"outcode\"], y=oc_comparison[\"hk_count\"],\n",
" name=\"Home.co.uk\", marker_color=\"#10b981\",\n",
"))\n",
"fig.add_trace(\n",
" go.Bar(\n",
" x=oc_comparison[\"outcode\"],\n",
" y=oc_comparison[\"rm_count\"],\n",
" name=\"Rightmove\",\n",
" marker_color=\"#2563eb\",\n",
" )\n",
")\n",
"fig.add_trace(\n",
" go.Bar(\n",
" x=oc_comparison[\"outcode\"],\n",
" y=oc_comparison[\"hk_count\"],\n",
" name=\"Home.co.uk\",\n",
" marker_color=\"#10b981\",\n",
" )\n",
")\n",
"fig.update_layout(\n",
" barmode=\"group\", height=400,\n",
" barmode=\"group\",\n",
" height=400,\n",
" title=\"Listings per Outcode: Rightmove vs Home.co.uk (outcodes with HK coverage)\",\n",
" xaxis_title=\"Outcode\", yaxis_title=\"Listings\",\n",
" xaxis_title=\"Outcode\",\n",
" yaxis_title=\"Listings\",\n",
")\n",
"fig.show()"
]
@ -3121,10 +3142,14 @@
"sample = covered.sample(n=min(30_000, len(covered)), seed=42)\n",
"\n",
"fig = px.scatter_map(\n",
" sample.to_pandas(), lat=\"lat\", lon=\"lon\",\n",
" sample.to_pandas(),\n",
" lat=\"lat\",\n",
" lon=\"lon\",\n",
" color=\"source\",\n",
" color_discrete_map={\"Rightmove\": \"#2563eb\", \"Home.co.uk\": \"#10b981\"},\n",
" zoom=7, opacity=0.4, size_max=4,\n",
" zoom=7,\n",
" opacity=0.4,\n",
" size_max=4,\n",
" title=\"Listing Locations in Covered Outcodes (by source)\",\n",
")\n",
"fig.update_layout(height=600)\n",
@ -3188,15 +3213,41 @@
"# For covered outcodes, compare home.co.uk listings against Rightmove\n",
"# to find near-matches (same postcode, same beds, price within 5%)\n",
"\n",
"hk = buy_oc.filter(pl.col(\"source\") == \"Home.co.uk\").select(\n",
" \"Postcode\", \"Bedrooms\", \"Asking price\", \"Property type\", \"Address per Property Register\"\n",
").rename({\"Asking price\": \"hk_price\", \"Property type\": \"hk_type\", \"Address per Property Register\": \"hk_addr\"})\n",
"hk = (\n",
" buy_oc.filter(pl.col(\"source\") == \"Home.co.uk\")\n",
" .select(\n",
" \"Postcode\",\n",
" \"Bedrooms\",\n",
" \"Asking price\",\n",
" \"Property type\",\n",
" \"Address per Property Register\",\n",
" )\n",
" .rename(\n",
" {\n",
" \"Asking price\": \"hk_price\",\n",
" \"Property type\": \"hk_type\",\n",
" \"Address per Property Register\": \"hk_addr\",\n",
" }\n",
" )\n",
")\n",
"\n",
"rm = buy_oc.filter(\n",
" pl.col(\"source\") == \"Rightmove\"\n",
").select(\n",
" \"Postcode\", \"Bedrooms\", \"Asking price\", \"Property type\", \"Address per Property Register\"\n",
").rename({\"Asking price\": \"rm_price\", \"Property type\": \"rm_type\", \"Address per Property Register\": \"rm_addr\"})\n",
"rm = (\n",
" buy_oc.filter(pl.col(\"source\") == \"Rightmove\")\n",
" .select(\n",
" \"Postcode\",\n",
" \"Bedrooms\",\n",
" \"Asking price\",\n",
" \"Property type\",\n",
" \"Address per Property Register\",\n",
" )\n",
" .rename(\n",
" {\n",
" \"Asking price\": \"rm_price\",\n",
" \"Property type\": \"rm_type\",\n",
" \"Address per Property Register\": \"rm_addr\",\n",
" }\n",
" )\n",
")\n",
"\n",
"# Join on postcode + bedrooms\n",
"joined = hk.join(rm, on=[\"Postcode\", \"Bedrooms\"], how=\"inner\")\n",
@ -3213,16 +3264,24 @@
"exact = joined.filter(pl.col(\"hk_price\") == pl.col(\"rm_price\"))\n",
"\n",
"print(f\"Home.co.uk listings (unique, in file): {len(hk):,}\")\n",
"print(f\"Rightmove listings in covered outcodes: {len(rm.filter(pl.col('Postcode').is_in(hk['Postcode']))):,}\")\n",
"print(\n",
" f\"Rightmove listings in covered outcodes: {len(rm.filter(pl.col('Postcode').is_in(hk['Postcode']))):,}\"\n",
")\n",
"print()\n",
"print(f\"Joined on (postcode, bedrooms): {len(joined):,} candidate pairs\")\n",
"print(f\" Exact price match: {len(exact):,} pairs (likely same property, different beds or already deduped)\")\n",
"print(f\" Price within 5%: {len(near):,} pairs (probable duplicates with price rounding)\")\n",
"print(\n",
" f\" Exact price match: {len(exact):,} pairs (likely same property, different beds or already deduped)\"\n",
")\n",
"print(\n",
" f\" Price within 5%: {len(near):,} pairs (probable duplicates with price rounding)\"\n",
")\n",
"print()\n",
"# Unique hk listings that have at least one near-match\n",
"hk_with_near = near.select(\"hk_price\", \"hk_addr\", \"Postcode\").unique()\n",
"print(f\"Home.co.uk listings with a near-match in RM: ~{len(hk_with_near):,}\")\n",
"print(f\"Estimated additional overlap: ~{len(hk_with_near)/len(hk)*100:.1f}% of unique HK listings\")"
"print(\n",
" f\"Estimated additional overlap: ~{len(hk_with_near) / len(hk) * 100:.1f}% of unique HK listings\"\n",
")"
]
},
{
@ -4178,9 +4237,13 @@
")\n",
"\n",
"fig = px.histogram(\n",
" clipped.to_pandas(), x=\"Asking price\", color=\"source\", nbins=80,\n",
" clipped.to_pandas(),\n",
" x=\"Asking price\",\n",
" color=\"source\",\n",
" nbins=80,\n",
" color_discrete_map={\"Rightmove\": \"#2563eb\", \"Home.co.uk\": \"#10b981\"},\n",
" barmode=\"overlay\", histnorm=\"probability density\",\n",
" barmode=\"overlay\",\n",
" histnorm=\"probability density\",\n",
" title=\"Price Distribution by Source (normalised, £50k–£2M)\",\n",
")\n",
"fig.update_traces(opacity=0.6)\n",
@ -5095,10 +5158,7 @@
],
"source": [
"# Property type distribution by source\n",
"type_by_src = (\n",
" buy.group_by(\"source\", \"Property type\")\n",
" .agg(pl.len().alias(\"count\"))\n",
")\n",
"type_by_src = buy.group_by(\"source\", \"Property type\").agg(pl.len().alias(\"count\"))\n",
"# Normalise within each source\n",
"totals = type_by_src.group_by(\"source\").agg(pl.col(\"count\").sum().alias(\"total\"))\n",
"type_by_src = type_by_src.join(totals, on=\"source\").with_columns(\n",
@ -5107,7 +5167,10 @@
"\n",
"fig = px.bar(\n",
" type_by_src.sort(\"Property type\").to_pandas(),\n",
" x=\"Property type\", y=\"pct\", color=\"source\", barmode=\"group\",\n",
" x=\"Property type\",\n",
" y=\"pct\",\n",
" color=\"source\",\n",
" barmode=\"group\",\n",
" color_discrete_map={\"Rightmove\": \"#2563eb\", \"Home.co.uk\": \"#10b981\"},\n",
" title=\"Property Type Distribution by Source (%)\",\n",
")\n",
@ -5186,7 +5249,9 @@
"# Property sub-type comparison — top home.co.uk sub-types\n",
"hk_subtypes = (\n",
" buy.filter(pl.col(\"source\") == \"Home.co.uk\")[\"Property sub-type\"]\n",
" .value_counts().sort(\"count\", descending=True).head(20)\n",
" .value_counts()\n",
" .sort(\"count\", descending=True)\n",
" .head(20)\n",
")\n",
"print(\"Top 20 Home.co.uk property sub-types:\")\n",
"hk_subtypes"
@ -5263,9 +5328,16 @@
"source": [
"# Field completeness by source\n",
"fields = [\n",
" \"Bedrooms\", \"Bathrooms\", \"Postcode\", \"Address per Property Register\",\n",
" \"Leasehold/Freehold\", \"Property type\", \"Total floor area (sqm)\",\n",
" \"Listing date\", \"Asking price\", \"Price qualifier\",\n",
" \"Bedrooms\",\n",
" \"Bathrooms\",\n",
" \"Postcode\",\n",
" \"Address per Property Register\",\n",
" \"Leasehold/Freehold\",\n",
" \"Property type\",\n",
" \"Total floor area (sqm)\",\n",
" \"Listing date\",\n",
" \"Asking price\",\n",
" \"Price qualifier\",\n",
"]\n",
"\n",
"rows = []\n",
@ -5276,17 +5348,19 @@
" non_null = n - subset[f].null_count()\n",
" # Also count empty strings as missing for string fields\n",
" if subset[f].dtype == pl.Utf8:\n",
" non_null = len(subset.filter(\n",
" pl.col(f).is_not_null() & (pl.col(f).str.len_chars() > 0)\n",
" ))\n",
" non_null = len(\n",
" subset.filter(pl.col(f).is_not_null() & (pl.col(f).str.len_chars() > 0))\n",
" )\n",
" rows.append({\"source\": src, \"field\": f, \"pct_available\": non_null / n * 100})\n",
"\n",
"completeness = pl.DataFrame(rows)\n",
"pivot = completeness.pivot(on=\"source\", index=\"field\", values=\"pct_available\")\n",
"pivot = pivot.with_columns([\n",
" pl.col(\"Rightmove\").round(1),\n",
" pl.col(\"Home.co.uk\").round(1),\n",
"])\n",
"pivot = pivot.with_columns(\n",
" [\n",
" pl.col(\"Rightmove\").round(1),\n",
" pl.col(\"Home.co.uk\").round(1),\n",
" ]\n",
")\n",
"print(\"Field completeness (% non-null/non-empty):\")\n",
"pivot"
]
@ -6198,19 +6272,26 @@
"# Bedroom distribution comparison\n",
"fig = make_subplots(rows=1, cols=2, subplot_titles=(\"Rightmove\", \"Home.co.uk\"))\n",
"for i, src in enumerate([\"Rightmove\", \"Home.co.uk\"], 1):\n",
" beds = buy.filter(\n",
" (pl.col(\"source\") == src) & (pl.col(\"Bedrooms\") <= 8)\n",
" )[\"Bedrooms\"].value_counts().sort(\"Bedrooms\")\n",
" beds = (\n",
" buy.filter((pl.col(\"source\") == src) & (pl.col(\"Bedrooms\") <= 8))[\"Bedrooms\"]\n",
" .value_counts()\n",
" .sort(\"Bedrooms\")\n",
" )\n",
" # Normalise\n",
" total = beds[\"count\"].sum()\n",
" fig.add_trace(\n",
" go.Bar(\n",
" x=beds[\"Bedrooms\"], y=beds[\"count\"] / total * 100,\n",
" x=beds[\"Bedrooms\"],\n",
" y=beds[\"count\"] / total * 100,\n",
" name=src,\n",
" marker_color=\"#2563eb\" if src == \"Rightmove\" else \"#10b981\",\n",
" ), row=1, col=i,\n",
" ),\n",
" row=1,\n",
" col=i,\n",
" )\n",
"fig.update_layout(height=350, title=\"Bedroom Distribution by Source (%)\", showlegend=False)\n",
"fig.update_layout(\n",
" height=350, title=\"Bedroom Distribution by Source (%)\", showlegend=False\n",
")\n",
"fig.update_yaxes(title_text=\"%\", row=1, col=1)\n",
"fig.show()"
]
@ -6287,17 +6368,23 @@
"\n",
"comparison_rows = []\n",
"for ptype in [\"Detached\", \"Semi-Detached\", \"Terraced\", \"Flats/Maisonettes\", \"Other\"]:\n",
" rm_p = rm_covered.filter(pl.col(\"Property type\") == ptype)[\"Asking price\"].drop_nulls()\n",
" rm_p = rm_covered.filter(pl.col(\"Property type\") == ptype)[\n",
" \"Asking price\"\n",
" ].drop_nulls()\n",
" hk_p = hk_only.filter(pl.col(\"Property type\") == ptype)[\"Asking price\"].drop_nulls()\n",
" if len(rm_p) > 0 and len(hk_p) > 0:\n",
" comparison_rows.append({\n",
" \"Property type\": ptype,\n",
" \"RM count\": len(rm_p),\n",
" \"RM median £\": int(rm_p.median()),\n",
" \"HK count\": len(hk_p),\n",
" \"HK median £\": int(hk_p.median()),\n",
" \"HK premium %\": round((hk_p.median() - rm_p.median()) / rm_p.median() * 100, 1),\n",
" })\n",
" comparison_rows.append(\n",
" {\n",
" \"Property type\": ptype,\n",
" \"RM count\": len(rm_p),\n",
" \"RM median £\": int(rm_p.median()),\n",
" \"HK count\": len(hk_p),\n",
" \"HK median £\": int(hk_p.median()),\n",
" \"HK premium %\": round(\n",
" (hk_p.median() - rm_p.median()) / rm_p.median() * 100, 1\n",
" ),\n",
" }\n",
" )\n",
"\n",
"comp = pl.DataFrame(comparison_rows)\n",
"print(\"Price comparison in covered outcodes (Home.co.uk unique listings vs Rightmove):\")\n",
@ -7245,9 +7332,13 @@
"# Listing age histogram comparison\n",
"age_plot = with_age.filter(pl.col(\"days_on_market\") <= 730) # cap at 2 years\n",
"fig = px.histogram(\n",
" age_plot.to_pandas(), x=\"days_on_market\", color=\"source\", nbins=60,\n",
" age_plot.to_pandas(),\n",
" x=\"days_on_market\",\n",
" color=\"source\",\n",
" nbins=60,\n",
" color_discrete_map={\"Rightmove\": \"#2563eb\", \"Home.co.uk\": \"#10b981\"},\n",
" barmode=\"overlay\", histnorm=\"probability density\",\n",
" barmode=\"overlay\",\n",
" histnorm=\"probability density\",\n",
" title=\"Days on Market Distribution by Source (normalised, capped at 2 years)\",\n",
")\n",
"fig.update_traces(opacity=0.6)\n",
@ -7330,7 +7421,9 @@
"print(f\" Projected home.co.uk total: ~{projected_hk:,}\")\n",
"print(f\" Projected cross-dedup: ~{projected_dedup:,}\")\n",
"print(f\" Projected unique additions: ~{projected_unique:,}\")\n",
"print(f\" Projected merged dataset: ~{rm_buy + projected_unique:,} ({projected_unique/rm_buy*100:.1f}% increase)\")\n",
"print(\n",
" f\" Projected merged dataset: ~{rm_buy + projected_unique:,} ({projected_unique / rm_buy * 100:.1f}% increase)\"\n",
")\n",
"print()\n",
"print(\"⚠️ These are rough estimates — the covered outcodes may not be representative\")"
]

313
analyses/travel_time_comparison.ipynb
View file

@ -54,11 +54,15 @@
}
],
"source": [
"r5_bank = pl.read_parquet(\"../property-data/travel-times/transit/000000-bank-tube-station.parquet\")\n",
"r5_bank = pl.read_parquet(\n",
" \"../property-data/travel-times/transit/000000-bank-tube-station.parquet\"\n",
")\n",
"manual_bank = pl.read_parquet(\"../manual-data/journey_times_bank.parquet\")\n",
"\n",
"print(f\"R5 Bank: {r5_bank.shape[0]:,} postcodes\")\n",
"print(f\"Manual Bank: {manual_bank.shape[0]:,} postcodes ({manual_bank['public_transport_easy_minutes'].null_count():,} null easy)\")"
"print(\n",
" f\"Manual Bank: {manual_bank.shape[0]:,} postcodes ({manual_bank['public_transport_easy_minutes'].null_count():,} null easy)\"\n",
")"
]
},
{
@ -116,25 +120,49 @@
"source": [
"# Join on postcode, keep only rows where both sources have values\n",
"bank = (\n",
" r5_bank\n",
" .join(manual_bank, left_on=\"pcds\", right_on=\"postcode\", how=\"inner\")\n",
" r5_bank.join(manual_bank, left_on=\"pcds\", right_on=\"postcode\", how=\"inner\")\n",
" .filter(\n",
" pl.col(\"public_transport_easy_minutes\").is_not_null()\n",
" & pl.col(\"public_transport_quick_minutes\").is_not_null()\n",
" )\n",
" .with_columns([\n",
" # Signed error: R5 - Manual (positive = R5 is slower)\n",
" (pl.col(\"travel_minutes\").cast(pl.Float64) - pl.col(\"public_transport_easy_minutes\").cast(pl.Float64)).alias(\"error_easy\"),\n",
" (pl.col(\"best_minutes\").cast(pl.Float64) - pl.col(\"public_transport_quick_minutes\").cast(pl.Float64)).alias(\"error_quick\"),\n",
" # Absolute error\n",
" (pl.col(\"travel_minutes\").cast(pl.Float64) - pl.col(\"public_transport_easy_minutes\").cast(pl.Float64)).abs().alias(\"abs_error_easy\"),\n",
" (pl.col(\"best_minutes\").cast(pl.Float64) - pl.col(\"public_transport_quick_minutes\").cast(pl.Float64)).abs().alias(\"abs_error_quick\"),\n",
" ])\n",
" .with_columns(\n",
" [\n",
" # Signed error: R5 - Manual (positive = R5 is slower)\n",
" (\n",
" pl.col(\"travel_minutes\").cast(pl.Float64)\n",
" - pl.col(\"public_transport_easy_minutes\").cast(pl.Float64)\n",
" ).alias(\"error_easy\"),\n",
" (\n",
" pl.col(\"best_minutes\").cast(pl.Float64)\n",
" - pl.col(\"public_transport_quick_minutes\").cast(pl.Float64)\n",
" ).alias(\"error_quick\"),\n",
" # Absolute error\n",
" (\n",
" pl.col(\"travel_minutes\").cast(pl.Float64)\n",
" - pl.col(\"public_transport_easy_minutes\").cast(pl.Float64)\n",
" )\n",
" .abs()\n",
" .alias(\"abs_error_easy\"),\n",
" (\n",
" pl.col(\"best_minutes\").cast(pl.Float64)\n",
" - pl.col(\"public_transport_quick_minutes\").cast(pl.Float64)\n",
" )\n",
" .abs()\n",
" .alias(\"abs_error_quick\"),\n",
" ]\n",
" )\n",
")\n",
"\n",
"print(f\"Joined (non-null): {bank.shape[0]:,} postcodes\")\n",
"bank.select(\"pcds\", \"travel_minutes\", \"public_transport_easy_minutes\", \"error_easy\",\n",
" \"best_minutes\", \"public_transport_quick_minutes\", \"error_quick\").head(10)"
"bank.select(\n",
" \"pcds\",\n",
" \"travel_minutes\",\n",
" \"public_transport_easy_minutes\",\n",
" \"error_easy\",\n",
" \"best_minutes\",\n",
" \"public_transport_quick_minutes\",\n",
" \"error_quick\",\n",
").head(10)"
]
},
{
@ -196,18 +224,23 @@
" percentiles = [5, 25, 50, 80, 90, 95, 99]\n",
" rows = []\n",
" for p in percentiles:\n",
" rows.append({\n",
" \"percentile\": f\"p{p}\",\n",
" f\"{label} signed error\": round(float(np.percentile(col, p)), 1),\n",
" f\"{label} absolute error\": round(float(np.percentile(abs_col, p)), 1),\n",
" })\n",
" rows.append({\n",
" \"percentile\": \"mean\",\n",
" f\"{label} signed error\": round(float(np.mean(col)), 1),\n",
" f\"{label} absolute error\": round(float(np.mean(abs_col)), 1),\n",
" })\n",
" rows.append(\n",
" {\n",
" \"percentile\": f\"p{p}\",\n",
" f\"{label} signed error\": round(float(np.percentile(col, p)), 1),\n",
" f\"{label} absolute error\": round(float(np.percentile(abs_col, p)), 1),\n",
" }\n",
" )\n",
" rows.append(\n",
" {\n",
" \"percentile\": \"mean\",\n",
" f\"{label} signed error\": round(float(np.mean(col)), 1),\n",
" f\"{label} absolute error\": round(float(np.mean(abs_col)), 1),\n",
" }\n",
" )\n",
" return pl.DataFrame(rows)\n",
"\n",
"\n",
"stats_easy = percentile_stats(\"error_easy\", \"Median (easy)\")\n",
"stats_quick = percentile_stats(\"error_quick\", \"Best (quick)\")\n",
"\n",
@ -1120,24 +1153,42 @@
}
],
"source": [
"fig = make_subplots(rows=1, cols=2, subplot_titles=[\n",
" \"Median transit time error (R5 − TfL)\",\n",
" \"Best transit time error (R5 − TfL)\"\n",
"])\n",
"fig = make_subplots(\n",
" rows=1,\n",
" cols=2,\n",
" subplot_titles=[\n",
" \"Median transit time error (R5 − TfL)\",\n",
" \"Best transit time error (R5 − TfL)\",\n",
" ],\n",
")\n",
"\n",
"# Clip for readability\n",
"easy_clipped = bank[\"error_easy\"].clip(-60, 60).to_numpy()\n",
"quick_clipped = bank[\"error_quick\"].clip(-60, 60).to_numpy()\n",
"\n",
"fig.add_trace(go.Histogram(x=easy_clipped, nbinsx=120, name=\"Median (easy)\",\n",
" marker_color=\"#0d9488\"), row=1, col=1)\n",
"fig.add_trace(go.Histogram(x=quick_clipped, nbinsx=120, name=\"Best (quick)\",\n",
" marker_color=\"#f59e0b\"), row=1, col=2)\n",
"fig.add_trace(\n",
" go.Histogram(\n",
" x=easy_clipped, nbinsx=120, name=\"Median (easy)\", marker_color=\"#0d9488\"\n",
" ),\n",
" row=1,\n",
" col=1,\n",
")\n",
"fig.add_trace(\n",
" go.Histogram(\n",
" x=quick_clipped, nbinsx=120, name=\"Best (quick)\", marker_color=\"#f59e0b\"\n",
" ),\n",
" row=1,\n",
" col=2,\n",
")\n",
"\n",
"fig.update_xaxes(title_text=\"Error (minutes)\", row=1, col=1)\n",
"fig.update_xaxes(title_text=\"Error (minutes)\", row=1, col=2)\n",
"fig.update_yaxes(title_text=\"Count\", row=1, col=1)\n",
"fig.update_layout(height=400, showlegend=False, title_text=\"Bank: Error Distribution (clipped ±60 min)\")\n",
"fig.update_layout(\n",
" height=400,\n",
" showlegend=False,\n",
" title_text=\"Bank: Error Distribution (clipped ±60 min)\",\n",
")\n",
"fig.show()"
]
},
@ -2104,34 +2155,55 @@
"# Sample for scatter plot performance\n",
"sample = bank.sample(n=min(20_000, bank.shape[0]), seed=42)\n",
"\n",
"fig = make_subplots(rows=1, cols=2, subplot_titles=[\n",
" \"Median: R5 vs TfL (easy)\",\n",
" \"Best: R5 vs TfL (quick)\"\n",
"])\n",
"fig = make_subplots(\n",
" rows=1,\n",
" cols=2,\n",
" subplot_titles=[\"Median: R5 vs TfL (easy)\", \"Best: R5 vs TfL (quick)\"],\n",
")\n",
"\n",
"fig.add_trace(go.Scattergl(\n",
" x=sample[\"public_transport_easy_minutes\"].to_numpy(),\n",
" y=sample[\"travel_minutes\"].cast(pl.Float64).to_numpy(),\n",
" mode=\"markers\", marker=dict(size=2, opacity=0.3, color=\"#0d9488\"),\n",
" name=\"Median\"\n",
"), row=1, col=1)\n",
"fig.add_trace(\n",
" go.Scattergl(\n",
" x=sample[\"public_transport_easy_minutes\"].to_numpy(),\n",
" y=sample[\"travel_minutes\"].cast(pl.Float64).to_numpy(),\n",
" mode=\"markers\",\n",
" marker=dict(size=2, opacity=0.3, color=\"#0d9488\"),\n",
" name=\"Median\",\n",
" ),\n",
" row=1,\n",
" col=1,\n",
")\n",
"\n",
"fig.add_trace(go.Scattergl(\n",
" x=sample[\"public_transport_quick_minutes\"].to_numpy(),\n",
" y=sample[\"best_minutes\"].cast(pl.Float64).to_numpy(),\n",
" mode=\"markers\", marker=dict(size=2, opacity=0.3, color=\"#f59e0b\"),\n",
" name=\"Best\"\n",
"), row=1, col=2)\n",
"fig.add_trace(\n",
" go.Scattergl(\n",
" x=sample[\"public_transport_quick_minutes\"].to_numpy(),\n",
" y=sample[\"best_minutes\"].cast(pl.Float64).to_numpy(),\n",
" mode=\"markers\",\n",
" marker=dict(size=2, opacity=0.3, color=\"#f59e0b\"),\n",
" name=\"Best\",\n",
" ),\n",
" row=1,\n",
" col=2,\n",
")\n",
"\n",
"# Perfect agreement line\n",
"for col in [1, 2]:\n",
" fig.add_trace(go.Scatter(x=[0, 200], y=[0, 200], mode=\"lines\",\n",
" line=dict(color=\"red\", dash=\"dash\", width=1),\n",
" showlegend=False), row=1, col=col)\n",
" fig.add_trace(\n",
" go.Scatter(\n",
" x=[0, 200],\n",
" y=[0, 200],\n",
" mode=\"lines\",\n",
" line=dict(color=\"red\", dash=\"dash\", width=1),\n",
" showlegend=False,\n",
" ),\n",
" row=1,\n",
" col=col,\n",
" )\n",
" fig.update_xaxes(title_text=\"TfL API (minutes)\", row=1, col=col)\n",
" fig.update_yaxes(title_text=\"R5 (minutes)\", row=1, col=col)\n",
"\n",
"fig.update_layout(height=500, showlegend=False, title_text=\"Bank: R5 vs TfL API (20k sample)\")\n",
"fig.update_layout(\n",
" height=500, showlegend=False, title_text=\"Bank: R5 vs TfL API (20k sample)\"\n",
")\n",
"fig.show()"
]
},
@ -403063,7 +403135,8 @@
"\n",
"fig = px.scatter_map(\n",
" map_sample.to_pandas(),\n",
" lat=\"lat\", lon=\"long\",\n",
" lat=\"lat\",\n",
" lon=\"long\",\n",
" color=\"error_easy\",\n",
" color_continuous_scale=\"RdBu_r\", # red=positive (R5 slower), blue=negative (R5 faster)\n",
" range_color=[-30, 30],\n",
@ -403071,8 +403144,14 @@
" center={\"lat\": 51.5, \"lon\": -0.1},\n",
" opacity=0.5,\n",
" title=\"Bank — Median transit error (R5 − TfL easy), minutes\",\n",
" hover_data={\"pcds\": True, \"travel_minutes\": True, \"public_transport_easy_minutes\": True,\n",
" \"error_easy\": \":.0f\", \"lat\": False, \"long\": False},\n",
" hover_data={\n",
" \"pcds\": True,\n",
" \"travel_minutes\": True,\n",
" \"public_transport_easy_minutes\": True,\n",
" \"error_easy\": \":.0f\",\n",
" \"lat\": False,\n",
" \"long\": False,\n",
" },\n",
" height=700,\n",
")\n",
"fig.update_layout(map_style=\"carto-positron\")\n",
@ -803994,7 +804073,8 @@
"source": [
"fig = px.scatter_map(\n",
" map_sample.to_pandas(),\n",
" lat=\"lat\", lon=\"long\",\n",
" lat=\"lat\",\n",
" lon=\"long\",\n",
" color=\"error_quick\",\n",
" color_continuous_scale=\"RdBu_r\",\n",
" range_color=[-30, 30],\n",
@ -804002,8 +804082,14 @@
" center={\"lat\": 51.5, \"lon\": -0.1},\n",
" opacity=0.5,\n",
" title=\"Bank — Best transit error (R5 − TfL quick), minutes\",\n",
" hover_data={\"pcds\": True, \"best_minutes\": True, \"public_transport_quick_minutes\": True,\n",
" \"error_quick\": \":.0f\", \"lat\": False, \"long\": False},\n",
" hover_data={\n",
" \"pcds\": True,\n",
" \"best_minutes\": True,\n",
" \"public_transport_quick_minutes\": True,\n",
" \"error_quick\": \":.0f\",\n",
" \"lat\": False,\n",
" \"long\": False,\n",
" },\n",
" height=700,\n",
")\n",
"fig.update_layout(map_style=\"carto-positron\")\n",
@ -1204925,7 +1205011,8 @@
"source": [
"fig = px.scatter_map(\n",
" map_sample.to_pandas(),\n",
" lat=\"lat\", lon=\"long\",\n",
" lat=\"lat\",\n",
" lon=\"long\",\n",
" color=\"abs_error_easy\",\n",
" color_continuous_scale=\"YlOrRd\",\n",
" range_color=[0, 30],\n",
@ -1204933,8 +1205020,14 @@
" center={\"lat\": 51.5, \"lon\": -0.1},\n",
" opacity=0.5,\n",
" title=\"Bank — Absolute median transit error |R5 − TfL easy|, minutes\",\n",
" hover_data={\"pcds\": True, \"travel_minutes\": True, \"public_transport_easy_minutes\": True,\n",
" \"abs_error_easy\": \":.0f\", \"lat\": False, \"long\": False},\n",
" hover_data={\n",
" \"pcds\": True,\n",
" \"travel_minutes\": True,\n",
" \"public_transport_easy_minutes\": True,\n",
" \"abs_error_easy\": \":.0f\",\n",
" \"lat\": False,\n",
" \"long\": False,\n",
" },\n",
" height=700,\n",
")\n",
"fig.update_layout(map_style=\"carto-positron\")\n",
@ -1204998,9 +1205091,15 @@
],
"source": [
"bank.sort(\"abs_error_easy\", descending=True).select(\n",
" \"pcds\", \"lat\", \"long\",\n",
" \"travel_minutes\", \"public_transport_easy_minutes\", \"error_easy\",\n",
" \"best_minutes\", \"public_transport_quick_minutes\", \"error_quick\",\n",
" \"pcds\",\n",
" \"lat\",\n",
" \"long\",\n",
" \"travel_minutes\",\n",
" \"public_transport_easy_minutes\",\n",
" \"error_easy\",\n",
" \"best_minutes\",\n",
" \"public_transport_quick_minutes\",\n",
" \"error_quick\",\n",
").head(30)"
]
},
@ -1205945,45 +1206044,75 @@
"\n",
"dist_df = bank.with_columns(\n",
" # Rough km distance using Haversine approximation\n",
" ((((pl.col(\"lat\") - BANK_LAT) * 111.32) ** 2 +\n",
" ((pl.col(\"long\") - BANK_LON) * 111.32 * np.cos(np.radians(BANK_LAT))) ** 2) ** 0.5\n",
" (\n",
" (\n",
" ((pl.col(\"lat\") - BANK_LAT) * 111.32) ** 2\n",
" + ((pl.col(\"long\") - BANK_LON) * 111.32 * np.cos(np.radians(BANK_LAT))) ** 2\n",
" )\n",
" ** 0.5\n",
" ).alias(\"dist_km\")\n",
")\n",
"\n",
"# Bin by 5km\n",
"binned = (\n",
" dist_df\n",
" .with_columns((pl.col(\"dist_km\") / 5).floor() * 5)\n",
" dist_df.with_columns((pl.col(\"dist_km\") / 5).floor() * 5)\n",
" .group_by(\"dist_km\")\n",
" .agg([\n",
" pl.col(\"error_easy\").median().alias(\"median_error_easy\"),\n",
" pl.col(\"error_quick\").median().alias(\"median_error_quick\"),\n",
" pl.col(\"abs_error_easy\").median().alias(\"median_abs_error_easy\"),\n",
" pl.len().alias(\"count\"),\n",
" ])\n",
" .agg(\n",
" [\n",
" pl.col(\"error_easy\").median().alias(\"median_error_easy\"),\n",
" pl.col(\"error_quick\").median().alias(\"median_error_quick\"),\n",
" pl.col(\"abs_error_easy\").median().alias(\"median_abs_error_easy\"),\n",
" pl.len().alias(\"count\"),\n",
" ]\n",
" )\n",
" .sort(\"dist_km\")\n",
" .filter(pl.col(\"count\") > 50)\n",
")\n",
"\n",
"fig = make_subplots(rows=1, cols=2, subplot_titles=[\n",
" \"Median signed error by distance\",\n",
" \"Median absolute error by distance\"\n",
"])\n",
"fig = make_subplots(\n",
" rows=1,\n",
" cols=2,\n",
" subplot_titles=[\n",
" \"Median signed error by distance\",\n",
" \"Median absolute error by distance\",\n",
" ],\n",
")\n",
"\n",
"fig.add_trace(go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(), y=binned[\"median_error_easy\"].to_numpy(),\n",
" mode=\"lines+markers\", name=\"Easy\", line=dict(color=\"#0d9488\")\n",
"), row=1, col=1)\n",
"fig.add_trace(go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(), y=binned[\"median_error_quick\"].to_numpy(),\n",
" mode=\"lines+markers\", name=\"Quick\", line=dict(color=\"#f59e0b\")\n",
"), row=1, col=1)\n",
"fig.add_trace(\n",
" go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(),\n",
" y=binned[\"median_error_easy\"].to_numpy(),\n",
" mode=\"lines+markers\",\n",
" name=\"Easy\",\n",
" line=dict(color=\"#0d9488\"),\n",
" ),\n",
" row=1,\n",
" col=1,\n",
")\n",
"fig.add_trace(\n",
" go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(),\n",
" y=binned[\"median_error_quick\"].to_numpy(),\n",
" mode=\"lines+markers\",\n",
" name=\"Quick\",\n",
" line=dict(color=\"#f59e0b\"),\n",
" ),\n",
" row=1,\n",
" col=1,\n",
")\n",
"\n",
"fig.add_trace(go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(), y=binned[\"median_abs_error_easy\"].to_numpy(),\n",
" mode=\"lines+markers\", name=\"|Easy|\", line=dict(color=\"#0d9488\"),\n",
" showlegend=False\n",
"), row=1, col=2)\n",
"fig.add_trace(\n",
" go.Scatter(\n",
" x=binned[\"dist_km\"].to_numpy(),\n",
" y=binned[\"median_abs_error_easy\"].to_numpy(),\n",
" mode=\"lines+markers\",\n",
" name=\"|Easy|\",\n",
" line=dict(color=\"#0d9488\"),\n",
" showlegend=False,\n",
" ),\n",
" row=1,\n",
" col=2,\n",
")\n",
"\n",
"for col in [1, 2]:\n",
" fig.update_xaxes(title_text=\"Distance from Bank (km)\", row=1, col=col)\n",

18
finder/constants.py
View file

@ -16,9 +16,21 @@ SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
# Whether to run a scrape immediately on startup
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
# Enable/disable individual sources
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes")
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes")
SCRAPE_OPENRENT = os.environ.get("SCRAPE_OPENRENT", "true").lower() in ("1", "true", "yes")
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in (
"1",
"true",
"yes",
)
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in (
"1",
"true",
"yes",
)
SCRAPE_OPENRENT = os.environ.get("SCRAPE_OPENRENT", "true").lower() in (
"1",
"true",
"yes",
)
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"

44
finder/homecouk.py
View file

@ -86,7 +86,8 @@ def solve_cloudflare() -> tuple[dict[str, str], str] | None:
log.info(
"Cloudflare solved — got %d cookies, UA: %s",
len(cookies), user_agent[:60],
len(cookies),
user_agent[:60],
)
flaresolverr_attempts_total.labels(result="success").inc()
return cookies, user_agent
@ -129,11 +130,13 @@ def make_client(cookies: dict[str, str], user_agent: str) -> Session:
Uses Chrome TLS impersonation so cf_clearance cookies (which are bound
to Chrome's JA3 fingerprint from FlareSolverr) remain valid."""
session = Session(impersonate="chrome")
session.headers.update({
"User-Agent": user_agent,
"Accept": "application/json, text/plain, */*",
"x-requested-with": "XMLHttpRequest",
})
session.headers.update(
{
"User-Agent": user_agent,
"Accept": "application/json, text/plain, */*",
"x-requested-with": "XMLHttpRequest",
}
)
# Laravel CSRF: the XSRF-TOKEN cookie value must also be sent as the
# X-XSRF-TOKEN request header (URL-decoded). Without this header, the
# server rejects every request with 419/403.
@ -165,7 +168,11 @@ def fetch_page(
return resp.json()
except json.JSONDecodeError:
homecouk_errors_total.labels(type="json_decode").inc()
log.error("Non-JSON response from %s (got %s)", url, resp.headers.get("content-type", "?"))
log.error(
"Non-JSON response from %s (got %s)",
url,
resp.headers.get("content-type", "?"),
)
return None
if resp.status_code == 403:
raise CookiesExpiredError("HTTP 403 — cookies likely expired")
@ -173,7 +180,11 @@ def fetch_page(
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
log.warning(
"HTTP %d from %s, retry %d/%d in %.1fs",
resp.status_code, url, attempt + 1, max_retries, delay,
resp.status_code,
url,
attempt + 1,
max_retries,
delay,
)
time.sleep(delay)
continue
@ -186,7 +197,11 @@ def fetch_page(
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
log.warning(
"%s from %s, retry %d/%d in %.1fs",
type(e).__name__, url, attempt + 1, max_retries, delay,
type(e).__name__,
url,
attempt + 1,
max_retries,
delay,
)
time.sleep(delay)
homecouk_errors_total.labels(type="retry_exhausted").inc()
@ -218,7 +233,12 @@ def map_property_type(raw_type: str | None) -> str:
# Home.co.uk uses types like "House", "Flat", "Apartment", "Detached", etc.
# Try common patterns
lower = raw_type.lower()
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower:
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
):
return "Flats/Maisonettes"
if "detached" in lower and "semi" not in lower:
return "Detached"
@ -231,7 +251,9 @@ def map_property_type(raw_type: str | None) -> str:
def transform_property(
prop: dict, channel: str, pc_index: PostcodeSpatialIndex,
prop: dict,
channel: str,
pc_index: PostcodeSpatialIndex,
) -> dict | None:
"""Transform a raw home.co.uk property dict into our output schema."""
lat = prop.get("latitude")

48
finder/http_client.py
View file

@ -11,7 +11,9 @@ from metrics import http_errors_total, http_requests_total, ip_rotations_total
log = logging.getLogger("rightmove")
_ua = UserAgent(browsers=["Chrome", "Edge"], os=["Windows", "Mac OS X"], min_version=120.0)
_ua = UserAgent(
browsers=["Chrome", "Edge"], os=["Windows", "Mac OS X"], min_version=120.0
)
def _endpoint_label(url: str) -> str:
@ -27,6 +29,7 @@ def _status_label(code: int) -> str:
return "5xx"
return str(code)
# Gluetun control API — runs on port 8000 inside the gluetun container.
# Since finder uses network_mode: service:gluetun, localhost IS gluetun.
GLUETUN_API = "http://127.0.0.1:8000"
@ -42,17 +45,25 @@ def rotate_ip() -> bool:
# Get current IP
with httpx.Client(timeout=10) as ctl:
old_ip_resp = ctl.get(f"{GLUETUN_API}/v1/publicip/ip")
old_ip = old_ip_resp.json().get("public_ip", "unknown") if old_ip_resp.status_code == 200 else "unknown"
old_ip = (
old_ip_resp.json().get("public_ip", "unknown")
if old_ip_resp.status_code == 200
else "unknown"
)
log.info("Current IP: %s", old_ip)
# Trigger server change — PUT with empty JSON body picks a random server
resp = ctl.put(f"{GLUETUN_API}/v1/vpn/status", json={"status": "stopped"})
resp = ctl.put(
f"{GLUETUN_API}/v1/vpn/status", json={"status": "stopped"}
)
if resp.status_code != 200:
log.error("Failed to stop VPN: %d %s", resp.status_code, resp.text)
return False
time.sleep(2)
resp = ctl.put(f"{GLUETUN_API}/v1/vpn/status", json={"status": "running"})
resp = ctl.put(
f"{GLUETUN_API}/v1/vpn/status", json={"status": "running"}
)
if resp.status_code != 200:
log.error("Failed to start VPN: %d %s", resp.status_code, resp.text)
return False
@ -99,7 +110,9 @@ def fetch_with_retry(
for attempt in range(MAX_RETRIES):
try:
resp = client.get(url, params=params)
http_requests_total.labels(status=_status_label(resp.status_code), endpoint=endpoint).inc()
http_requests_total.labels(
status=_status_label(resp.status_code), endpoint=endpoint
).inc()
if resp.status_code == 200:
return resp.json()
if resp.status_code == 403 and on_403:
@ -111,15 +124,34 @@ def fetch_with_retry(
return None
if resp.status_code in (429, 500, 502, 503, 504):
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
log.warning("HTTP %d from %s, retry %d/%d in %.1fs", resp.status_code, url, attempt + 1, MAX_RETRIES, delay)
log.warning(
"HTTP %d from %s, retry %d/%d in %.1fs",
resp.status_code,
url,
attempt + 1,
MAX_RETRIES,
delay,
)
time.sleep(delay)
continue
log.error("HTTP %d from %s (non-retryable)", resp.status_code, url)
return None
except (httpx.ConnectError, httpx.ReadTimeout, httpx.WriteTimeout, httpx.PoolTimeout) as e:
except (
httpx.ConnectError,
httpx.ReadTimeout,
httpx.WriteTimeout,
httpx.PoolTimeout,
) as e:
http_errors_total.labels(type=type(e).__name__).inc()
delay = RETRY_BASE_DELAY * (2**attempt) + random.uniform(0, 1)
log.warning("%s from %s, retry %d/%d in %.1fs", type(e).__name__, url, attempt + 1, MAX_RETRIES, delay)
log.warning(
"%s from %s, retry %d/%d in %.1fs",
type(e).__name__,
url,
attempt + 1,
MAX_RETRIES,
delay,
)
time.sleep(delay)
http_errors_total.labels(type="retry_exhausted").inc()
log.error("All %d retries exhausted for %s", MAX_RETRIES, url)

46
finder/main.py
View file

@ -7,7 +7,14 @@ from pathlib import Path
from flask import Flask, Response, jsonify, send_from_directory
from prometheus_client import generate_latest, CONTENT_TYPE_LATEST
from constants import DATA_DIR, RUN_ON_STARTUP, SCHEDULE_HOUR, SCRAPE_HOMECOUK, SCRAPE_OPENRENT, SCRAPE_RIGHTMOVE
from constants import (
DATA_DIR,
RUN_ON_STARTUP,
SCHEDULE_HOUR,
SCRAPE_HOMECOUK,
SCRAPE_OPENRENT,
SCRAPE_RIGHTMOVE,
)
from homecouk import load_cookies as load_homecouk_cookies
from openrent import load_cookies as load_openrent_cookies
from rightmove import outcode_cache
@ -49,8 +56,13 @@ log.info("Loading arcgis data...")
OUTCODES = load_outcodes()
PC_INDEX = build_postcode_index()
PC_COORDS = build_postcode_coords() if SCRAPE_OPENRENT else None
log.info("Ready — %d outcodes, postcode index built (rightmove=%s, homecouk=%s, openrent=%s)",
len(OUTCODES), SCRAPE_RIGHTMOVE, SCRAPE_HOMECOUK, SCRAPE_OPENRENT)
log.info(
"Ready — %d outcodes, postcode index built (rightmove=%s, homecouk=%s, openrent=%s)",
len(OUTCODES),
SCRAPE_RIGHTMOVE,
SCRAPE_HOMECOUK,
SCRAPE_OPENRENT,
)
# ---------------------------------------------------------------------------
# Scheduler
@ -63,7 +75,9 @@ def _start_scrape() -> bool:
if status.state == "running":
return False
status.state = "running"
thread = threading.Thread(target=run_scrape, args=(OUTCODES, PC_INDEX, PC_COORDS), daemon=True)
thread = threading.Thread(
target=run_scrape, args=(OUTCODES, PC_INDEX, PC_COORDS), daemon=True
)
thread.start()
return True
@ -82,7 +96,9 @@ def _scheduler_loop() -> None:
log.info("Scheduler active — will run daily at %02d:00 UTC", SCHEDULE_HOUR)
while True:
wait = _seconds_until(SCHEDULE_HOUR)
log.info("Next scheduled scrape in %.0f seconds (%.1f hours)", wait, wait / 3600)
log.info(
"Next scheduled scrape in %.0f seconds (%.1f hours)", wait, wait / 3600
)
time.sleep(wait)
log.info("Scheduled scrape triggered")
if not _start_scrape():
@ -144,15 +160,17 @@ def get_status():
def get_debug():
hk_cookies = load_homecouk_cookies() if SCRAPE_HOMECOUK else None
or_cookies = load_openrent_cookies() if SCRAPE_OPENRENT else None
return jsonify({
"outcode_cache_size": len(outcode_cache),
"outcode_cache_sample": dict(list(outcode_cache.items())[:20]),
"scrape_rightmove": SCRAPE_RIGHTMOVE,
"scrape_homecouk": SCRAPE_HOMECOUK,
"scrape_openrent": SCRAPE_OPENRENT,
"homecouk_cookies_available": hk_cookies is not None,
"openrent_cookies_available": or_cookies is not None,
})
return jsonify(
{
"outcode_cache_size": len(outcode_cache),
"outcode_cache_sample": dict(list(outcode_cache.items())[:20]),
"scrape_rightmove": SCRAPE_RIGHTMOVE,
"scrape_homecouk": SCRAPE_HOMECOUK,
"scrape_openrent": SCRAPE_OPENRENT,
"homecouk_cookies_available": hk_cookies is not None,
"openrent_cookies_available": or_cookies is not None,
}
)
@app.route("/metrics")

78
finder/openrent.py
View file

@ -79,7 +79,8 @@ def solve_waf() -> tuple[dict[str, str], str] | None:
if "AwsWafIntegration" in content:
log.info("Got WAF challenge page, waiting for resolution...")
page.wait_for_selector(
"a.pli, .pli, .search-property-card", timeout=30000,
"a.pli, .pli, .search-property-card",
timeout=30000,
)
raw_cookies = context.cookies()
@ -94,7 +95,8 @@ def solve_waf() -> tuple[dict[str, str], str] | None:
log.info(
"AWS WAF solved — got %d cookies, UA: %s",
len(cookies), user_agent[:60],
len(cookies),
user_agent[:60],
)
flaresolverr_attempts_total.labels(result="success").inc()
return cookies, user_agent
@ -130,11 +132,13 @@ def make_client(cookies: dict[str, str], user_agent: str) -> Session:
"""Create a curl_cffi Session configured for OpenRent.
Uses Chrome TLS impersonation so AWS WAF cookies remain valid."""
session = Session(impersonate="chrome")
session.headers.update({
"User-Agent": user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-GB,en;q=0.9",
})
session.headers.update(
{
"User-Agent": user_agent,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8",
"Accept-Language": "en-GB,en;q=0.9",
}
)
for name, value in cookies.items():
session.cookies.set(name, value, domain="openrent.co.uk")
return session
@ -152,7 +156,9 @@ def _status_label(code: int) -> str:
def fetch_page(
client: Session, url: str, max_retries: int = 3,
client: Session,
url: str,
max_retries: int = 3,
) -> str | None:
"""GET HTML with retries on 429/5xx. Returns None on permanent failure.
WAF challenge (202 or 403 with challenge JS) raises WafChallengeError."""
@ -165,17 +171,25 @@ def fetch_page(
html = resp.text
# Detect WAF challenge page masquerading as 200
if "AwsWafIntegration" in html and "challenge.js" in html:
raise WafChallengeError("Got AWS WAF challenge page — cookies expired")
raise WafChallengeError(
"Got AWS WAF challenge page — cookies expired"
)
return html
if resp.status_code in (202, 403):
raise WafChallengeError(f"HTTP {resp.status_code} — cookies likely expired")
raise WafChallengeError(
f"HTTP {resp.status_code} — cookies likely expired"
)
if resp.status_code in (429, 500, 502, 503, 504):
delay = RETRY_BASE_DELAY * (2 ** attempt)
delay = RETRY_BASE_DELAY * (2**attempt)
log.warning(
"HTTP %d from %s, retry %d/%d in %.1fs",
resp.status_code, url, attempt + 1, max_retries, delay,
resp.status_code,
url,
attempt + 1,
max_retries,
delay,
)
time.sleep(delay)
continue
@ -187,10 +201,14 @@ def fetch_page(
raise
except RequestsError as e:
openrent_errors_total.labels(type=type(e).__name__).inc()
delay = RETRY_BASE_DELAY * (2 ** attempt)
delay = RETRY_BASE_DELAY * (2**attempt)
log.warning(
"%s from %s, retry %d/%d in %.1fs",
type(e).__name__, url, attempt + 1, max_retries, delay,
type(e).__name__,
url,
attempt + 1,
max_retries,
delay,
)
time.sleep(delay)
@ -247,7 +265,9 @@ def _extract_bedrooms_from_title(title: str) -> int | None:
return None
def _extract_beds_baths_from_features(feature_items: list) -> tuple[int | None, int | None]:
def _extract_beds_baths_from_features(
feature_items: list,
) -> tuple[int | None, int | None]:
"""Extract bedrooms and bathrooms from feature list items.
OpenRent search cards have <ul> with items like:
@ -442,11 +462,7 @@ def parse_search_results(html: str) -> list[dict]:
# --- Coordinates from data attributes (may not be present on cards) ---
for el in [card] + card.select("[data-lat], [data-latitude]"):
lat = el.get("data-lat") or el.get("data-latitude")
lng = (
el.get("data-lng")
or el.get("data-longitude")
or el.get("data-lon")
)
lng = el.get("data-lng") or el.get("data-longitude") or el.get("data-lon")
if lat and lng:
try:
prop["lat"] = float(lat)
@ -543,9 +559,7 @@ def parse_property_detail(html: str) -> dict:
break
# --- Description for floor area ---
desc_el = soup.select_one(
".description, [class*='description'], #description"
)
desc_el = soup.select_one(".description, [class*='description'], #description")
if desc_el:
details["description"] = desc_el.get_text(strip=True)
@ -567,7 +581,12 @@ def map_property_type(raw_type: str | None) -> str:
lower = raw_type.lower()
if "room" in lower or "shared" in lower:
return "Other"
if "flat" in lower or "apartment" in lower or "maisonette" in lower or "studio" in lower:
if (
"flat" in lower
or "apartment" in lower
or "maisonette" in lower
or "studio" in lower
):
return "Flats/Maisonettes"
if "detached" in lower and "semi" not in lower:
return "Detached"
@ -647,7 +666,8 @@ def transform_property(
elif search_data.get("outcode"):
# No spatial index — try outcode lookup as fallback
outcode_pcs = _resolve_outcode_postcodes(
search_data["outcode"], pc_coords,
search_data["outcode"],
pc_coords,
)
if outcode_pcs:
postcode = outcode_pcs[0]
@ -708,7 +728,8 @@ def transform_property(
prop_id = search_data.get("id", "")
listing_url = search_data.get(
"url", f"{OPENRENT_BASE}/{prop_id}" if prop_id else "",
"url",
f"{OPENRENT_BASE}/{prop_id}" if prop_id else "",
)
description = detail.get("description") or search_data.get("description", "")
@ -775,7 +796,10 @@ def search_outcode(
time.sleep(DELAY_BETWEEN_PAGES * 0.5)
transformed = transform_property(
search_data, detail_data, pc_index, pc_coords,
search_data,
detail_data,
pc_index,
pc_coords,
)
if transformed:
properties.append(transformed)

11
finder/rightmove.py
View file

@ -24,7 +24,9 @@ def resolve_outcode_id(client: httpx.Client, outcode: str) -> str | None:
if outcode in outcode_cache:
return outcode_cache[outcode]
data = fetch_with_retry(client, TYPEAHEAD_URL, {"query": outcode, "limit": "10", "exclude": "STREET"})
data = fetch_with_retry(
client, TYPEAHEAD_URL, {"query": outcode, "limit": "10", "exclude": "STREET"}
)
if not data:
return None
@ -61,7 +63,12 @@ def search_outcode(
data = fetch_with_retry(client, SEARCH_URL, params)
if not data:
log.warning("Failed to fetch index %d for %s/%s", index, outcode, channel_cfg["channel"])
log.warning(
"Failed to fetch index %d for %s/%s",
index,
outcode,
channel_cfg["channel"],
)
break
raw_props = data.get("properties", [])

142
finder/scraper.py
View file

@ -6,7 +6,16 @@ from dataclasses import dataclass, field
import polars as pl
from constants import ARCGIS_PATH, CHANNELS, DATA_DIR, DELAY_BETWEEN_OUTCODES, SCRAPE_HOMECOUK, SCRAPE_OPENRENT, SCRAPE_RIGHTMOVE, SEED
from constants import (
ARCGIS_PATH,
CHANNELS,
DATA_DIR,
DELAY_BETWEEN_OUTCODES,
SCRAPE_HOMECOUK,
SCRAPE_OPENRENT,
SCRAPE_RIGHTMOVE,
SEED,
)
from homecouk import CookiesExpiredError
from homecouk import load_cookies as load_homecouk_cookies
from homecouk import make_client as make_homecouk_client
@ -64,13 +73,23 @@ def _sync_gauges() -> None:
scrape_outcodes_done.set(status.outcodes_done)
scrape_outcodes_total.set(status.outcodes_total)
# Total properties (both sources combined)
scrape_properties_total.labels(channel="buy", source="total").set(status.properties_buy)
scrape_properties_total.labels(channel="rent", source="total").set(status.properties_rent)
scrape_properties_total.labels(channel="buy", source="total").set(
status.properties_buy
)
scrape_properties_total.labels(channel="rent", source="total").set(
status.properties_rent
)
# Per-source breakdown for current channel
ch = "buy" if status.channel == "BUY" else "rent"
scrape_properties_total.labels(channel=ch, source="rightmove").set(status.rm_properties)
scrape_properties_total.labels(channel=ch, source="homecouk").set(status.hk_properties)
scrape_properties_total.labels(channel=ch, source="openrent").set(status.or_properties)
scrape_properties_total.labels(channel=ch, source="rightmove").set(
status.rm_properties
)
scrape_properties_total.labels(channel=ch, source="homecouk").set(
status.hk_properties
)
scrape_properties_total.labels(channel=ch, source="openrent").set(
status.or_properties
)
if status.started_at:
end = status.finished_at if status.finished_at else time.time()
scrape_elapsed_seconds.set(end - status.started_at)
@ -86,7 +105,9 @@ def load_outcodes() -> list[str]:
log.info("England postcodes: %d", len(england))
outcodes = (
england.select(pl.col("pcd").str.extract(r"^([A-Z]{1,2}\d[A-Z0-9]?)", 1).alias("outcode"))
england.select(
pl.col("pcd").str.extract(r"^([A-Z]{1,2}\d[A-Z0-9]?)", 1).alias("outcode")
)
.drop_nulls()
.get_column("outcode")
.unique()
@ -101,7 +122,9 @@ def build_postcode_index() -> PostcodeSpatialIndex:
"""Build spatial index from arcgis England postcodes."""
log.info("Building postcode spatial index from %s", ARCGIS_PATH)
df = pl.read_parquet(ARCGIS_PATH, columns=["pcd", "ctry", "lat", "long"])
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(subset=["lat", "long"])
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(
subset=["lat", "long"]
)
return PostcodeSpatialIndex(
england.get_column("lat").to_list(),
england.get_column("long").to_list(),
@ -114,7 +137,9 @@ def build_postcode_coords() -> dict[str, tuple[float, float]]:
Used by OpenRent scraper to resolve coordinates from postcodes."""
log.info("Building postcode coords lookup from %s", ARCGIS_PATH)
df = pl.read_parquet(ARCGIS_PATH, columns=["pcd", "ctry", "lat", "long"])
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(subset=["lat", "long"])
england = df.filter(pl.col("ctry") == "E92000001").drop_nulls(
subset=["lat", "long"]
)
coords: dict[str, tuple[float, float]] = {}
for pcd, lat, lng in zip(
england.get_column("pcd").to_list(),
@ -179,7 +204,9 @@ def run_scrape(
log.info("home.co.uk scraping ENABLED")
homecouk_enabled.set(1)
else:
log.info("home.co.uk scraping DISABLED (need FlareSolverr or HOMECOUK_CF_CLEARANCE + HOMECOUK_SESSION)")
log.info(
"home.co.uk scraping DISABLED (need FlareSolverr or HOMECOUK_CF_CLEARANCE + HOMECOUK_SESSION)"
)
homecouk_enabled.set(0)
# OpenRent: must be enabled via SCRAPE_OPENRENT + cookies available
@ -195,7 +222,9 @@ def run_scrape(
log.info("OpenRent scraping ENABLED")
openrent_enabled.set(1)
else:
log.info("OpenRent scraping DISABLED (need FlareSolverr or OPENRENT_WAF_TOKEN)")
log.info(
"OpenRent scraping DISABLED (need FlareSolverr or OPENRENT_WAF_TOKEN)"
)
openrent_enabled.set(0)
# Build postcode coords if OpenRent is active and caller didn't provide them
@ -207,7 +236,9 @@ def run_scrape(
channel_name = channel_cfg["channel"]
file_suffix = "buy" if channel_name == "BUY" else "rent"
all_properties: dict[str, dict] = {} # dedup by id
seen_dedup_keys: set[tuple] = set() # cross-source dedup by (postcode, beds, price)
seen_dedup_keys: set[tuple] = (
set()
) # cross-source dedup by (postcode, beds, price)
rm_count = 0 # Rightmove properties this channel
hk_count = 0 # home.co.uk properties this channel
hk_dedup_count = 0 # home.co.uk skipped as cross-source duplicates
@ -222,15 +253,22 @@ def run_scrape(
status.hk_properties = 0
status.or_properties = 0
log.info("=== Starting %s channel (%d outcodes) ===", channel_name, len(shuffled))
log.info(
"=== Starting %s channel (%d outcodes) ===", channel_name, len(shuffled)
)
for i, outcode in enumerate(shuffled):
with status_lock:
status.outcode = outcode
status.outcodes_done = i
log.debug("Outcode %s (%d/%d) — %d properties so far",
outcode, i + 1, len(shuffled), len(all_properties))
log.debug(
"Outcode %s (%d/%d) — %d properties so far",
outcode,
i + 1,
len(shuffled),
len(all_properties),
)
made_requests = False
@ -240,9 +278,13 @@ def run_scrape(
try:
outcode_id = resolve_outcode_id(client, outcode)
if not outcode_id:
log.debug("No Rightmove ID for outcode %s, skipping", outcode)
log.debug(
"No Rightmove ID for outcode %s, skipping", outcode
)
else:
props = search_outcode(client, outcode_id, outcode, channel_cfg, pc_index)
props = search_outcode(
client, outcode_id, outcode, channel_cfg, pc_index
)
for p in props:
pid = p["id"]
if pid not in all_properties:
@ -261,7 +303,10 @@ def run_scrape(
made_requests = True
try:
hk_props = homecouk_search_outcode(
hk_client, outcode, channel_name, pc_index,
hk_client,
outcode,
channel_name,
pc_index,
)
for p in hk_props:
pid = p["id"]
@ -276,9 +321,13 @@ def run_scrape(
seen_dedup_keys.add(key)
hk_count += 1
if hk_props:
log.info("home.co.uk %s: +%d properties", outcode, len(hk_props))
log.info(
"home.co.uk %s: +%d properties", outcode, len(hk_props)
)
except CookiesExpiredError:
log.warning("home.co.uk cookies expired — attempting refresh via FlareSolverr")
log.warning(
"home.co.uk cookies expired — attempting refresh via FlareSolverr"
)
hk_client.close()
hk_result = load_homecouk_cookies()
if hk_result:
@ -286,13 +335,17 @@ def run_scrape(
log.info("home.co.uk cookies refreshed, continuing")
cookie_refreshes_total.labels(result="success").inc()
else:
log.warning("Cookie refresh failed, disabling home.co.uk for rest of scrape")
log.warning(
"Cookie refresh failed, disabling home.co.uk for rest of scrape"
)
hk_client = None
hk_failed = True
homecouk_enabled.set(0)
cookie_refreshes_total.labels(result="failure").inc()
with status_lock:
status.errors.append("home.co.uk cookies expired and refresh failed")
status.errors.append(
"home.co.uk cookies expired and refresh failed"
)
except Exception as e:
msg = f"Error scraping home.co.uk {outcode}/{channel_name}: {e}"
log.error(msg)
@ -305,7 +358,10 @@ def run_scrape(
made_requests = True
try:
or_props = openrent_search_outcode(
or_client, outcode, pc_index, pc_coords,
or_client,
outcode,
pc_index,
pc_coords,
)
for p in or_props:
pid = p["id"]
@ -318,9 +374,13 @@ def run_scrape(
seen_dedup_keys.add(key)
or_count += 1
if or_props:
log.info("OpenRent %s: +%d properties", outcode, len(or_props))
log.info(
"OpenRent %s: +%d properties", outcode, len(or_props)
)
except WafChallengeError:
log.warning("OpenRent WAF cookies expired — attempting refresh via FlareSolverr")
log.warning(
"OpenRent WAF cookies expired — attempting refresh via FlareSolverr"
)
or_client.close()
or_result = load_openrent_cookies()
if or_result:
@ -328,13 +388,17 @@ def run_scrape(
log.info("OpenRent cookies refreshed, continuing")
cookie_refreshes_total.labels(result="success").inc()
else:
log.warning("Cookie refresh failed, disabling OpenRent for rest of scrape")
log.warning(
"Cookie refresh failed, disabling OpenRent for rest of scrape"
)
or_client = None
or_failed = True
openrent_enabled.set(0)
cookie_refreshes_total.labels(result="failure").inc()
with status_lock:
status.errors.append("OpenRent WAF cookies expired and refresh failed")
status.errors.append(
"OpenRent WAF cookies expired and refresh failed"
)
except Exception as e:
msg = f"Error scraping OpenRent {outcode}/{channel_name}: {e}"
log.error(msg)
@ -352,8 +416,14 @@ def run_scrape(
status.or_properties = or_count
_sync_gauges()
log.info("Outcode %s: total %d (rm: %d, hk: %d, or: %d)",
outcode, len(all_properties), rm_count, hk_count, or_count)
log.info(
"Outcode %s: total %d (rm: %d, hk: %d, or: %d)",
outcode,
len(all_properties),
rm_count,
hk_count,
or_count,
)
if made_requests and i < len(shuffled) - 1:
time.sleep(DELAY_BETWEEN_OUTCODES)
@ -373,7 +443,11 @@ def run_scrape(
log.info(
"=== %s channel complete: %d unique (rm: %d, hk: %d, or: %d, cross-dedup: %d) ===",
channel_name, len(deduped), rm_count, hk_count, or_count,
channel_name,
len(deduped),
rm_count,
hk_count,
or_count,
hk_dedup_count + or_dedup_count,
)
@ -382,8 +456,12 @@ def run_scrape(
status.finished_at = time.time()
_sync_gauges()
elapsed = status.finished_at - status.started_at
log.info("Scrape complete in %.0fs — buy: %d, rent: %d",
elapsed, status.properties_buy, status.properties_rent)
log.info(
"Scrape complete in %.0fs — buy: %d, rent: %d",
elapsed,
status.properties_buy,
status.properties_rent,
)
except Exception as e:
log.exception("Fatal scrape error")

8
finder/spatial.py
View file

@ -11,12 +11,16 @@ class PostcodeSpatialIndex:
"""Grid-based spatial index over arcgis postcodes for nearest-lookup."""
def __init__(self, lats: list[float], lngs: list[float], postcodes: list[str]):
self.grid: dict[tuple[int, int], list[tuple[float, float, str]]] = defaultdict(list)
self.grid: dict[tuple[int, int], list[tuple[float, float, str]]] = defaultdict(
list
)
for lat, lng, pcd in zip(lats, lngs, postcodes):
gx = int(math.floor(lng / GRID_CELL_SIZE))
gy = int(math.floor(lat / GRID_CELL_SIZE))
self.grid[(gx, gy)].append((lat, lng, pcd))
log.info("Postcode spatial index: %d cells, %d postcodes", len(self.grid), len(lats))
log.info(
"Postcode spatial index: %d cells, %d postcodes", len(self.grid), len(lats)
)
def nearest(self, lat: float, lng: float) -> str | None:
gx = int(math.floor(lng / GRID_CELL_SIZE))

4
finder/storage.py
View file

@ -60,9 +60,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
"Property type": [p["Property type"] for p in properties],
"Property sub-type": [p["Property sub-type"] for p in properties],
"Price qualifier": [p["Price qualifier"] for p in properties],
"Total floor area (sqm)": [
p["Total floor area (sqm)"] for p in properties
],
"Total floor area (sqm)": [p["Total floor area (sqm)"] for p in properties],
"Listing URL": [p["Listing URL"] for p in properties],
"Listing features": [p["Listing features"] for p in properties],
"Listing date": listing_dates,

28
finder/transform.py
View file

@ -51,9 +51,19 @@ def fix_coords(lat: float, lng: float) -> tuple[float, float]:
if 49 <= lat <= 56 and -7 <= lng <= 2:
return lat, lng
if 49 <= lng <= 56 and -7 <= lat <= 2:
log.debug("Swapping reversed coords: lat=%.4f lng=%.4f → lat=%.4f lng=%.4f", lat, lng, lng, lat)
log.debug(
"Swapping reversed coords: lat=%.4f lng=%.4f → lat=%.4f lng=%.4f",
lat,
lng,
lng,
lat,
)
return lng, lat
log.warning("Coords outside England bounds even after swap attempt: lat=%.4f lng=%.4f", lat, lng)
log.warning(
"Coords outside England bounds even after swap attempt: lat=%.4f lng=%.4f",
lat,
lng,
)
return lat, lng
@ -66,7 +76,9 @@ def normalize_price(amount: int, frequency: str) -> int:
return amount
def transform_property(prop: dict, outcode: str, pc_index: PostcodeSpatialIndex) -> dict | None:
def transform_property(
prop: dict, outcode: str, pc_index: PostcodeSpatialIndex
) -> dict | None:
"""Transform a raw Rightmove property dict into our output schema."""
loc = prop.get("location")
if not loc:
@ -86,13 +98,19 @@ def transform_property(prop: dict, outcode: str, pc_index: PostcodeSpatialIndex)
price = normalize_price(int(amount), frequency)
display_prices = price_obj.get("displayPrices", [])
price_qualifier = display_prices[0].get("displayPriceQualifier", "") if display_prices else ""
price_qualifier = (
display_prices[0].get("displayPriceQualifier", "") if display_prices else ""
)
sub_type = prop.get("propertySubType", "")
bedrooms = prop.get("bedrooms", 0) or 0
bathrooms = prop.get("bathrooms", 0) or 0
key_features = [kf.get("description", "") for kf in prop.get("keyFeatures", []) if kf.get("description")]
key_features = [
kf.get("description", "")
for kf in prop.get("keyFeatures", [])
if kf.get("description")
]
postcode = pc_index.nearest(lat, lng)
if not postcode:

8
pipeline/download/lsoa_population.py
View file

@ -40,7 +40,9 @@ def download_and_convert(output_path: Path) -> None:
df = pl.concat(frames)
print(f"Total rows: {df.height}")
result = df.rename({"GEOGRAPHY_CODE": "lsoa21", "OBS_VALUE": "population"}).with_columns(
result = df.rename(
{"GEOGRAPHY_CODE": "lsoa21", "OBS_VALUE": "population"}
).with_columns(
pl.col("population").cast(pl.UInt32),
)
@ -48,7 +50,9 @@ def download_and_convert(output_path: Path) -> None:
result = result.filter(pl.col("lsoa21").str.starts_with("E"))
print(f"England LSOAs: {result.height}")
print(f"Population range: {result['population'].min()} - {result['population'].max()}")
print(
f"Population range: {result['population'].min()} - {result['population'].max()}"
)
print(f"Mean population: {result['population'].mean():.0f}")
output_path.parent.mkdir(parents=True, exist_ok=True)

10
pipeline/download/places.py
View file

@ -119,7 +119,11 @@ class PlaceHandler(osmium.SimpleHandler):
station_tag = tags.get("station", "")
network = tags.get("network", "").lower()
# Skip tram stops
if station_tag == "light_rail" or "tramlink" in network or "tram" in network:
if (
station_tag == "light_rail"
or "tramlink" in network
or "tram" in network
):
return
display_name = _station_display_name(name, tags)
self._add(display_name, "station", lat, lon, population)
@ -131,9 +135,7 @@ def main() -> None:
parser.add_argument(
"--output", type=Path, required=True, help="Output parquet file path"
)
parser.add_argument(
"--pbf", type=Path, required=True, help="Path to OSM PBF file"
)
parser.add_argument("--pbf", type=Path, required=True, help="Path to OSM PBF file")
parser.add_argument(
"--boundary",
type=Path,

4
pipeline/download/pois.py
View file

@ -111,9 +111,7 @@ def main() -> None:
parser.add_argument(
"--output", type=Path, required=True, help="Output parquet file path"
)
parser.add_argument(
"--pbf", type=Path, required=True, help="Path to OSM PBF file"
)
parser.add_argument("--pbf", type=Path, required=True, help="Path to OSM PBF file")
parser.add_argument(
"--boundary",
type=Path,

12
pipeline/download/rental_prices.py
View file

@ -99,10 +99,14 @@ def convert_to_parquet(xls_path: Path, parquet_path: Path) -> None:
combined = pl.concat(frames)
# Remap old LA codes to new unitary authority codes and average medians
combined = combined.with_columns(
pl.col("area_code").replace(LA_CONSOLIDATION),
).group_by("area_code", "bedrooms").agg(
pl.col("median_monthly_rent").mean(),
combined = (
combined.with_columns(
pl.col("area_code").replace(LA_CONSOLIDATION),
)
.group_by("area_code", "bedrooms")
.agg(
pl.col("median_monthly_rent").mean(),
)
)
print(f"Combined: {combined.shape}")

16
pipeline/download/rightmove_outcodes.py
View file

@ -13,9 +13,7 @@ TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
def fetch_outcode_ids(postcodes_path: Path, output: Path) -> None:
df = pl.read_parquet(postcodes_path, columns=["Postcode"])
outcodes = sorted(
set(df["Postcode"].str.split(" ").list.first().to_list()) - {""}
)
outcodes = sorted(set(df["Postcode"].str.split(" ").list.first().to_list()) - {""})
print(f"Querying Rightmove typeahead for {len(outcodes)} outcodes...")
mapping: dict[str, str] = {}
@ -28,11 +26,9 @@ def fetch_outcode_ids(postcodes_path: Path, output: Path) -> None:
data = resp.json()
found = False
for m in data.get("matches", []):
if (
m["type"] == "OUTCODE"
and m["displayName"].upper().replace(" ", "")
== oc.upper().replace(" ", "")
):
if m["type"] == "OUTCODE" and m["displayName"].upper().replace(
" ", ""
) == oc.upper().replace(" ", ""):
mapping[oc] = str(m["id"])
found = True
break
@ -57,9 +53,7 @@ def fetch_outcode_ids(postcodes_path: Path, output: Path) -> None:
def main() -> None:
parser = argparse.ArgumentParser(
description="Fetch Rightmove outcode ID mapping"
)
parser = argparse.ArgumentParser(description="Fetch Rightmove outcode ID mapping")
parser.add_argument(
"--postcodes", type=Path, required=True, help="postcode.parquet path"
)

4
pipeline/download/tiles.py
View file

@ -64,7 +64,9 @@ def ensure_pmtiles_cli(bin_path: Path, version: str) -> None:
def main():
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--output", type=Path, required=True, help="Output .pmtiles path")
parser.add_argument(
"--output", type=Path, required=True, help="Output .pmtiles path"
)
parser.add_argument(
"--pmtiles-version", default="1.22.3", help="go-pmtiles release version"
)

114
pipeline/download/transit_network.py
View file

@ -56,7 +56,9 @@ NR_TIMETABLE_URL = "https://opendata.nationalrail.co.uk/api/staticfeeds/3.0/time
USER_AGENT = "property-map-pipeline/1.0 (https://github.com)"
def _download_http(url: str, dest: Path, *, desc: str, headers: dict | None = None) -> None:
def _download_http(
url: str, dest: Path, *, desc: str, headers: dict | None = None
) -> None:
"""Stream-download a URL to a file with progress bar."""
dest.parent.mkdir(parents=True, exist_ok=True)
tmp = dest.with_suffix(dest.suffix + ".tmp")
@ -117,9 +119,10 @@ def clean_gtfs(src: Path, dst: Path) -> None:
return
print("Cleaning GTFS for R5 compatibility...")
with zipfile.ZipFile(src, "r") as zin, zipfile.ZipFile(
dst, "w", zipfile.ZIP_DEFLATED
) as zout:
with (
zipfile.ZipFile(src, "r") as zin,
zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zout,
):
for info in zin.infolist():
if info.filename == "stop_times.txt":
dropped = 0
@ -127,7 +130,9 @@ def clean_gtfs(src: Path, dst: Path) -> None:
header = f.readline()
header_str = header.decode("utf-8").strip()
cols = header_str.split(",")
arr_idx = cols.index("arrival_time") if "arrival_time" in cols else -1
arr_idx = (
cols.index("arrival_time") if "arrival_time" in cols else -1
)
dep_idx = (
cols.index("departure_time") if "departure_time" in cols else -1
)
@ -179,7 +184,9 @@ def clean_gtfs(src: Path, dst: Path) -> None:
year = int(date_val[:4])
if year > 2100:
parts[i] = "20991231"
print(f" feed_info: capped end_date {date_val} → 20991231")
print(
f" feed_info: capped end_date {date_val} → 20991231"
)
fixed_lines.append(",".join(parts))
zout.writestr("feed_info.txt", "\n".join(fixed_lines) + "\n")
else:
@ -334,7 +341,9 @@ def convert_high_freq_to_frequency_based(
end_secs = trips[-1][1] + int(median_hw)
headway_rounded = max(60, round(median_hw / 60) * 60)
frequency_entries.append((template_trip_id, start_secs, end_secs, headway_rounded))
frequency_entries.append(
(template_trip_id, start_secs, end_secs, headway_rounded)
)
for trip_id, _ in trips[1:]:
trips_to_remove.add(trip_id)
groups_converted += 1
@ -344,9 +353,10 @@ def convert_high_freq_to_frequency_based(
print(f" Created {len(frequency_entries)} frequency entries")
# Step 5: Write modified GTFS
with zipfile.ZipFile(src, "r") as zin, zipfile.ZipFile(
dst, "w", zipfile.ZIP_DEFLATED
) as zout:
with (
zipfile.ZipFile(src, "r") as zin,
zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zout,
):
for info in zin.infolist():
if info.filename == "trips.txt":
with zin.open(info) as f:
@ -466,15 +476,22 @@ def download_national_rail_cif(raw_dir: Path) -> Path | None:
email = os.environ.get("NATIONAL_RAIL_EMAIL")
password = os.environ.get("NATIONAL_RAIL_PASSWORD")
if not email or not password:
print("Warning: NATIONAL_RAIL_EMAIL/NATIONAL_RAIL_PASSWORD not set, skipping national rail")
print(
"Warning: NATIONAL_RAIL_EMAIL/NATIONAL_RAIL_PASSWORD not set, skipping national rail"
)
return None
print("Authenticating with National Rail Open Data...")
auth_data = urllib.parse.urlencode({"username": email, "password": password}).encode()
auth_data = urllib.parse.urlencode(
{"username": email, "password": password}
).encode()
auth_req = urllib.request.Request(
NR_AUTH_URL,
data=auth_data,
headers={"User-Agent": USER_AGENT, "Content-Type": "application/x-www-form-urlencoded"},
headers={
"User-Agent": USER_AGENT,
"Content-Type": "application/x-www-form-urlencoded",
},
)
with urllib.request.urlopen(auth_req) as resp:
token_data = json.loads(resp.read())
@ -565,9 +582,10 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
coords_fixed = 0
route_types_fixed = 0
with zipfile.ZipFile(src, "r") as zin, zipfile.ZipFile(
dst, "w", zipfile.ZIP_DEFLATED
) as zout:
with (
zipfile.ZipFile(src, "r") as zin,
zipfile.ZipFile(dst, "w", zipfile.ZIP_DEFLATED) as zout,
):
for info in zin.infolist():
# Skip non-standard links.txt
if info.filename == "links.txt":
@ -581,8 +599,12 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
trip_id_idx = cols.index("trip_id")
stop_id_idx = cols.index("stop_id")
seq_idx = cols.index("stop_sequence")
pickup_idx = cols.index("pickup_type") if "pickup_type" in cols else -1
dropoff_idx = cols.index("drop_off_type") if "drop_off_type" in cols else -1
pickup_idx = (
cols.index("pickup_type") if "pickup_type" in cols else -1
)
dropoff_idx = (
cols.index("drop_off_type") if "drop_off_type" in cols else -1
)
tmp = tempfile.NamedTemporaryFile(
mode="wb", delete=False, suffix=".txt"
@ -769,16 +791,27 @@ def _docker_run_dtd2mysql(
) -> None:
"""Run dtd2mysql in a Node.js container on the same Docker network as MariaDB."""
cmd = [
"docker", "run", "--rm", "--network", network,
"-e", f"DATABASE_HOSTNAME={db_container}",
"-e", "DATABASE_USERNAME=root",
"-e", "DATABASE_PASSWORD=root",
"-e", "DATABASE_NAME=dtd",
"docker",
"run",
"--rm",
"--network",
network,
"-e",
f"DATABASE_HOSTNAME={db_container}",
"-e",
"DATABASE_USERNAME=root",
"-e",
"DATABASE_PASSWORD=root",
"-e",
"DATABASE_NAME=dtd",
]
for v in volumes:
cmd.extend(["-v", v])
# Install zip (needed for --gtfs-zip) then run dtd2mysql
inner = "apt-get update -qq && apt-get install -y -qq zip > /dev/null 2>&1 && npx --yes dtd2mysql " + " ".join(args)
inner = (
"apt-get update -qq && apt-get install -y -qq zip > /dev/null 2>&1 && npx --yes dtd2mysql "
+ " ".join(args)
)
cmd.extend(["node:20", "bash", "-c", inner])
subprocess.run(cmd, check=True)
@ -805,11 +838,17 @@ def convert_national_rail_to_gtfs(raw_dir: Path, output_dir: Path) -> Path:
subprocess.run(["docker", "network", "create", network], capture_output=True)
subprocess.run(
[
"docker", "run", "-d",
"--name", db_container,
"--network", network,
"-e", "MARIADB_ROOT_PASSWORD=root",
"-e", "MARIADB_DATABASE=dtd",
"docker",
"run",
"-d",
"--name",
db_container,
"--network",
network,
"-e",
"MARIADB_ROOT_PASSWORD=root",
"-e",
"MARIADB_DATABASE=dtd",
"mariadb:latest",
],
check=True,
@ -820,7 +859,16 @@ def convert_national_rail_to_gtfs(raw_dir: Path, output_dir: Path) -> Path:
print(" Waiting for MariaDB to be ready...")
for attempt in range(30):
result = subprocess.run(
["docker", "exec", db_container, "mariadb", "-uroot", "-proot", "-e", "SELECT 1"],
[
"docker",
"exec",
db_container,
"mariadb",
"-uroot",
"-proot",
"-e",
"SELECT 1",
],
capture_output=True,
)
if result.returncode == 0:
@ -833,14 +881,16 @@ def convert_national_rail_to_gtfs(raw_dir: Path, output_dir: Path) -> Path:
print("Importing CIF timetable into MariaDB...")
_docker_run_dtd2mysql(
network, db_container,
network,
db_container,
volumes=[f"{raw_abs}:/data:ro"],
args=["--timetable", "/data/national_rail_cif.zip"],
)
print("Exporting GTFS from MariaDB...")
_docker_run_dtd2mysql(
network, db_container,
network,
db_container,
volumes=[f"{raw_abs}:/output"],
args=["--gtfs-zip", "/output/national_rail_gtfs_raw.zip"],
)

49
pipeline/transform/merge.py
View file

@ -94,11 +94,18 @@ def _build(
# Remap terminated postcodes to nearest active successor
postcode_mapping = build_postcode_mapping(arcgis_path)
wide = wide.join(
postcode_mapping.lazy(), left_on="postcode", right_on="old_postcode", how="left"
).with_columns(
pl.coalesce("new_postcode", "postcode").alias("postcode"),
).drop("new_postcode")
wide = (
wide.join(
postcode_mapping.lazy(),
left_on="postcode",
right_on="old_postcode",
how="left",
)
.with_columns(
pl.coalesce("new_postcode", "postcode").alias("postcode"),
)
.drop("new_postcode")
)
arcgis = (
pl.scan_parquet(arcgis_path)
@ -252,16 +259,18 @@ def _build(
.otherwise(pl.col("pp_property_type"))
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes",
# collapse terrace sub-types, and fold rare types into "Other"
.replace({
"Flat": "Flats/Maisonettes",
"Maisonette": "Flats/Maisonettes",
"End-Terrace": "Terraced",
"Mid-Terrace": "Terraced",
"Enclosed End-Terrace": "Terraced",
"Enclosed Mid-Terrace": "Terraced",
"Bungalow": "Other",
"Park home": "Other",
})
.replace(
{
"Flat": "Flats/Maisonettes",
"Maisonette": "Flats/Maisonettes",
"End-Terrace": "Terraced",
"Mid-Terrace": "Terraced",
"Enclosed End-Terrace": "Terraced",
"Enclosed Mid-Terrace": "Terraced",
"Bungalow": "Other",
"Park home": "Other",
}
)
.alias("property_type")
)
@ -426,10 +435,16 @@ def main():
help="Census 2021 population by LSOA parquet file",
)
parser.add_argument(
"--output-postcodes", type=Path, required=True, help="Output postcode parquet file path"
"--output-postcodes",
type=Path,
required=True,
help="Output postcode parquet file path",
)
parser.add_argument(
"--output-properties", type=Path, required=True, help="Output properties parquet file path"
"--output-properties",
type=Path,
required=True,
help="Output properties parquet file path",
)
args = parser.parse_args()

4
pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
View file

@ -454,9 +454,7 @@ class TestFillHoles:
hole1 = [(10, 10), (20, 10), (20, 20), (10, 20), (10, 10)]
outer2 = [(60, 60), (110, 60), (110, 110), (60, 110), (60, 60)]
hole2 = [(70, 70), (80, 70), (80, 80), (70, 80), (70, 70)]
mp = MultiPolygon(
[Polygon(outer1, [hole1]), Polygon(outer2, [hole2])]
)
mp = MultiPolygon([Polygon(outer1, [hole1]), Polygon(outer2, [hole2])])
result = _fill_holes(mp)
assert result.geom_type == "MultiPolygon"
for p in result.geoms:

17
pipeline/transform/price_estimation/backtest.py
View file

@ -112,7 +112,9 @@ def predict(test: pl.DataFrame, index: pl.DataFrame) -> pl.DataFrame:
def compute_metrics(actual: np.ndarray, predicted: np.ndarray) -> dict:
valid = np.isfinite(predicted) & np.isfinite(actual) & (actual > 0) & (predicted > 0)
valid = (
np.isfinite(predicted) & np.isfinite(actual) & (actual > 0) & (predicted > 0)
)
actual = actual[valid]
predicted = predicted[valid]
@ -176,7 +178,10 @@ def main():
"--input", type=Path, required=True, help="Path to properties.parquet"
)
parser.add_argument(
"--postcodes", type=Path, required=True, help="Path to postcode.parquet (for lat/lon)"
"--postcodes",
type=Path,
required=True,
help="Path to postcode.parquet (for lat/lon)",
)
parser.add_argument(
"--output", type=Path, required=True, help="Output backtest_results.parquet"
@ -185,7 +190,9 @@ def main():
# Build index from pre-test data only (temporal holdout)
print(f"Building price index (pairs with year2 < {TEST_YEAR_MIN})...")
index = build_index(args.input, max_pair_year=TEST_YEAR_MIN, postcodes_path=args.postcodes)
index = build_index(
args.input, max_pair_year=TEST_YEAR_MIN, postcodes_path=args.postcodes
)
print(
f"\nHoldout index: {len(index):,} rows, {index['sector'].n_unique():,} sectors, "
f"{index['type_group'].n_unique()} type groups"
@ -233,7 +240,9 @@ def main():
knn_est = knn_psm * fa * temporal_adj
n_knn = int((np.isfinite(knn_est) & (knn_est > 0)).sum())
print(f" kNN estimates: {n_knn:,} of {len(test):,} ({n_knn / len(test) * 100:.1f}%)")
print(
f" kNN estimates: {n_knn:,} of {len(test):,} ({n_knn / len(test) * 100:.1f}%)"
)
# Blend: (1-w)*index + w*kNN where both available
index_est = test["predicted"].to_numpy().astype(np.float64)

4
pipeline/transform/price_estimation/estimate.py
View file

@ -107,9 +107,7 @@ def main():
pl.when(has_price)
.then(
pl.col("Last known price").cast(pl.Float64)
* (
pl.col("_log_index_current_interp") - pl.col("_log_index_sale_interp")
)
* (pl.col("_log_index_current_interp") - pl.col("_log_index_sale_interp"))
.clip(-MAX_LOG_ADJUSTMENT, MAX_LOG_ADJUSTMENT)
.exp()
)

12
pipeline/transform/price_estimation/index.py
View file

@ -105,9 +105,7 @@ def extract_pairs(input_path: Path, max_year2: int | None = None) -> pl.DataFram
.alias("log_ratio"),
(
1.0
/ (pl.col("frac_year2") - pl.col("frac_year1"))
.cast(pl.Float64)
.sqrt()
/ (pl.col("frac_year2") - pl.col("frac_year1")).cast(pl.Float64).sqrt()
).alias("weight"),
)
.filter(pl.col("log_ratio").abs() <= OUTLIER_THRESHOLD)
@ -453,8 +451,12 @@ def main():
description="Build improved repeat-sales price index"
)
parser.add_argument("--input", type=Path, required=True)
parser.add_argument("--postcodes", type=Path, required=True,
help="Path to postcode.parquet (for lat/lon centroids)")
parser.add_argument(
"--postcodes",
type=Path,
required=True,
help="Path to postcode.parquet (for lat/lon centroids)",
)
parser.add_argument("--output", type=Path, required=True)
args = parser.parse_args()

63
pipeline/transform/price_estimation/knn.py
View file

@ -43,48 +43,39 @@ def build_knn_pool(
"""
print("Building kNN pool...")
lf = pl.scan_parquet(source) if isinstance(source, Path) else source
query = (
lf
.select(
"Postcode",
"Property type",
"lat",
"lon",
"Total floor area (sqm)",
"Last known price",
"Date of last transaction",
)
.filter(
pl.col("lat").is_not_null(),
pl.col("lon").is_not_null(),
pl.col("Total floor area (sqm)").is_not_null(),
pl.col("Total floor area (sqm)") > 0,
pl.col("Last known price").is_not_null(),
pl.col("Last known price") > 0,
pl.col("Postcode").is_not_null(),
pl.col("Date of last transaction").is_not_null(),
)
query = lf.select(
"Postcode",
"Property type",
"lat",
"lon",
"Total floor area (sqm)",
"Last known price",
"Date of last transaction",
).filter(
pl.col("lat").is_not_null(),
pl.col("lon").is_not_null(),
pl.col("Total floor area (sqm)").is_not_null(),
pl.col("Total floor area (sqm)") > 0,
pl.col("Last known price").is_not_null(),
pl.col("Last known price") > 0,
pl.col("Postcode").is_not_null(),
pl.col("Date of last transaction").is_not_null(),
)
if max_sale_year is not None:
query = query.filter(
pl.col("Date of last transaction").dt.year() < max_sale_year
)
pool = (
query.with_columns(
sector_expr(),
type_group_expr(),
(
pl.col("Date of last transaction").dt.year().cast(pl.Float64)
+ (
pl.col("Date of last transaction").dt.month().cast(pl.Float64)
- 1.0
)
/ 12.0
).alias("_sale_fy"),
pl.lit(ref_frac_year).alias("_ref_fy"),
).collect()
)
pool = query.with_columns(
sector_expr(),
type_group_expr(),
(
pl.col("Date of last transaction").dt.year().cast(pl.Float64)
+ (pl.col("Date of last transaction").dt.month().cast(pl.Float64) - 1.0)
/ 12.0
).alias("_sale_fy"),
pl.lit(ref_frac_year).alias("_ref_fy"),
).collect()
pool = pool.filter(pl.col("type_group").is_not_null())
print(f" {len(pool):,} pool properties with lat/lon, floor area, price")

4
pipeline/transform/transform_poi.py
View file

@ -1085,7 +1085,9 @@ def transform(
if cat not in all_set:
mapped_but_absent.append(cat)
if mapped_but_absent:
print(f"CATEGORY_MAP categories not in data (skipped): {sorted(mapped_but_absent)}")
print(
f"CATEGORY_MAP categories not in data (skipped): {sorted(mapped_but_absent)}"
)
# Drop unwanted categories
lf = lf.filter(~pl.col("category").is_in(list(DROP_CATEGORIES)))

4
pipeline/utils/england_geometry.py
View file

@ -27,7 +27,9 @@ def load_england_polygon(geojson_path: Path) -> PreparedGeometry:
return prep(geometry)
def in_england_mask(geojson_path: Path, lats: np.ndarray, lngs: np.ndarray) -> np.ndarray:
def in_england_mask(
geojson_path: Path, lats: np.ndarray, lngs: np.ndarray
) -> np.ndarray:
"""Vectorized check: which (lat, lng) points are within England.
Returns a boolean numpy array.

8
pipeline/utils/poi_counts.py
View file

@ -106,7 +106,9 @@ def count_pois_per_postcode(
if nearby is None:
continue
distances = haversine_km(poi_lats[nearby], poi_lngs[nearby], pc_lats[i], pc_lons[i])
distances = haversine_km(
poi_lats[nearby], poi_lngs[nearby], pc_lats[i], pc_lons[i]
)
within_mask = distances <= radius_km
within_indices = nearby[within_mask]
@ -179,7 +181,9 @@ def min_distance_per_postcode(
if nearby is None:
continue
distances = haversine_km(poi_lats[nearby], poi_lngs[nearby], pc_lats[i], pc_lons[i])
distances = haversine_km(
poi_lats[nearby], poi_lngs[nearby], pc_lats[i], pc_lons[i]
)
for group, cat_mask in category_masks.items():
group_mask = cat_mask[nearby]

45
pipeline/utils/postcode_mapping.py
View file

@ -15,26 +15,49 @@ def build_postcode_mapping(arcgis_path: Path) -> pl.DataFrame:
"""
arcgis = pl.scan_parquet(arcgis_path).filter(pl.col("ctry") == "E92000001")
active = arcgis.filter(pl.col("doterm").is_null()).select("pcds", "oseast1m", "osnrth1m").collect()
terminated = arcgis.filter(pl.col("doterm").is_not_null()).select("pcds", "oseast1m", "osnrth1m").collect()
active = (
arcgis.filter(pl.col("doterm").is_null())
.select("pcds", "oseast1m", "osnrth1m")
.collect()
)
terminated = (
arcgis.filter(pl.col("doterm").is_not_null())
.select("pcds", "oseast1m", "osnrth1m")
.collect()
)
print(f"Active postcodes: {active.height}, terminated postcodes: {terminated.height}")
print(
f"Active postcodes: {active.height}, terminated postcodes: {terminated.height}"
)
if terminated.height == 0:
return pl.DataFrame({"old_postcode": pl.Series([], dtype=pl.Utf8), "new_postcode": pl.Series([], dtype=pl.Utf8)})
return pl.DataFrame(
{
"old_postcode": pl.Series([], dtype=pl.Utf8),
"new_postcode": pl.Series([], dtype=pl.Utf8),
}
)
active_coords = np.column_stack([active["oseast1m"].to_numpy(), active["osnrth1m"].to_numpy()])
terminated_coords = np.column_stack([terminated["oseast1m"].to_numpy(), terminated["osnrth1m"].to_numpy()])
active_coords = np.column_stack(
[active["oseast1m"].to_numpy(), active["osnrth1m"].to_numpy()]
)
terminated_coords = np.column_stack(
[terminated["oseast1m"].to_numpy(), terminated["osnrth1m"].to_numpy()]
)
tree = cKDTree(active_coords)
distances, indices = tree.query(terminated_coords)
active_postcodes = active["pcds"]
mapping = pl.DataFrame({
"old_postcode": terminated["pcds"],
"new_postcode": active_postcodes.gather(indices),
})
mapping = pl.DataFrame(
{
"old_postcode": terminated["pcds"],
"new_postcode": active_postcodes.gather(indices),
}
)
print(f"Postcode mapping: max distance = {distances.max():.0f}m, median = {np.median(distances):.0f}m")
print(
f"Postcode mapping: max distance = {distances.max():.0f}m, median = {np.median(distances):.0f}m"
)
return mapping

8
pipeline/utils/test_poi_counts.py
View file

@ -72,7 +72,9 @@ def test_no_pois_returns_zeros(postcodes):
"category": pl.Series([], dtype=pl.String),
}
)
result = count_pois_per_postcode(postcodes, empty_pois, groups=POI_GROUPS, radius_km=2.0)
result = count_pois_per_postcode(
postcodes, empty_pois, groups=POI_GROUPS, radius_km=2.0
)
for group in POI_GROUPS:
col = f"{group}_2km"
@ -125,7 +127,9 @@ def test_min_distance_no_pois_returns_nan(postcodes):
"category": pl.Series([], dtype=pl.String),
}
)
result = min_distance_per_postcode(postcodes, empty_pois, groups={"train_tube": ["Rail station"]})
result = min_distance_per_postcode(
postcodes, empty_pois, groups={"train_tube": ["Rail station"]}
)
assert "train_tube_nearest_km" in result.columns
assert all(np.isnan(v) for v in result["train_tube_nearest_km"].to_list())

288
price_model.ipynb
View file

@ -28,8 +28,8 @@
"import matplotlib.pyplot as plt\n",
"\n",
"\n",
"pd.set_option('display.max_columns', None)\n",
"pd.set_option('display.max_colwidth', 60)"
"pd.set_option(\"display.max_columns\", None)\n",
"pd.set_option(\"display.max_colwidth\", 60)"
]
},
{
@ -47,7 +47,7 @@
"metadata": {},
"outputs": [],
"source": [
"param_import_path = '/bulk/wide-2.parquet'\n",
"param_import_path = \"/bulk/wide-2.parquet\"\n",
"\n",
"param_lookback = 3"
]
@ -128,7 +128,7 @@
],
"source": [
"data = pl.scan_parquet(param_import_path).unique(subset=[\"Postcode\", \"Address per EPC\"])\n",
"data = data.filter(pl.col('Total floor area (sqm)') > 10)\n",
"data = data.filter(pl.col(\"Total floor area (sqm)\") > 10)\n",
"\n",
"# print(data.collect_schema()) # column names and types\n",
"print(data.select(pl.len()).collect()) # row count\n",
@ -145,22 +145,20 @@
"outputs": [],
"source": [
"columns_required = [\n",
" # absolute neccesity \n",
" 'Postcode',\n",
" 'Address per EPC',\n",
" 'historical_prices',\n",
" 'Price per sqm',\n",
"\n",
" # absolute neccesity\n",
" \"Postcode\",\n",
" \"Address per EPC\",\n",
" \"historical_prices\",\n",
" \"Price per sqm\",\n",
" # faily fixed attributes\n",
" 'Property type', # or 'epc_property_type' or 'built_form'\n",
" 'Leashold/Freehold',\n",
" 'Total floor area (sqm)',\n",
" 'Rooms (including bedrooms & bathrooms)',\n",
" 'Approximate construction age',\n",
"\n",
" \"Property type\", # or 'epc_property_type' or 'built_form'\n",
" \"Leashold/Freehold\",\n",
" \"Total floor area (sqm)\",\n",
" \"Rooms (including bedrooms & bathrooms)\",\n",
" \"Approximate construction age\",\n",
" # latest\n",
" # 'date_of_transfer'\n",
" 'Last known price'\n",
" \"Last known price\",\n",
"]"
]
},
@ -440,8 +438,13 @@
],
"source": [
"# temp_Postcodes = [\"LE5 4ED\", \"E14 9GU\", \"YO8 9PW\", \"SW1P 3AN\", \"BH3 7DX\", \"E14 2DG\"]\n",
"temp_Postcodes = data.select('Postcode').collect().sample(10000)['Postcode'].to_list()\n",
"data_small = data.filter(pl.col(\"Postcode\").is_in(temp_Postcodes)).select(columns_required).collect().to_pandas()\n",
"temp_Postcodes = data.select(\"Postcode\").collect().sample(10000)[\"Postcode\"].to_list()\n",
"data_small = (\n",
" data.filter(pl.col(\"Postcode\").is_in(temp_Postcodes))\n",
" .select(columns_required)\n",
" .collect()\n",
" .to_pandas()\n",
")\n",
"data_small = data_small.explode(\"historical_prices\")\n",
"data_small[\"year\"] = data_small[\"historical_prices\"].apply(lambda x: x[\"year\"])\n",
"data_small[\"price\"] = data_small[\"historical_prices\"].apply(lambda x: x[\"price\"])\n",
@ -458,7 +461,7 @@
"outputs": [],
"source": [
"# data_small[\n",
"# (data_small['Postcode'] == 'E14 2DG') \n",
"# (data_small['Postcode'] == 'E14 2DG')\n",
"# & data_small['epc_address'].str.contains('76')\n",
"# ]"
]
@ -908,35 +911,45 @@
"from typing import Any\n",
"from pandas.core.frame import DataFrame\n",
"\n",
"print(f'rolling periods (relative): {[i for i in range(-param_lookback, 1)]}')\n",
"print(f\"rolling periods (relative): {[i for i in range(-param_lookback, 1)]}\")\n",
"\n",
"# Rolling average (±2 year), weighted by number of sales per year\n",
"pc_avg_raw = data_small.groupby(['Postcode', 'year']).agg(\n",
" ppsqm_sum=('Price per sqm', 'sum'),\n",
" ppsqm_count=('Price per sqm', 'count')\n",
").reset_index().sort_values(by=['Postcode', 'year'], ascending=False)\n",
"pc_avg_raw = (\n",
" data_small.groupby([\"Postcode\", \"year\"])\n",
" .agg(ppsqm_sum=(\"Price per sqm\", \"sum\"), ppsqm_count=(\"Price per sqm\", \"count\"))\n",
" .reset_index()\n",
" .sort_values(by=[\"Postcode\", \"year\"], ascending=False)\n",
")\n",
"\n",
"display(pc_avg_raw)\n",
"\n",
"# Each year's totals contribute to year-1, year, and year+1\n",
"pc_avg_expanded = pd.concat([\n",
" pc_avg_raw.assign(year=pc_avg_raw['year'] + offset) for offset in range(-param_lookback, 1) # \n",
"])\n",
"pc_avg_expanded = pd.concat(\n",
" [\n",
" pc_avg_raw.assign(year=pc_avg_raw[\"year\"] + offset)\n",
" for offset in range(-param_lookback, 1) #\n",
" ]\n",
")\n",
"\n",
"display(pc_avg_expanded)\n",
"\n",
"# Sum counts and sums, then divide to get weighted mean\n",
"pc_avg_complex = pc_avg_expanded.groupby(['Postcode', 'year']).agg(\n",
" ppsqm_sum=('ppsqm_sum', 'sum'),\n",
" ppsqm_count=('ppsqm_count', 'sum')\n",
").reset_index()\n",
"pc_avg_complex['Price per sqm PC AVG'] = pc_avg_complex['ppsqm_sum'] / pc_avg_complex['ppsqm_count']\n",
"pc_avg_complex: Any | DataFrame = pc_avg_complex[['Postcode', 'year', 'Price per sqm PC AVG']].sort_values(by=['Postcode', 'year'], ascending=False)\n",
"pc_avg_complex = (\n",
" pc_avg_expanded.groupby([\"Postcode\", \"year\"])\n",
" .agg(ppsqm_sum=(\"ppsqm_sum\", \"sum\"), ppsqm_count=(\"ppsqm_count\", \"sum\"))\n",
" .reset_index()\n",
")\n",
"pc_avg_complex[\"Price per sqm PC AVG\"] = (\n",
" pc_avg_complex[\"ppsqm_sum\"] / pc_avg_complex[\"ppsqm_count\"]\n",
")\n",
"pc_avg_complex: Any | DataFrame = pc_avg_complex[\n",
" [\"Postcode\", \"year\", \"Price per sqm PC AVG\"]\n",
"].sort_values(by=[\"Postcode\", \"year\"], ascending=False)\n",
"display(pc_avg_complex)\n",
"\n",
"temp_df = pc_avg_complex[pc_avg_complex['Postcode'] == data_small['Postcode'].iloc[0]]\n",
"print(data_small['Postcode'].iloc[0])\n",
"temp_df.plot.line(x='year', y='Price per sqm PC AVG')"
"temp_df = pc_avg_complex[pc_avg_complex[\"Postcode\"] == data_small[\"Postcode\"].iloc[0]]\n",
"print(data_small[\"Postcode\"].iloc[0])\n",
"temp_df.plot.line(x=\"year\", y=\"Price per sqm PC AVG\")"
]
},
{
@ -1111,9 +1124,13 @@
}
],
"source": [
"data_small = data_small.merge(pc_avg_complex, on=['Postcode', 'year'], suffixes=('', ' pc_avg_complex'))\n",
"data_small['c'] = data_small['Price per sqm'] / data_small['Price per sqm PC AVG']\n",
"data_small[['Postcode', 'Address per EPC', 'Price per sqm', 'Price per sqm PC AVG', 'c']]"
"data_small = data_small.merge(\n",
" pc_avg_complex, on=[\"Postcode\", \"year\"], suffixes=(\"\", \" pc_avg_complex\")\n",
")\n",
"data_small[\"c\"] = data_small[\"Price per sqm\"] / data_small[\"Price per sqm PC AVG\"]\n",
"data_small[\n",
" [\"Postcode\", \"Address per EPC\", \"Price per sqm\", \"Price per sqm PC AVG\", \"c\"]\n",
"]"
]
},
{
@ -1445,17 +1462,21 @@
],
"source": [
"# 1. Coefficient of Variation (std/mean) per property, filtered to 3+ sales\n",
"c_stats = data_small.groupby(['Postcode', 'Address per EPC']).agg(\n",
" n_sales=('c', 'count'),\n",
" year_min=('year', 'min'),\n",
" year_max=('year', 'max'),\n",
" c_mean=('c', 'mean'),\n",
" c_std=('c', 'std'),\n",
").dropna()\n",
"c_stats['c_cv'] = c_stats['c_std'] / c_stats['c_mean']\n",
"c_stats = (\n",
" data_small.groupby([\"Postcode\", \"Address per EPC\"])\n",
" .agg(\n",
" n_sales=(\"c\", \"count\"),\n",
" year_min=(\"year\", \"min\"),\n",
" year_max=(\"year\", \"max\"),\n",
" c_mean=(\"c\", \"mean\"),\n",
" c_std=(\"c\", \"std\"),\n",
" )\n",
" .dropna()\n",
")\n",
"c_stats[\"c_cv\"] = c_stats[\"c_std\"] / c_stats[\"c_mean\"]\n",
"# c_stats_3plus = c_stats[c_stats['n_sales'] >= 3]\n",
"# print(f\"Properties with 3+ sales: {len(c_stats_3plus)} / {len(c_stats)}\")\n",
"c_stats.sort_values('c_cv', ascending=False).head(20)"
"c_stats.sort_values(\"c_cv\", ascending=False).head(20)"
]
},
{
@ -2265,42 +2286,44 @@
"display(random_c)\n",
"\n",
"# pc avg trend\n",
"temp_pc_avg = pc_avg_complex[pc_avg_complex['Postcode'] == random_c.index[0][0]].sort_values(by='year')\n",
"temp_pc_avg = pc_avg_complex[\n",
" pc_avg_complex[\"Postcode\"] == random_c.index[0][0]\n",
"].sort_values(by=\"year\")\n",
"display(temp_pc_avg)\n",
"\n",
"# c for specific address\n",
"temp_postcode = data_small[\n",
" (data_small['Postcode'] == random_c.index[0][0]) \n",
" # & (data_small['Address per EPC'] == random_c.index[0][1]) \n",
"].sort_values(by='year')\n",
" (data_small[\"Postcode\"] == random_c.index[0][0])\n",
" # & (data_small['Address per EPC'] == random_c.index[0][1])\n",
"].sort_values(by=\"year\")\n",
"display(temp_postcode)\n",
"\n",
"temp_address = data_small[\n",
" (data_small['Postcode'] == random_c.index[0][0]) \n",
" & (data_small['Address per EPC'] == random_c.index[0][1]) \n",
"].sort_values(by='year')\n",
" (data_small[\"Postcode\"] == random_c.index[0][0])\n",
" & (data_small[\"Address per EPC\"] == random_c.index[0][1])\n",
"].sort_values(by=\"year\")\n",
"display(temp_address)\n",
"\n",
"# plot\n",
"\n",
"fig, ax1 = plt.subplots()\n",
"\n",
"temp_pc_avg.plot.line(x='year', y='Price per sqm PC AVG', ax=ax1, color='black')\n",
"temp_address.plot.line(x='year', y='Price per sqm', ax=ax1, color='green') \n",
"temp_pc_avg.plot.line(x=\"year\", y=\"Price per sqm PC AVG\", ax=ax1, color=\"black\")\n",
"temp_address.plot.line(x=\"year\", y=\"Price per sqm\", ax=ax1, color=\"green\")\n",
"\n",
"ax2 = ax1.twinx()\n",
"ax2.set_ylim(0, 3)\n",
"\n",
"for property in temp_postcode['Address per EPC'].unique():\n",
" property_data = temp_postcode[temp_postcode['Address per EPC'] == property]\n",
" property_data.plot.line(x='year', y='c', ax=ax2, color='orange', style=':')\n",
"for property in temp_postcode[\"Address per EPC\"].unique():\n",
" property_data = temp_postcode[temp_postcode[\"Address per EPC\"] == property]\n",
" property_data.plot.line(x=\"year\", y=\"c\", ax=ax2, color=\"orange\", style=\":\")\n",
"\n",
"temp_address.plot.line(x='year', y='c', ax=ax2, color='red', style=':')\n",
"temp_address.plot.line(x=\"year\", y=\"c\", ax=ax2, color=\"red\", style=\":\")\n",
"\n",
"ax1.set_ylabel('Price per sqm')\n",
"ax2.set_ylabel('c')\n",
"ax1.set_ylabel(\"Price per sqm\")\n",
"ax2.set_ylabel(\"c\")\n",
"\n",
"plt.show()\n"
"plt.show()"
]
},
{
@ -2640,17 +2663,21 @@
],
"source": [
"# 1. Coefficient of Variation (std/mean) per property, filtered to 3+ sales\n",
"c_stats = data_small.groupby(['Postcode', 'Address per EPC']).agg(\n",
" n_sales=('c', 'count'),\n",
" year_min=('year', 'min'),\n",
" year_max=('year', 'max'),\n",
" c_mean=('c', 'mean'),\n",
" c_std=('c', 'std'),\n",
").dropna()\n",
"c_stats['c_cv'] = c_stats['c_std'] / c_stats['c_mean']\n",
"c_stats = (\n",
" data_small.groupby([\"Postcode\", \"Address per EPC\"])\n",
" .agg(\n",
" n_sales=(\"c\", \"count\"),\n",
" year_min=(\"year\", \"min\"),\n",
" year_max=(\"year\", \"max\"),\n",
" c_mean=(\"c\", \"mean\"),\n",
" c_std=(\"c\", \"std\"),\n",
" )\n",
" .dropna()\n",
")\n",
"c_stats[\"c_cv\"] = c_stats[\"c_std\"] / c_stats[\"c_mean\"]\n",
"# c_stats_3plus = c_stats[c_stats['n_sales'] >= 3]\n",
"# print(f\"Properties with 3+ sales: {len(c_stats_3plus)} / {len(c_stats)}\")\n",
"c_stats.sort_values('c_cv', ascending=False).head(20)"
"c_stats.sort_values(\"c_cv\", ascending=False).head(20)"
]
},
{
@ -2685,31 +2712,41 @@
"\n",
"fig, axes = plt.subplots(1, 2, figsize=(12, 4))\n",
"\n",
"axes[0].hist(c_stats['c_std'], bins=100, edgecolor='black')\n",
"axes[0].set_xlabel('Std of c')\n",
"axes[0].set_ylabel('Number of properties')\n",
"axes[0].set_title('Distribution of c stability (std)')\n",
"axes[0].axvline(x=c_stats['c_std'].median(), color='red', linestyle='--', label=f'Median ({c_stats['c_std'].median()}) threshold')\n",
"axes[0].hist(c_stats[\"c_std\"], bins=100, edgecolor=\"black\")\n",
"axes[0].set_xlabel(\"Std of c\")\n",
"axes[0].set_ylabel(\"Number of properties\")\n",
"axes[0].set_title(\"Distribution of c stability (std)\")\n",
"axes[0].axvline(\n",
" x=c_stats[\"c_std\"].median(),\n",
" color=\"red\",\n",
" linestyle=\"--\",\n",
" label=f\"Median ({c_stats['c_std'].median()}) threshold\",\n",
")\n",
"axes[0].legend()\n",
"\n",
"axes[1].hist(c_stats['c_cv'], bins=100, edgecolor='black')\n",
"axes[1].set_xlabel('CV of c (std/mean)')\n",
"axes[1].set_ylabel('Number of properties')\n",
"axes[1].set_title('Distribution of c stability (CV)')\n",
"axes[1].axvline(x=c_stats['c_cv'].median(), color='red', linestyle='--', label=f'Median ({c_stats['c_cv'].median()}) threshold')\n",
"axes[1].hist(c_stats[\"c_cv\"], bins=100, edgecolor=\"black\")\n",
"axes[1].set_xlabel(\"CV of c (std/mean)\")\n",
"axes[1].set_ylabel(\"Number of properties\")\n",
"axes[1].set_title(\"Distribution of c stability (CV)\")\n",
"axes[1].axvline(\n",
" x=c_stats[\"c_cv\"].median(),\n",
" color=\"red\",\n",
" linestyle=\"--\",\n",
" label=f\"Median ({c_stats['c_cv'].median()}) threshold\",\n",
")\n",
"axes[1].legend()\n",
"\n",
"plt.tight_layout()\n",
"plt.show()\n",
"\n",
"# output text\n",
"pct_stable = (c_stats['c_cv'] < 0.1).mean() * 100\n",
"pct_stable = (c_stats[\"c_cv\"] < 0.1).mean() * 100\n",
"print(f\"{pct_stable:.1f}% of properties have CV < 0.1\")\n",
"\n",
"pct_stable = (c_stats['c_cv'] < 0.2).mean() * 100\n",
"pct_stable = (c_stats[\"c_cv\"] < 0.2).mean() * 100\n",
"print(f\"{pct_stable:.1f}% of properties have CV < 0.2\")\n",
"\n",
"pct_stable = (c_stats['c_cv'] < 0.3).mean() * 100\n",
"pct_stable = (c_stats[\"c_cv\"] < 0.3).mean() * 100\n",
"print(f\"{pct_stable:.1f}% of properties have CV < 0.3\")"
]
},
@ -3299,7 +3336,7 @@
}
],
"source": [
"unstable_c = c_stats.sort_values('c_cv', ascending=False)['c_cv'][:20]\n",
"unstable_c = c_stats.sort_values(\"c_cv\", ascending=False)[\"c_cv\"][:20]\n",
"display(unstable_c)\n",
"\n",
"unstable_c_specific = random.randint(0, 20)\n",
@ -3308,41 +3345,43 @@
"print(unstable_c.index[unstable_c_specific][1])\n",
"\n",
"# pc avg trend\n",
"temp_pc_avg = pc_avg_complex[pc_avg_complex['Postcode'] == unstable_c.index[unstable_c_specific][0]].sort_values(by='year')\n",
"temp_pc_avg = pc_avg_complex[\n",
" pc_avg_complex[\"Postcode\"] == unstable_c.index[unstable_c_specific][0]\n",
"].sort_values(by=\"year\")\n",
"display(temp_pc_avg)\n",
"\n",
"# c for specific postcode\n",
"temp_postcode = data_small[\n",
" (data_small['Postcode'] == unstable_c.index[unstable_c_specific][0]) \n",
" # & (data_small['Address per EPC'] == unstable_c.index[unstable_c_specific][1]) \n",
"].sort_values(by='year')\n",
" (data_small[\"Postcode\"] == unstable_c.index[unstable_c_specific][0])\n",
" # & (data_small['Address per EPC'] == unstable_c.index[unstable_c_specific][1])\n",
"].sort_values(by=\"year\")\n",
"display(temp_address)\n",
"\n",
"# c for specific address\n",
"temp_address = data_small[\n",
" (data_small['Postcode'] == unstable_c.index[unstable_c_specific][0]) \n",
" & (data_small['Address per EPC'] == unstable_c.index[unstable_c_specific][1]) \n",
"].sort_values(by='year')\n",
" (data_small[\"Postcode\"] == unstable_c.index[unstable_c_specific][0])\n",
" & (data_small[\"Address per EPC\"] == unstable_c.index[unstable_c_specific][1])\n",
"].sort_values(by=\"year\")\n",
"display(temp_address)\n",
"\n",
"# plot\n",
"\n",
"fig, ax1 = plt.subplots()\n",
"\n",
"temp_pc_avg.plot.line(x='year', y='Price per sqm PC AVG', ax=ax1, color='black')\n",
"temp_address.plot.line(x='year', y='Price per sqm', ax=ax1, color='green') \n",
"temp_pc_avg.plot.line(x=\"year\", y=\"Price per sqm PC AVG\", ax=ax1, color=\"black\")\n",
"temp_address.plot.line(x=\"year\", y=\"Price per sqm\", ax=ax1, color=\"green\")\n",
"\n",
"ax2 = ax1.twinx()\n",
"\n",
"for property in temp_postcode['Address per EPC'].unique():\n",
" property_data = temp_postcode[temp_postcode['Address per EPC'] == property]\n",
" property_data.plot.line(x='year', y='c', ax=ax2, color='orange', style=':')\n",
"temp_address.plot.line(x='year', y='c', ax=ax2, color='red', style=':')\n",
"for property in temp_postcode[\"Address per EPC\"].unique():\n",
" property_data = temp_postcode[temp_postcode[\"Address per EPC\"] == property]\n",
" property_data.plot.line(x=\"year\", y=\"c\", ax=ax2, color=\"orange\", style=\":\")\n",
"temp_address.plot.line(x=\"year\", y=\"c\", ax=ax2, color=\"red\", style=\":\")\n",
"\n",
"ax1.set_ylabel('Price per sqm')\n",
"ax2.set_ylabel('c')\n",
"ax1.set_ylabel(\"Price per sqm\")\n",
"ax2.set_ylabel(\"c\")\n",
"\n",
"plt.show()\n"
"plt.show()"
]
},
{
@ -3370,11 +3409,11 @@
],
"source": [
"# select random address\n",
"one_property = data_small.sample(1)[['Postcode', 'Address per EPC']].iloc[0]\n",
"postcode = one_property['Postcode']\n",
"address = one_property['Address per EPC']\n",
"print(f'Postcode: {postcode}')\n",
"print(f'Address: {address}')"
"one_property = data_small.sample(1)[[\"Postcode\", \"Address per EPC\"]].iloc[0]\n",
"postcode = one_property[\"Postcode\"]\n",
"address = one_property[\"Address per EPC\"]\n",
"print(f\"Postcode: {postcode}\")\n",
"print(f\"Address: {address}\")"
]
},
{
@ -3481,22 +3520,21 @@
],
"source": [
"property_data = data_small[\n",
" (data_small['Postcode'] == postcode) \n",
" & (data_small['Address per EPC'] == address) \n",
" (data_small[\"Postcode\"] == postcode) & (data_small[\"Address per EPC\"] == address)\n",
"]\n",
"latest_year = property_data['year'].max()\n",
"print(f'Latest year of data: {latest_year}')\n",
"latest_year = property_data[\"year\"].max()\n",
"print(f\"Latest year of data: {latest_year}\")\n",
"\n",
"# Get only the latest year's data for this property (this is what we want to predict)\n",
"data_small_test = property_data[property_data['year'] == latest_year]\n",
"data_small_test = property_data[property_data[\"year\"] == latest_year]\n",
"\n",
"# Remove only the latest year's data from training (keep historical data for this property)\n",
"data_small_train = data_small.drop(data_small_test.index)\n",
"\n",
"print()\n",
"print(f'data_small.shape = {data_small.shape}')\n",
"print(f'data_small_train.shape = {data_small_train.shape}')\n",
"print(f'data_small_test.shape = {data_small_test.shape}')\n",
"print(f\"data_small.shape = {data_small.shape}\")\n",
"print(f\"data_small_train.shape = {data_small_train.shape}\")\n",
"print(f\"data_small_test.shape = {data_small_test.shape}\")\n",
"display(data_small_test)\n",
"data_small.shape[0] == data_small_test.shape[0] + data_small_train.shape[0]"
]
@ -3607,9 +3645,9 @@
"# get latest c in data_small_train\n",
"\n",
"latest_train_address = data_small_train[\n",
" (data_small_train['Postcode'] == postcode) \n",
" & (data_small_train['Address per EPC'] == address) \n",
"].sort_values(by='year')\n",
" (data_small_train[\"Postcode\"] == postcode)\n",
" & (data_small_train[\"Address per EPC\"] == address)\n",
"].sort_values(by=\"year\")\n",
"\n",
"latest_train_address"
]
@ -3630,10 +3668,10 @@
}
],
"source": [
"latest_train_c = latest_train_address['c'].iloc[-1]\n",
"latest_train_pc_avg = latest_train_address['Price per sqm PC AVG'].iloc[-1]\n",
"print(f'Latest c in training data: {latest_train_c:.3f}')\n",
"print(f'Latest price per sqm in training data: {latest_train_pc_avg:.2f}') "
"latest_train_c = latest_train_address[\"c\"].iloc[-1]\n",
"latest_train_pc_avg = latest_train_address[\"Price per sqm PC AVG\"].iloc[-1]\n",
"print(f\"Latest c in training data: {latest_train_c:.3f}\")\n",
"print(f\"Latest price per sqm in training data: {latest_train_pc_avg:.2f}\")"
]
},
{
@ -3654,7 +3692,7 @@
}
],
"source": [
"latest_train_c * latest_train_pc_avg * data_small_test['Total floor area (sqm)'].iloc[0]"
"latest_train_c * latest_train_pc_avg * data_small_test[\"Total floor area (sqm)\"].iloc[0]"
]
},
{

2
scripts/remove_bg.py
View file

@ -4,6 +4,7 @@ import sys
from collections import deque
from PIL import Image
def remove_white_bg(path: str, tolerance: int = 20, out: str | None = None):
img = Image.open(path).convert("RGBA")
pixels = img.load()
@ -43,6 +44,7 @@ def remove_white_bg(path: str, tolerance: int = 20, out: str | None = None):
img.save(dest)
print(f"Saved to {dest} ({img.size[0]}x{img.size[1]})")
if __name__ == "__main__":
if len(sys.argv) < 2:
print("Usage: python remove_bg.py <image> [tolerance] [output]")

2
server-rs/Cargo.toml
View file

@ -9,7 +9,7 @@ clap = { version = "4", features = ["derive", "env"] }
axum = "0.8"
tower-http = { version = "0.6", features = ["cors", "fs", "compression-gzip", "compression-zstd", "trace"] }
tokio = { version = "1", features = ["full"] }
polars = { version = "0.46", features = ["parquet", "lazy", "dtype-struct", "dtype-u8", "dtype-u16", "dtype-i8", "dtype-i16"] }
polars = { version = "0.46", features = ["parquet", "lazy", "dtype-struct", "dtype-u8", "dtype-u16", "dtype-i8", "dtype-i16", "round_series"] }
h3o = "0.7"
serde = { version = "1", features = ["derive"] }
serde_json = "1"

981
server-rs/logs/server.log.2026-03-15 Normal file
View file

@ -0,0 +1,981 @@
2026-03-15T19:07:11.371851Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:07:11.372040Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:07:11.372050Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:07:11.473120Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:07:11.473130Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:07:22.441464Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T19:07:22.441476Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T19:07:22.848304Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T19:07:22.848315Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T19:07:22.913269Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T19:07:22.913279Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T19:07:33.981737Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T19:07:33.981832Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T19:07:35.443457Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T19:07:35.594896Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T19:07:36.004267Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T19:07:37.616212Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T19:07:39.772016Z INFO property_map_server::data::property: Building enum features
2026-03-15T19:07:41.167109Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T19:07:49.922117Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T19:07:49.922128Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T19:07:50.511333Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T19:07:50.511342Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T19:07:51.475798Z INFO property_map_server::data::property: Building interned strings
2026-03-15T19:07:57.644399Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T19:08:00.205219Z INFO property_map_server::data::property: Data loading complete
2026-03-15T19:08:01.951719Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T19:08:01.951728Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T19:08:02.049262Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T19:08:02.049271Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T19:08:02.477049Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T19:08:02.477606Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T19:08:02.477614Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T19:08:02.501985Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T19:08:02.612770Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T19:08:02.613426Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T19:08:02.652324Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T19:08:02.652334Z INFO property_map_server: Building POI spatial grid index
2026-03-15T19:08:02.658355Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T19:08:02.658366Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T19:08:02.660009Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T19:08:02.660839Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T19:08:02.660901Z INFO property_map_server: Place data loaded places=3474
2026-03-15T19:08:02.660910Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T19:08:02.660914Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T19:08:02.664216Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T19:08:04.225988Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T19:08:04.225999Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T19:08:04.226018Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T19:08:04.265284Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T19:08:04.310151Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T19:08:04.343659Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T19:08:04.343821Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T19:08:04.343836Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T19:08:04.432857Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T19:08:04.438796Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T19:08:04.443153Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T19:08:04.727071Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfectpostcodes.schmelczer.dev/pb
2026-03-15T19:08:04.742347Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T19:08:04.742398Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T19:08:04.742414Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T19:08:04.783484Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T19:08:04.784506Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T19:08:04.785961Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T19:08:04.794234Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T19:08:04.794259Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T19:08:04.794312Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T19:08:07.250052Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T19:08:07.250147Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T19:08:43.332953Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=905 cells_after_filter=904 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=70.2 total_ms=82.4
2026-03-15T19:08:44.121660Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1337 cells_after_filter=1335 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=68.3 total_ms=84.2
2026-03-15T19:08:45.206450Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1337 cells_after_filter=1335 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=71.1 total_ms=94.8
2026-03-15T19:08:45.422613Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1356 cells_after_filter=1351 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1077.7 total_ms=1109.0
2026-03-15T19:08:47.208854Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1337 cells_after_filter=1335 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=68.0 total_ms=79.9
2026-03-15T19:08:47.294458Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1356 cells_after_filter=1351 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1074.4 total_ms=1104.8
2026-03-15T19:08:48.178659Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=905 cells_after_filter=904 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=79.6 total_ms=86.6
2026-03-15T19:08:48.443812Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1337 cells_after_filter=1335 truncated=false bounds=50.5989,-1.9210,52.4537,1.7355 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=71.5 total_ms=83.5
2026-03-15T19:08:51.505423Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=602 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.0
2026-03-15T19:08:52.293946Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1004 cells_after_filter=1004 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=55.5 total_ms=64.3
2026-03-15T19:09:12.606611Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=86195dac7ffffff resolution=6 total=602 returned=100 offset=0 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.4
2026-03-15T19:09:17.589934Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=25877 filters=1 filters_raw="Listing status:Historical sale" ms=12.0
2026-03-15T19:09:19.152498Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1021 cells_after_filter=1017 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=944.3 total_ms=960.2
2026-03-15T19:09:19.587280Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=602 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.0
2026-03-15T19:09:20.120819Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1004 cells_after_filter=1004 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=59.6 total_ms=68.9
2026-03-15T19:09:21.757704Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=86195dac7ffffff resolution=6 total=602 returned=100 offset=0 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.3
2026-03-15T19:09:26.458314Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=25877 filters=1 filters_raw="Listing status:Historical sale" ms=12.2
2026-03-15T19:09:26.734420Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=602 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.1
2026-03-15T19:09:27.270779Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1004 cells_after_filter=1004 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=59.2 total_ms=68.7
2026-03-15T19:09:27.465769Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=29 filters=1 filters_raw="Listing status:For rent" ms=1.5
2026-03-15T19:09:27.551401Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1021 cells_after_filter=1017 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=938.9 total_ms=955.1
2026-03-15T19:09:27.636702Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=688 cells_after_filter=688 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=64.2 total_ms=69.4
2026-03-15T19:09:28.107949Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=602 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=1.0
2026-03-15T19:09:28.647682Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1004 cells_after_filter=1004 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=57.9 total_ms=66.8
2026-03-15T19:10:43.696300Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=29 filters=1 filters_raw="Listing status:For rent" ms=1.5
2026-03-15T19:10:43.917385Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=688 cells_after_filter=688 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=66.4 total_ms=72.3
2026-03-15T19:10:44.773099Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=86195dac7ffffff resolution=6 total_count=602 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" ms=0.9
2026-03-15T19:10:45.309165Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1004 cells_after_filter=1004 truncated=false bounds=50.5989,-1.3279,52.4537,1.1425 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=55.8 total_ms=65.3
2026-03-15T19:11:44.827084Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:11:45.019108Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:11:45.539093Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=67.3 total_ms=79.7
2026-03-15T19:12:22.900048Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1266.0 total_ms=1293.0
2026-03-15T19:12:43.487506Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=97.7 total_ms=119.2
2026-03-15T19:12:56.981906Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=92.3 total_ms=105.6
2026-03-15T19:15:16.007945Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:15:16.008103Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:15:16.008110Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:15:16.129677Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:15:16.129690Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:15:25.994671Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T19:15:25.994690Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T19:15:26.528002Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T19:15:26.528016Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T19:15:26.610377Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T19:15:26.610387Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T19:16:11.721573Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T19:16:11.721688Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T19:16:13.315773Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T19:16:13.525913Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T19:16:14.026360Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T19:16:15.880642Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T19:16:18.439507Z INFO property_map_server::data::property: Building enum features
2026-03-15T19:16:20.159326Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T19:16:22.764657Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T19:16:22.764667Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T19:16:23.416811Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T19:16:23.416820Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T19:16:24.612917Z INFO property_map_server::data::property: Building interned strings
2026-03-15T19:16:31.709728Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T19:16:34.550669Z INFO property_map_server::data::property: Data loading complete
2026-03-15T19:16:36.027023Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T19:16:36.027032Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T19:16:36.427705Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T19:16:36.427713Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T19:16:36.848803Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T19:16:36.848848Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T19:16:36.848862Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T19:16:36.922825Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T19:16:37.040700Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T19:16:37.041316Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T19:16:37.080709Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T19:16:37.080719Z INFO property_map_server: Building POI spatial grid index
2026-03-15T19:16:37.087492Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T19:16:37.087505Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T19:16:37.092797Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T19:16:37.093630Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T19:16:37.093696Z INFO property_map_server: Place data loaded places=3474
2026-03-15T19:16:37.093709Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T19:16:37.093714Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T19:16:37.097696Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T19:16:40.166666Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T19:16:40.166676Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T19:16:40.166689Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T19:16:40.223398Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T19:16:40.267971Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T19:16:40.329251Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T19:16:40.329417Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T19:16:40.329432Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T19:16:40.466894Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T19:16:40.474329Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T19:16:40.488014Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T19:16:40.724141Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfectpostcodes.schmelczer.dev/pb
2026-03-15T19:16:40.728811Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T19:16:40.728841Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T19:16:40.728857Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T19:16:40.781239Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T19:16:40.786720Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T19:16:40.789828Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T19:16:40.812650Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T19:16:40.812688Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T19:16:40.812745Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T19:16:44.806149Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T19:16:44.806208Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T19:18:41.554399Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1102.3 total_ms=1142.0
2026-03-15T19:18:51.943051Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1089.3 total_ms=1111.0
2026-03-15T19:18:52.235271Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=88.8 total_ms=101.0
2026-03-15T19:18:53.034978Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=71.2 total_ms=79.2
2026-03-15T19:18:54.485285Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=86.6 total_ms=98.8
2026-03-15T19:18:56.331564Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1100.7 total_ms=1122.4
2026-03-15T19:18:57.013412Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=86.3 total_ms=98.3
2026-03-15T19:18:57.497646Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=80.4 total_ms=87.3
2026-03-15T19:18:58.307229Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=86.4 total_ms=98.6
2026-03-15T19:18:59.209272Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=92.3 total_ms=115.9
2026-03-15T19:18:59.605162Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1104.1 total_ms=1125.8
2026-03-15T19:19:00.641552Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=101.5 total_ms=114.0
2026-03-15T19:19:01.055691Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1102.8 total_ms=1133.7
2026-03-15T19:19:01.767817Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=70.5 total_ms=77.5
2026-03-15T19:19:02.094672Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=95.1 total_ms=107.1
2026-03-15T19:19:04.696909Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=1094.7 total_ms=1116.4
2026-03-15T19:20:18.135252Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=87.8 total_ms=99.9
2026-03-15T19:21:06.589853Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=70.6 total_ms=77.5
2026-03-15T19:22:21.723624Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:22:21.723777Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:22:21.723788Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:22:21.792919Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:22:21.792931Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:24:04.122070Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:24:04.122238Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:24:04.122243Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:24:04.183691Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:24:04.183700Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:24:10.703405Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:24:10.703567Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:24:10.703573Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:24:10.764393Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:24:10.764404Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:24:13.175087Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T19:24:13.175096Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T19:24:14.743132Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T19:24:14.743145Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T19:24:15.276707Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T19:24:15.276719Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T19:32:31.326505Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T19:32:31.326667Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T19:32:31.326674Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T19:32:31.468299Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T19:32:31.468311Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T19:32:36.199335Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T19:32:36.199348Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T19:32:36.594288Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T19:32:36.594299Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T19:32:36.659669Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T19:32:36.659679Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T19:33:03.760178Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T19:33:03.760265Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T19:33:05.203275Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T19:33:05.385487Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T19:33:05.825530Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T19:33:07.452637Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T19:33:09.737556Z INFO property_map_server::data::property: Building enum features
2026-03-15T19:33:11.190065Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T19:33:13.640495Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T19:33:13.640506Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T19:33:14.224090Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T19:33:14.224101Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T19:33:15.218314Z INFO property_map_server::data::property: Building interned strings
2026-03-15T19:33:21.691381Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T19:33:24.347401Z INFO property_map_server::data::property: Data loading complete
2026-03-15T19:33:26.411071Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T19:33:26.411081Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T19:33:26.535487Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T19:33:26.535498Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T19:33:27.113000Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T19:33:27.113028Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T19:33:27.113036Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T19:33:27.165121Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T19:33:27.304878Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T19:33:27.305504Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T19:33:27.346347Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T19:33:27.346357Z INFO property_map_server: Building POI spatial grid index
2026-03-15T19:33:27.352391Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T19:33:27.352399Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T19:33:27.365758Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T19:33:27.366584Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T19:33:27.366644Z INFO property_map_server: Place data loaded places=3474
2026-03-15T19:33:27.366655Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T19:33:27.366659Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T19:33:27.392761Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T19:33:36.895174Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T19:33:36.895188Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T19:33:36.895208Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T19:33:36.998297Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T19:33:37.053356Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T19:33:37.109062Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T19:33:37.109355Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T19:33:37.109374Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T19:33:37.762412Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T19:33:37.767896Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T19:33:37.775340Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T19:33:38.060153Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfect-postcodes.co.uk/pb
2026-03-15T19:33:38.063925Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T19:33:38.063950Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T19:33:38.063967Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T19:33:38.278834Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T19:33:38.287416Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T19:33:38.292977Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T19:33:38.359034Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T19:33:38.359070Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T19:33:38.359129Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T19:33:48.937764Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T19:33:48.937811Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T19:33:49.510480Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:49.711250Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:49.717966Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:49.758705Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:50.515563Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:50.515611Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:50.526782Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:50.526799Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:50.624015Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:50.954642Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:51.089161Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:33:51.089169Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:33:51.344202Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=148.7 total_ms=177.0
2026-03-15T19:34:49.486571Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:34:49.486761Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:34:50.105387Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=147.2 total_ms=181.6
2026-03-15T19:35:47.333901Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:35:47.633561Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:35:47.964989Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=127.2 total_ms=151.0
2026-03-15T19:36:10.914163Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:36:10.935314Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:36:11.591360Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated current price:23297.51:inf" travel_entries=0 agg_ms=148.2 total_ms=179.1
2026-03-15T19:37:10.487304Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For sale;;Asking price:23297.51:inf" travel_entries=0 agg_ms=30.4 total_ms=42.4
2026-03-15T19:37:11.520281Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=34.6 total_ms=41.1
2026-03-15T19:37:12.429971Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=92.6 total_ms=114.8
2026-03-15T19:39:01.324173Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1374 cells_after_filter=1370 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated monthly rent:300:7750" travel_entries=0 agg_ms=78.6 total_ms=100.6
2026-03-15T19:39:11.452418Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=34.6 total_ms=46.5
2026-03-15T19:39:12.258031Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=104.1 total_ms=126.5
2026-03-15T19:39:13.209101Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=30.2 total_ms=41.9
2026-03-15T19:39:13.542817Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=86.2 total_ms=108.5
2026-03-15T19:39:17.811115Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1374 cells_after_filter=1370 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated monthly rent:300:7750" travel_entries=0 agg_ms=78.0 total_ms=100.1
2026-03-15T19:39:18.543391Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=31.1 total_ms=42.7
2026-03-15T19:39:21.422553Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=99.7 total_ms=122.0
2026-03-15T19:39:24.089893Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1374 cells_after_filter=1370 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated monthly rent:300:7750" travel_entries=0 agg_ms=65.8 total_ms=87.6
2026-03-15T19:39:34.096698Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=878 cells_after_filter=877 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For rent;;Asking rent (monthly):300:7750" travel_entries=0 agg_ms=34.2 total_ms=40.6
2026-03-15T19:39:35.136941Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1374 cells_after_filter=1370 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated monthly rent:300:7750" travel_entries=0 agg_ms=77.3 total_ms=99.4
2026-03-15T19:39:35.997965Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=878 cells_after_filter=877 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For rent;;Asking rent (monthly):300:7750" travel_entries=0 agg_ms=27.5 total_ms=34.4
2026-03-15T19:39:36.896448Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=30.0 total_ms=43.4
2026-03-15T19:39:37.822906Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=97.3 total_ms=119.7
2026-03-15T19:39:38.005882Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=31.8 total_ms=43.6
2026-03-15T19:39:38.307634Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=32.0 total_ms=38.7
2026-03-15T19:39:39.034035Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=32.7 total_ms=44.5
2026-03-15T19:39:39.374475Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=100.4 total_ms=122.4
2026-03-15T19:39:39.415811Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=39.0 total_ms=50.9
2026-03-15T19:39:40.267881Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=28.4 total_ms=35.2
2026-03-15T19:39:40.542606Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=35.2 total_ms=47.1
2026-03-15T19:39:40.845586Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=88.6 total_ms=110.3
2026-03-15T19:39:42.206069Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=32.7 total_ms=44.7
2026-03-15T19:39:43.318640Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=27.1 total_ms=33.6
2026-03-15T19:39:44.688592Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=96.4 total_ms=117.7
2026-03-15T19:39:45.172008Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=29.2 total_ms=36.4
2026-03-15T19:39:45.850790Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=40.1 total_ms=51.9
2026-03-15T19:39:46.189922Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=29.0 total_ms=35.7
2026-03-15T19:39:46.998212Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=86.0 total_ms=108.9
2026-03-15T19:39:47.177336Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=29.1 total_ms=35.8
2026-03-15T19:39:47.696691Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=96.6 total_ms=118.0
2026-03-15T19:39:48.556326Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=911 cells_after_filter=910 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=28.2 total_ms=35.0
2026-03-15T19:39:48.906118Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=107.8 total_ms=129.7
2026-03-15T19:39:51.395259Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1374 cells_after_filter=1370 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:Historical sale;;Estimated monthly rent:300:7750" travel_entries=0 agg_ms=66.3 total_ms=88.4
2026-03-15T19:39:58.878189Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=878 cells_after_filter=877 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=2 filters_raw="Listing status:For rent;;Asking rent (monthly):300:7750" travel_entries=0 agg_ms=31.0 total_ms=37.7
2026-03-15T19:40:04.517960Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1349 cells_after_filter=1348 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=31.5 total_ms=44.3
2026-03-15T19:40:17.346685Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=96.9 total_ms=119.8
2026-03-15T19:47:52.591641Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T19:47:52.591925Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T19:47:53.240629Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=7065688 parallel=true cells_before_filter=1376 cells_after_filter=1372 truncated=false bounds=50.5887,-1.9407,52.4636,1.7553 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=95.9 total_ms=117.5
2026-03-15T19:48:12.279153Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=4253126 parallel=true cells_before_filter=3093 cells_after_filter=3064 truncated=false bounds=50.9497,-0.9772,51.9159,0.9233 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=54.4 total_ms=137.0
2026-03-15T19:48:13.512341Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=2883111 parallel=true cells_before_filter=1105 cells_after_filter=1088 truncated=false bounds=51.1146,-0.5552,51.6729,0.5420 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=36.5 total_ms=54.4
2026-03-15T19:48:14.110557Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=8 rows=1687410 parallel=true cells_before_filter=2479 cells_after_filter=2434 truncated=false bounds=51.2171,-0.3474,51.5584,0.3232 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=22.6 total_ms=61.3
2026-03-15T19:48:16.632555Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=1007854 parallel=true cells_before_filter=5324 cells_after_filter=5035 truncated=false bounds=51.3626,-0.1740,51.5564,0.2074 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=16.9 total_ms=117.8
2026-03-15T19:51:26.541773Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=190938 parallel=true cells_before_filter=1076 cells_after_filter=1033 truncated=false bounds=51.4310,-0.0323,51.5159,0.1349 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=3.8 total_ms=20.0
2026-03-15T19:51:30.533448Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=384437 parallel=true cells_before_filter=2704 cells_after_filter=2497 truncated=false bounds=51.4379,0.0032,51.5871,0.2970 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=7.4 total_ms=46.9
2026-03-15T19:51:31.069688Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=632131 parallel=true cells_before_filter=3388 cells_after_filter=3150 truncated=false bounds=51.4091,-0.1211,51.5583,0.1727 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=10.9 total_ms=60.6
2026-03-15T19:51:35.046811Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=65919 parallel=true cells_before_filter=341 cells_after_filter=288 truncated=false bounds=51.4524,-0.0211,51.4936,0.0599 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.6 total_ms=6.2
2026-03-15T19:51:36.299041Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=63016 parallel=true cells_before_filter=366 cells_after_filter=298 truncated=false bounds=51.4463,-0.0284,51.4875,0.0526 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.9 total_ms=6.5
2026-03-15T19:54:34.842844Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ec7ffff resolution=9 total_count=102 filters=1 filters_raw="Listing status:Historical sale" ms=0.1
2026-03-15T19:54:35.089458Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=295 cells_after_filter=209 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.6 total_ms=5.0
2026-03-15T19:54:36.130976Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=89194ad2ec7ffff resolution=9 total=102 returned=100 offset=0 filters=1 filters_raw="Listing status:Historical sale" ms=0.4
2026-03-15T19:54:38.679953Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ec7ffff resolution=9 total_count=5 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:54:38.832683Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.6 total_ms=2.4
2026-03-15T19:55:05.573712Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ec7ffff resolution=9 total_count=102 filters=1 filters_raw="Listing status:Historical sale" ms=0.1
2026-03-15T19:55:06.053141Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=295 cells_after_filter=209 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.4 total_ms=4.8
2026-03-15T19:55:06.289059Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ec7ffff resolution=9 total_count=5 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:55:06.464008Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.7 total_ms=2.4
2026-03-15T19:55:07.257113Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=63016 parallel=true cells_before_filter=329 cells_after_filter=275 truncated=false bounds=51.4463,-0.0284,51.4875,0.0526 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.8 total_ms=3.6
2026-03-15T19:55:10.176114Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ecfffff resolution=9 total_count=20 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:55:10.449590Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.8 total_ms=2.6
2026-03-15T19:55:11.581365Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=89194ad2ecfffff resolution=9 total=20 returned=20 offset=0 filters=1 filters_raw="Listing status:For sale" ms=0.3
2026-03-15T19:55:12.826310Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ecfffff resolution=9 total_count=1 filters=1 filters_raw="Listing status:For rent" ms=0.1
2026-03-15T19:55:13.296370Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=69 cells_after_filter=54 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For rent" travel_entries=0 agg_ms=0.4 total_ms=0.8
2026-03-15T19:55:14.274431Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ecfffff resolution=9 total_count=20 filters=1 filters_raw="Listing status:For sale" ms=0.2
2026-03-15T19:55:14.745705Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.8 total_ms=2.6
2026-03-15T19:55:15.410862Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.7 total_ms=2.6
2026-03-15T19:55:15.973540Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ecfffff resolution=9 total_count=285 filters=1 filters_raw="Listing status:Historical sale" ms=0.2
2026-03-15T19:55:16.149036Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=295 cells_after_filter=209 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.6 total_ms=5.2
2026-03-15T19:55:50.335806Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=63016 parallel=true cells_before_filter=366 cells_after_filter=298 truncated=false bounds=51.4463,-0.0284,51.4875,0.0526 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.8 total_ms=6.5
2026-03-15T19:55:51.579153Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2e53ffff resolution=9 total_count=119 filters=1 filters_raw="Listing status:Historical sale" ms=0.1
2026-03-15T19:55:51.847817Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=295 cells_after_filter=209 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:Historical sale" travel_entries=0 agg_ms=1.6 total_ms=4.8
2026-03-15T19:57:08.597832Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=5 rows=14962301 parallel=true cells_before_filter=687 cells_after_filter=687 truncated=false bounds=46.0000,-12.0000,56.5000,12.0000 filters=0 filters_raw="-" travel_entries=0 agg_ms=206.2 total_ms=218.4
2026-03-15T19:58:56.459660Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2e53ffff resolution=9 total_count=5 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:58:56.611313Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.8 total_ms=2.8
2026-03-15T19:58:58.208823Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=63016 parallel=true cells_before_filter=329 cells_after_filter=275 truncated=false bounds=51.4463,-0.0284,51.4875,0.0526 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=1.0 total_ms=4.1
2026-03-15T19:58:59.418179Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2e5bffff resolution=9 total_count=5 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:58:59.646741Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=51710 parallel=true cells_before_filter=263 cells_after_filter=194 truncated=false bounds=51.4463,-0.0153,51.4875,0.0394 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.7 total_ms=2.5
2026-03-15T19:59:13.261566Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad2ecfffff resolution=9 total_count=20 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:59:16.381213Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad216fffff resolution=9 total_count=2 filters=1 filters_raw="Listing status:For sale" ms=0.1
2026-03-15T19:59:19.468079Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=65337 parallel=true cells_before_filter=263 cells_after_filter=187 truncated=false bounds=51.4575,-0.0360,51.4986,0.0187 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.8 total_ms=2.8
2026-03-15T19:59:20.912780Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=57987 parallel=true cells_before_filter=230 cells_after_filter=192 truncated=false bounds=51.4560,-0.0422,51.4972,0.0126 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.6 total_ms=2.4
2026-03-15T19:59:22.439657Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=45 filters=1 filters_raw="Listing status:For sale" ms=0.2
2026-03-15T19:59:25.975325Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=89194ad04a7ffff resolution=9 total=45 returned=45 offset=0 filters=1 filters_raw="Listing status:For sale" ms=0.4
2026-03-15T19:59:44.130233Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=27673 parallel=false cells_before_filter=124 cells_after_filter=79 truncated=false bounds=51.4630,-0.0335,51.4874,-0.0010 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.6 total_ms=1.4
2026-03-15T19:59:44.863513Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=23785 parallel=false cells_before_filter=105 cells_after_filter=82 truncated=false bounds=51.4612,-0.0378,51.4857,-0.0053 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.6 total_ms=1.4
2026-03-15T19:59:45.429953Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=10363 parallel=false cells_before_filter=39 cells_after_filter=27 truncated=false bounds=51.4668,-0.0300,51.4794,-0.0133 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=0.2 total_ms=0.6
2026-03-15T19:59:47.052444Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=69 postcodes_after_filter=22 filtered_out=47 truncated=false bounds=51.469424,-0.026465,51.476579,-0.016948 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.8
2026-03-15T19:59:48.970011Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=34 postcodes_after_filter=8 filtered_out=26 truncated=false bounds=51.470330,-0.025601,51.476006,-0.018052 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.5
2026-03-15T19:59:49.463765Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=34 postcodes_after_filter=4 filtered_out=30 truncated=false bounds=51.471351,-0.024627,51.475359,-0.019295 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.3
2026-03-15T19:59:51.056899Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=34 postcodes_after_filter=9 filtered_out=25 truncated=false bounds=51.471780,-0.026331,51.475789,-0.020998 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.4
2026-03-15T19:59:51.581180Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=34 postcodes_after_filter=6 filtered_out=28 truncated=false bounds=51.471674,-0.025659,51.475683,-0.020327 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.4
2026-03-15T19:59:58.001176Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=34 postcodes_after_filter=8 filtered_out=26 truncated=false bounds=51.471124,-0.025948,51.476103,-0.019326 filters=1 filters_raw="Listing status:For sale" travel_entries=0 total_ms=0.6
2026-03-15T20:00:01.147529Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=44 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold|Leasehold" ms=0.2
2026-03-15T20:00:01.649333Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=33 postcodes_after_filter=7 filtered_out=26 truncated=false bounds=51.471124,-0.025948,51.476103,-0.019326 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold|Leasehold" travel_entries=0 total_ms=0.3
2026-03-15T20:00:03.524035Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=1 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" ms=0.2
2026-03-15T20:00:03.982950Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=5 postcodes_after_filter=1 filtered_out=4 truncated=false bounds=51.471124,-0.025948,51.476103,-0.019326 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 total_ms=0.2
2026-03-15T20:00:05.734272Z INFO property_map_server::routes::postcodes: GET /api/postcodes postcodes_before_filter=16 postcodes_after_filter=10 filtered_out=6 truncated=false bounds=51.468325,-0.029578,51.477865,-0.016889 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 total_ms=0.4
2026-03-15T20:00:07.598747Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=371660 parallel=true cells_before_filter=1032 cells_after_filter=874 truncated=false bounds=51.4033,-0.1135,51.5201,0.0418 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=2.3 total_ms=9.7
2026-03-15T20:00:07.902439Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=277117 parallel=true cells_before_filter=961 cells_after_filter=829 truncated=false bounds=51.4051,-0.0436,51.5218,0.1117 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=2.0 total_ms=9.7
2026-03-15T20:00:08.868329Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=410619 parallel=true cells_before_filter=1446 cells_after_filter=1288 truncated=false bounds=51.3867,-0.0636,51.5340,0.1322 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=2.5 total_ms=14.6
2026-03-15T20:00:10.739310Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=38 filters=2 filters_raw="Listing status:Historical sale;;Leasehold/Freehold:Freehold" ms=0.2
2026-03-15T20:00:11.236022Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=410619 parallel=true cells_before_filter=2260 cells_after_filter=1984 truncated=false bounds=51.3867,-0.0636,51.5340,0.1322 filters=2 filters_raw="Listing status:Historical sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=5.3 total_ms=38.1
2026-03-15T20:00:12.173963Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=1 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" ms=0.2
2026-03-15T20:00:12.666440Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=410619 parallel=true cells_before_filter=1446 cells_after_filter=1288 truncated=false bounds=51.3867,-0.0636,51.5340,0.1322 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=2.6 total_ms=15.7
2026-03-15T20:00:19.409064Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=89194ad04a7ffff resolution=9 total_count=0 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" ms=0.2
2026-03-15T20:00:19.968653Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=571405 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.3867,-0.1106,51.5340,0.1792 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=2.2 total_ms=2.2
2026-03-15T20:00:20.722189Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=571405 parallel=true cells_before_filter=1971 cells_after_filter=1801 truncated=false bounds=51.3867,-0.1106,51.5340,0.1792 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=3.7 total_ms=6.2
2026-03-15T20:00:21.190329Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=571405 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.3867,-0.1106,51.5340,0.1792 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=2.3 total_ms=2.3
2026-03-15T20:00:27.942711Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=571405 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.3867,-0.1106,51.5340,0.1792 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:9200" travel_entries=0 agg_ms=2.1 total_ms=2.1
2026-03-15T20:00:29.461340Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=525321 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.4145,-0.0678,51.5616,0.2220 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:9200" travel_entries=0 agg_ms=1.9 total_ms=1.9
2026-03-15T20:00:31.709909Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=525321 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.4145,-0.0678,51.5616,0.2220 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:9200" travel_entries=0 agg_ms=2.1 total_ms=2.1
2026-03-15T20:00:32.512895Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=525321 parallel=true cells_before_filter=1836 cells_after_filter=1678 truncated=false bounds=51.4145,-0.0678,51.5616,0.2220 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=3.2 total_ms=5.6
2026-03-15T20:00:33.940425Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=525321 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.4145,-0.0678,51.5616,0.2220 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1700" travel_entries=0 agg_ms=1.6 total_ms=1.6
2026-03-15T20:00:34.849329Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3141817 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.1389,-0.4994,51.7648,0.7321 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1700" travel_entries=0 agg_ms=14.9 total_ms=14.9
2026-03-15T20:00:35.804047Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3560390 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.0590,-0.6228,51.8363,0.9068 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1700" travel_entries=0 agg_ms=9.3 total_ms=9.3
2026-03-15T20:00:36.131775Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3610110 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.1661,-0.6093,51.9416,0.9202 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1700" travel_entries=0 agg_ms=9.3 total_ms=9.3
2026-03-15T20:00:37.313585Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3610110 parallel=true cells_before_filter=1788 cells_after_filter=1780 truncated=false bounds=51.1661,-0.6093,51.9416,0.9202 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=11.8 total_ms=14.2
2026-03-15T20:00:40.426451Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3610110 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.1661,-0.6093,51.9416,0.9202 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=10.8 total_ms=10.8
2026-03-15T20:00:42.361059Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=9.4 total_ms=9.4
2026-03-15T20:00:43.409927Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1193 cells_after_filter=1182 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=10.5 total_ms=12.0
2026-03-15T20:00:46.333117Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=8.8 total_ms=8.8
2026-03-15T20:00:48.082494Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=8.4 total_ms=8.4
2026-03-15T20:00:49.833842Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1193 cells_after_filter=1182 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=10.1 total_ms=23.3
2026-03-15T20:01:02.011997Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1094 cells_after_filter=1088 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=11.0 total_ms=21.9
2026-03-15T20:01:03.418647Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1193 cells_after_filter=1182 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=2 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold" travel_entries=0 agg_ms=11.4 total_ms=13.1
2026-03-15T20:01:03.571497Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1094 cells_after_filter=1088 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=11.2 total_ms=12.9
2026-03-15T20:01:08.874337Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1054 cells_after_filter=1048 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204" travel_entries=0 agg_ms=12.2 total_ms=23.6
2026-03-15T20:01:14.467687Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=9.8 total_ms=9.8
2026-03-15T20:01:15.872578Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=1054 cells_after_filter=1048 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204" travel_entries=0 agg_ms=12.0 total_ms=13.6
2026-03-15T20:01:16.342009Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=10.0 total_ms=10.0
2026-03-15T20:01:18.114175Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4515,51.8453,0.7617 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=10.2 total_ms=10.2
2026-03-15T20:01:40.296993Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3094495 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2300,-0.4517,51.8453,0.7619 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=9.8 total_ms=9.8
2026-03-15T20:02:48.056601Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3111647 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2295,-0.4528,51.8458,0.7630 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=9.6 total_ms=9.6
2026-03-15T20:02:49.785245Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Est. price per sqm:20:918535" travel_entries=0 agg_ms=9.4 total_ms=9.4
2026-03-15T20:03:02.786058Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=1285 cells_after_filter=1278 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204" travel_entries=0 agg_ms=13.1 total_ms=26.2
2026-03-15T20:03:23.742245Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=4 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Total floor area (sqm):0.1:204;;Price per sqm:5:1519231" travel_entries=0 agg_ms=10.0 total_ms=10.0
2026-03-15T20:03:30.904544Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=3 filters_raw="Listing status:For sale;;Leasehold/Freehold:Freehold;;Price per sqm:5:1519231" travel_entries=0 agg_ms=9.0 total_ms=9.0
2026-03-15T20:03:31.700115Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=2 filters_raw="Listing status:For sale;;Price per sqm:5:1519231" travel_entries=0 agg_ms=9.5 total_ms=9.5
2026-03-15T20:03:33.123759Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=2 filters_raw="Listing status:For sale;;Price per sqm:5:1519231" travel_entries=0 agg_ms=10.7 total_ms=10.7
2026-03-15T20:03:33.264092Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=2 filters_raw="Listing status:For sale;;Price per sqm:5:2500" travel_entries=0 agg_ms=8.3 total_ms=8.3
2026-03-15T20:03:34.235435Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=1479 cells_after_filter=1465 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=10.7 total_ms=12.9
2026-03-15T20:03:34.567324Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=9.0 total_ms=9.0
2026-03-15T20:03:36.096362Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=3349513 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=51.2195,-0.5956,51.8557,0.9057 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=7.6 total_ms=7.6
2026-03-15T20:03:37.751074Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=7 rows=4814137 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=50.9471,-0.9801,52.1136,1.7728 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=11.8 total_ms=11.8
2026-03-15T20:03:38.835277Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=56.1 total_ms=56.1
2026-03-15T20:03:40.887729Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=49.8 total_ms=49.8
2026-03-15T20:03:42.529302Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3274 cells_after_filter=3270 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:Historical sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=84.4 total_ms=146.3
2026-03-15T20:03:44.390098Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=0 cells_after_filter=0 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Price per sqm:1400:2500" travel_entries=0 agg_ms=54.6 total_ms=54.6
2026-03-15T20:04:33.371452Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:04:33.371601Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:04:33.371608Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:04:33.457625Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:04:33.457635Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:04:35.974733Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:04:35.974742Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:04:36.398745Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:04:36.398757Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:04:36.476688Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:04:36.476699Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:04:46.252075Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:04:46.252184Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T20:04:47.610246Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T20:04:47.804418Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T20:04:48.220314Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T20:04:49.335558Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T20:04:51.577686Z INFO property_map_server::data::property: Building enum features
2026-03-15T20:04:53.025870Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T20:04:55.151481Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T20:04:55.151490Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T20:04:55.671708Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T20:04:55.671716Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T20:04:56.573495Z INFO property_map_server::data::property: Building interned strings
2026-03-15T20:05:02.540797Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T20:05:05.103322Z INFO property_map_server::data::property: Data loading complete
2026-03-15T20:05:06.794645Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T20:05:06.794655Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T20:05:06.892926Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T20:05:06.892936Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T20:05:07.310766Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T20:05:07.310799Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T20:05:07.310815Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T20:05:07.355286Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T20:05:07.468648Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T20:05:07.469230Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T20:05:07.506618Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T20:05:07.506627Z INFO property_map_server: Building POI spatial grid index
2026-03-15T20:05:07.512084Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T20:05:07.512091Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T20:05:07.525343Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T20:05:07.526115Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T20:05:07.526170Z INFO property_map_server: Place data loaded places=3474
2026-03-15T20:05:07.526181Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T20:05:07.526186Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T20:05:07.536262Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T20:05:15.228511Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T20:05:15.228521Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T20:05:15.228532Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T20:05:15.231530Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T20:05:15.273794Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T20:05:15.309107Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T20:05:15.309278Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T20:05:15.309292Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T20:05:15.446941Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T20:05:15.454051Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T20:05:15.459669Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T20:05:15.591403Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfect-postcodes.co.uk/pb
2026-03-15T20:05:15.597637Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T20:05:15.597658Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T20:05:15.597670Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T20:05:15.643613Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T20:05:15.644670Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T20:05:15.646043Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T20:05:15.653830Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T20:05:15.653847Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T20:05:15.653891Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T20:05:19.156377Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T20:05:19.156422Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T20:05:27.513039Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:05:27.517384Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:05:27.524994Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:05:27.531774Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:05:27.537007Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:05:27.542722Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:05:27.549237Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:05:27.563523Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:05:28.096739Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:05:28.096753Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:06:11.902556Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3248 cells_after_filter=3246 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=55.7 total_ms=84.8
2026-03-15T20:08:11.734568Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=56.9 total_ms=83.5
2026-03-15T20:08:12.991872Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3248 cells_after_filter=3246 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=70.2 total_ms=76.3
2026-03-15T20:08:13.507957Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=59.5 total_ms=63.9
2026-03-15T20:08:19.338486Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=74.5 total_ms=102.6
2026-03-15T20:08:20.686317Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3248 cells_after_filter=3246 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=55.0 total_ms=59.5
2026-03-15T20:08:20.922174Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=60.6 total_ms=65.2
2026-03-15T20:08:26.772910Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3082 cells_after_filter=3081 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:788" travel_entries=0 agg_ms=62.6 total_ms=89.4
2026-03-15T20:08:28.805969Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3248 cells_after_filter=3246 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=59.6 total_ms=64.4
2026-03-15T20:08:28.952002Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3082 cells_after_filter=3081 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:788" travel_entries=0 agg_ms=57.7 total_ms=62.3
2026-03-15T20:08:30.558244Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=58.2 total_ms=84.6
2026-03-15T20:08:33.038408Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3248 cells_after_filter=3246 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=56.6 total_ms=61.3
2026-03-15T20:08:33.304924Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=3113 cells_after_filter=3112 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):0.1:8855327" travel_entries=0 agg_ms=59.1 total_ms=63.7
2026-03-15T20:08:34.406141Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=13490411 parallel=true cells_before_filter=1106 cells_after_filter=1106 truncated=false bounds=49.0536,-4.4123,53.8271,6.8427 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=48.2 total_ms=55.7
2026-03-15T20:08:36.292323Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=6 rows=9738472 parallel=true cells_before_filter=765 cells_after_filter=765 truncated=false bounds=49.7628,-3.2615,52.9110,4.1398 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=36.2 total_ms=41.7
2026-03-15T20:08:39.628619Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=8 rows=2480095 parallel=true cells_before_filter=165 cells_after_filter=164 truncated=false bounds=51.3348,-0.4935,51.7042,0.3778 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=5.9 total_ms=7.2
2026-03-15T20:08:41.653852Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=18 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" ms=0.1
2026-03-15T20:08:41.915336Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=8 rows=2176649 parallel=true cells_before_filter=139 cells_after_filter=139 truncated=false bounds=51.3348,-0.3793,51.7042,0.2636 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=5.4 total_ms=6.5
2026-03-15T20:08:44.824263Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=40 cells_after_filter=37 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=1.1 total_ms=1.4
2026-03-15T20:08:45.089912Z INFO property_map_server::routes::properties: GET /api/hexagon-properties h3=88195da457fffff resolution=8 total=18 returned=18 offset=0 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" ms=0.2
2026-03-15T20:09:25.568766Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1197 cells_after_filter=1045 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=2.1 total_ms=3.8
2026-03-15T20:09:26.074762Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=40 cells_after_filter=37 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):833:8855327" travel_entries=0 agg_ms=1.2 total_ms=1.3
2026-03-15T20:09:28.666758Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=53 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:8855327" ms=0.1
2026-03-15T20:09:29.155687Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1096 cells_after_filter=963 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:8855327" travel_entries=0 agg_ms=2.6 total_ms=12.1
2026-03-15T20:09:29.978092Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1197 cells_after_filter=1045 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=2.1 total_ms=3.8
2026-03-15T20:09:30.496935Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1096 cells_after_filter=963 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:8855327" travel_entries=0 agg_ms=2.1 total_ms=3.7
2026-03-15T20:09:33.901621Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=19 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:204" ms=0.1
2026-03-15T20:09:34.385794Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:204" travel_entries=0 agg_ms=2.4 total_ms=10.3
2026-03-15T20:09:35.102671Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1197 cells_after_filter=1045 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=2.2 total_ms=3.9
2026-03-15T20:09:35.587723Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:204" travel_entries=0 agg_ms=2.2 total_ms=3.7
2026-03-15T20:09:36.148540Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=19 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):98:204" ms=0.1
2026-03-15T20:09:36.644673Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):98:204" travel_entries=0 agg_ms=2.5 total_ms=10.4
2026-03-15T20:09:37.160139Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1197 cells_after_filter=1045 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=2.2 total_ms=3.9
2026-03-15T20:09:37.683238Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):98:204" travel_entries=0 agg_ms=2.2 total_ms=3.8
2026-03-15T20:09:37.944776Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=19 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):89:204" ms=0.1
2026-03-15T20:09:37.969098Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):89:204" travel_entries=0 agg_ms=2.2 total_ms=10.4
2026-03-15T20:09:42.074237Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1197 cells_after_filter=1045 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=1 filters_raw="Listing status:For sale" travel_entries=0 agg_ms=2.1 total_ms=4.2
2026-03-15T20:09:42.532178Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):89:204" travel_entries=0 agg_ms=2.4 total_ms=4.0
2026-03-15T20:09:50.016920Z INFO property_map_server::routes::hexagon_stats: GET /api/hexagon-stats h3=88195da457fffff resolution=8 total_count=19 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:204" ms=0.1
2026-03-15T20:09:50.508364Z INFO property_map_server::routes::hexagons: GET /api/hexagons resolution=9 rows=308665 parallel=true cells_before_filter=1081 cells_after_filter=949 truncated=false bounds=51.5155,-0.2104,51.6023,-0.0591 filters=2 filters_raw="Listing status:For sale;;Total floor area (sqm):106:204" travel_entries=0 agg_ms=2.2 total_ms=10.2
2026-03-15T20:10:13.683691Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:10:13.683848Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:10:13.683854Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:10:13.750258Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:10:13.750268Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:10:16.179096Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:10:16.179106Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:10:16.456525Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:10:16.456537Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:10:16.514061Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:10:16.514070Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:10:22.681306Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:10:22.681402Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T20:10:24.012054Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T20:10:24.180353Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T20:10:24.607077Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T20:10:25.773925Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T20:10:27.857482Z INFO property_map_server::data::property: Building enum features
2026-03-15T20:10:29.227608Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T20:10:31.336600Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T20:10:31.336609Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T20:10:31.843715Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T20:10:31.843723Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T20:10:32.766778Z INFO property_map_server::data::property: Building interned strings
2026-03-15T20:10:38.715142Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T20:10:41.203246Z INFO property_map_server::data::property: Data loading complete
2026-03-15T20:10:42.829684Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T20:10:42.829695Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T20:10:42.925550Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T20:10:42.925560Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T20:10:43.323292Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T20:10:43.323313Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T20:10:43.323319Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T20:10:43.341356Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T20:10:43.453718Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T20:10:43.454297Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T20:10:43.490938Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T20:10:43.490947Z INFO property_map_server: Building POI spatial grid index
2026-03-15T20:10:43.496143Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T20:10:43.496149Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T20:10:43.496678Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T20:10:43.497419Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T20:10:43.497476Z INFO property_map_server: Place data loaded places=3474
2026-03-15T20:10:43.497486Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T20:10:43.497491Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T20:10:43.498313Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T20:10:50.089457Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T20:10:50.089468Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T20:10:50.089483Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T20:10:50.089693Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T20:10:50.134239Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T20:10:50.160720Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T20:10:50.160908Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T20:10:50.160921Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T20:10:50.220618Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T20:10:50.224719Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T20:10:50.247220Z INFO property_map_server::pocketbase: Added notes text field to PocketBase collection 'saved_searches'
2026-03-15T20:10:50.251061Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T20:10:50.257749Z INFO property_map_server::pocketbase: Added notes text field to PocketBase collection 'saved_properties'
2026-03-15T20:10:50.304219Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfect-postcodes.co.uk/pb
2026-03-15T20:10:50.308723Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T20:10:50.308749Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T20:10:50.308761Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T20:10:50.314963Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T20:10:50.315108Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T20:10:50.315266Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T20:10:50.316776Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T20:10:50.316796Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T20:10:50.316843Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T20:10:51.706624Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T20:10:51.706663Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T20:10:52.074357Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:10:52.074443Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:10:52.507461Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:10:52.508607Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:10:52.516615Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:10:52.522899Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:10:52.536710Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:10:52.541257Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:10:52.549244Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:10:52.550031Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:18:43.657651Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:18:43.657816Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:18:43.657822Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:18:43.746197Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:18:43.746208Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:18:46.373581Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:18:46.373592Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:18:46.679114Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:18:46.679124Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:18:46.747208Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:18:46.747221Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:19:00.212275Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:19:00.212447Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:19:00.212456Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:19:00.289849Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:19:00.289859Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:19:02.760385Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:19:02.760396Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:19:03.052345Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:19:03.052355Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:19:03.116051Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:19:03.116060Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:19:09.885025Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:19:09.885115Z INFO property_map_server::data::property: Feature columns from config numeric=54 enums=13 total=67
2026-03-15T20:19:11.176403Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T20:19:11.363917Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T20:19:11.771906Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T20:19:13.025850Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T20:19:15.244059Z INFO property_map_server::data::property: Building enum features
2026-03-15T20:19:16.598869Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T20:19:18.813936Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T20:19:18.813945Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T20:19:19.353132Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T20:19:19.353141Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T20:19:20.231462Z INFO property_map_server::data::property: Building interned strings
2026-03-15T20:19:26.257466Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T20:19:28.766985Z INFO property_map_server::data::property: Data loading complete
2026-03-15T20:19:30.388072Z INFO property_map_server: Property data loaded rows=15702691 features=67 enums=13
2026-03-15T20:19:30.388081Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T20:19:30.483869Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T20:19:30.483878Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T20:19:30.866157Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T20:19:30.866199Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T20:19:30.866214Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T20:19:30.902414Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T20:19:31.015549Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T20:19:31.016109Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T20:19:31.054466Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T20:19:31.054474Z INFO property_map_server: Building POI spatial grid index
2026-03-15T20:19:31.059717Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T20:19:31.059723Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T20:19:31.069876Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T20:19:31.070582Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T20:19:31.070632Z INFO property_map_server: Place data loaded places=3474
2026-03-15T20:19:31.070640Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T20:19:31.070644Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T20:19:31.079677Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T20:19:39.095251Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T20:19:39.095261Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T20:19:39.095277Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T20:19:39.185235Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T20:19:39.228620Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T20:19:39.315674Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T20:19:39.315892Z INFO property_map_server: Precomputed features response groups=9
2026-03-15T20:19:39.315908Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T20:19:39.370690Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T20:19:39.373624Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T20:19:39.378395Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T20:19:39.427367Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfect-postcodes.co.uk/pb
2026-03-15T20:19:39.430988Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T20:19:39.431004Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T20:19:39.431017Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T20:19:39.437636Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T20:19:39.437807Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T20:19:39.437966Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T20:19:39.439692Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T20:19:39.439715Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T20:19:39.439777Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T20:19:42.954025Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T20:19:42.954067Z INFO property_map_server: Server listening on 0.0.0.0:8001
2026-03-15T20:19:43.261880Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:19:43.263169Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:43.749947Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:43.751657Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:44.102549Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:44.111787Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:44.529068Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:44.529991Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:44.546765Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:44.551558Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:44.558857Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:44.565720Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:44.574121Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:44.577816Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.493074Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:50.493085Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.514758Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:50.515163Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.526484Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.530275Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:50.535884Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.542453Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:50.551188Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:20:50.556666Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:51.102224Z INFO property_map_server::routes::features: GET /api/features
2026-03-15T20:20:51.110261Z INFO property_map_server::routes::pois: GET /api/poi-categories count=74 groups=11
2026-03-15T20:21:45.349498Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:21:45.349655Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:21:45.349664Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:21:45.421388Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:21:45.421400Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:21:47.937781Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:21:47.937791Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:21:48.227331Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:21:48.227342Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:21:48.286224Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:21:48.286233Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:22:05.552980Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:31:37.445987Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:31:37.446148Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:31:37.446155Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:31:37.534049Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:31:37.534061Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:31:40.177015Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:31:40.177028Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:31:40.482618Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:31:40.482630Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:31:40.546018Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:31:40.546027Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:31:53.986877Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:31:53.987039Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:31:53.987045Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:31:54.063239Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:31:54.063248Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:31:56.648053Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:31:56.648065Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:31:56.965183Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:31:56.965194Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:31:57.027327Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:31:57.027342Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:31:59.916992Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:32:04.424692Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:32:04.424881Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:32:04.424890Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:32:04.493173Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:32:04.493184Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:32:07.324815Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:32:07.324827Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:32:07.628171Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:32:07.628182Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:32:07.697976Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:32:07.697987Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:32:09.987127Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:32:15.239857Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:32:15.240016Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:32:15.240027Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:32:15.312610Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:32:15.312619Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:32:17.889502Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:32:17.889512Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:32:18.181929Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:32:18.181939Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:32:18.245757Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:32:18.245767Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:32:23.084864Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:32:23.085017Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:32:23.085025Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:32:23.149174Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:32:23.149184Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:32:25.785485Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:32:25.785496Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:32:26.076631Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:32:26.076644Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:32:26.135954Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:32:26.135967Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:32:28.972888Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:42:17.088723Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:42:17.088899Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:42:17.088907Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:42:17.150999Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:42:17.151009Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:42:19.827707Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:42:19.827719Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:42:20.135500Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:42:20.135509Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:42:20.197192Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:42:20.197202Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:42:56.103982Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:42:56.104138Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:42:56.104143Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:42:56.204428Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:42:56.204439Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:42:58.832976Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:42:58.832987Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:42:59.132876Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:42:59.132886Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:42:59.192137Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:42:59.192148Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:43:26.892462Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T20:56:11.923543Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:56:11.923747Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:56:11.923760Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:56:12.004141Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:56:12.004153Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:56:18.202087Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:56:18.202098Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:56:18.603337Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:56:18.603351Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:56:18.676290Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:56:18.676299Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:56:51.555700Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T20:56:51.555882Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T20:56:51.555890Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T20:56:51.649380Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T20:56:51.649390Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T20:56:54.479400Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T20:56:54.479413Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T20:56:54.787050Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T20:56:54.787063Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T20:56:54.853848Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T20:56:54.853857Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T20:57:26.731834Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T21:00:33.407080Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T21:00:33.407232Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T21:00:33.407238Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T21:00:33.499072Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T21:00:33.499081Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T21:00:36.274914Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T21:00:36.274924Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T21:00:36.575562Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T21:00:36.575572Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T21:00:36.638808Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T21:00:36.638817Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T21:00:50.949722Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T21:03:11.209421Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T21:03:11.209606Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T21:03:11.209612Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T21:03:11.289498Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T21:03:11.289508Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T21:03:13.875043Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T21:03:13.875053Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T21:03:14.208971Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T21:03:14.208982Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T21:03:14.272084Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T21:03:14.272095Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T21:03:28.932912Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T21:03:28.933073Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T21:03:28.933082Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T21:03:29.004139Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T21:03:29.004150Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T21:03:31.538874Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T21:03:31.538884Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T21:03:31.819659Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T21:03:31.819669Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T21:03:31.879197Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T21:03:31.879206Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T21:03:46.003768Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T21:19:16.107508Z INFO property_map_server: Prometheus metrics initialized
2026-03-15T21:19:16.107694Z INFO property_map_server: Loading property data from /app/data/properties.parquet, /app/data/postcode.parquet, /app/data-scraped/online_listings_buy.parquet, /app/data-scraped/online_listings_rent.parquet
2026-03-15T21:19:16.107702Z INFO property_map_server::data::property: Loading postcode features from "/app/data/postcode.parquet"
2026-03-15T21:19:16.272588Z INFO property_map_server::data::property: Postcode features loaded rows=1262367
2026-03-15T21:19:16.272599Z INFO property_map_server::data::property: Loading properties from "/app/data/properties.parquet"
2026-03-15T21:19:22.944816Z INFO property_map_server::data::property: Properties joined with postcodes rows=15203381
2026-03-15T21:19:22.944828Z INFO property_map_server::data::property: Loading buy listings from "/app/data-scraped/online_listings_buy.parquet"
2026-03-15T21:19:23.587534Z INFO property_map_server::data::property: buy listings joined rows=474965
2026-03-15T21:19:23.589329Z INFO property_map_server::data::property: Loading rent listings from "/app/data-scraped/online_listings_rent.parquet"
2026-03-15T21:19:23.673638Z INFO property_map_server::data::property: rent listings joined rows=24345
2026-03-15T21:19:23.673648Z INFO property_map_server::data::property: Concatenating all data sources
2026-03-15T21:20:10.134367Z INFO property_map_server::data::property: All data sources combined properties=15203381 buy_listings=474965 rent_listings=24345 total=15702691
2026-03-15T21:20:10.173078Z INFO property_map_server::data::property: Feature columns from config numeric=55 enums=13 total=68
2026-03-15T21:20:11.640174Z INFO property_map_server::data::property: Combined data selected rows=15702691
2026-03-15T21:20:11.825306Z INFO property_map_server::data::property: Extracting numeric feature columns
2026-03-15T21:20:12.283833Z INFO property_map_server::data::property: Computing histograms for numeric features
2026-03-15T21:20:13.735551Z INFO property_map_server::data::property: Extracting string columns
2026-03-15T21:20:16.091555Z INFO property_map_server::data::property: Building enum features
2026-03-15T21:20:17.505895Z INFO property_map_server::data::property: Extracting renovation history
2026-03-15T21:20:19.730770Z INFO property_map_server::data::property: Renovation history extracted properties_with_events=1829807
2026-03-15T21:20:19.730780Z INFO property_map_server::data::property: Extracting listing features
2026-03-15T21:20:20.299294Z INFO property_map_server::data::property: Listing features extracted properties_with_features=412749
2026-03-15T21:20:20.299302Z INFO property_map_server::data::property: Sorting rows by spatial locality
2026-03-15T21:20:21.284951Z INFO property_map_server::data::property: Building interned strings
2026-03-15T21:20:27.822185Z INFO property_map_server::data::property: Transposing to row-major layout (spatially sorted, quantized to u16)
2026-03-15T21:20:30.681373Z INFO property_map_server::data::property: Data loading complete
2026-03-15T21:20:32.249224Z INFO property_map_server: Property data loaded rows=15702691 features=68 enums=13
2026-03-15T21:20:32.249237Z INFO property_map_server: Building spatial grid index (0.01° cells)
2026-03-15T21:20:32.646329Z INFO property_map_server: Precomputing H3 cells at resolution 12
2026-03-15T21:20:32.646339Z INFO property_map_server::data::property: Precomputing H3 cells at resolution 12
2026-03-15T21:20:33.084366Z INFO property_map_server::data::property: H3 precomputation complete (15702691 cells)
2026-03-15T21:20:33.084392Z INFO property_map_server: Loading POI data from /app/data/filtered_uk_pois.parquet
2026-03-15T21:20:33.084431Z INFO property_map_server::data::poi: Loading POI data from "/app/data/filtered_uk_pois.parquet"...
2026-03-15T21:20:33.115170Z INFO property_map_server::data::poi: Loaded 678242 POIs
2026-03-15T21:20:33.231940Z INFO property_map_server::data::poi: POI string columns interned category_unique=74 group_unique=11 emoji_unique=71
2026-03-15T21:20:33.232512Z INFO property_map_server::data::poi: POI data loading complete.
2026-03-15T21:20:33.270242Z INFO property_map_server: POI data loaded pois=678242
2026-03-15T21:20:33.270252Z INFO property_map_server: Building POI spatial grid index
2026-03-15T21:20:33.275905Z INFO property_map_server: Loading place data from /app/data/places.parquet
2026-03-15T21:20:33.275913Z INFO property_map_server::data::places: Loading place data from "/app/data/places.parquet"...
2026-03-15T21:20:33.280404Z INFO property_map_server::data::places: Loaded 3474 places
2026-03-15T21:20:33.281434Z INFO property_map_server::data::places: Place data loaded places=3474 types=2 with_population=71 with_city=3392
2026-03-15T21:20:33.281491Z INFO property_map_server: Place data loaded places=3474
2026-03-15T21:20:33.281502Z INFO property_map_server: Loading postcode boundaries from /app/data/postcode_boundaries
2026-03-15T21:20:33.281509Z INFO property_map_server::data::postcodes: Loading postcode boundaries from "/app/data/postcode_boundaries"
2026-03-15T21:20:33.282358Z INFO property_map_server::data::postcodes: Found GeoJSON files to process files=2361
2026-03-15T21:20:41.788648Z INFO property_map_server::data::postcodes: Postcode boundary data ready postcodes=1490140
2026-03-15T21:20:42.014614Z INFO property_map_server: Postcode boundaries loaded postcodes=1490140
2026-03-15T21:20:42.014635Z INFO property_map_server: Loading PMTiles from /app/data/uk.pmtiles
2026-03-15T21:20:42.107058Z INFO property_map_server: PMTiles loaded successfully
2026-03-15T21:20:42.150975Z INFO property_map_server: No --dist provided; static serving and OG injection disabled
2026-03-15T21:20:42.241257Z INFO property_map_server: Screenshot service configured: http://screenshot:8002
2026-03-15T21:20:42.241801Z INFO property_map_server: Precomputed features response groups=8
2026-03-15T21:20:42.241820Z INFO property_map_server: PocketBase configured: http://pocketbase:8090
2026-03-15T21:20:42.361970Z INFO property_map_server::pocketbase: PocketBase users collection already has all required fields
2026-03-15T21:20:42.370041Z INFO property_map_server::pocketbase: PocketBase collection 'saved_searches' API rules updated
2026-03-15T21:20:42.375199Z INFO property_map_server::pocketbase: PocketBase collection 'saved_properties' API rules updated
2026-03-15T21:20:42.642209Z INFO property_map_server::pocketbase: PocketBase meta.appURL set to https://perfect-postcodes.co.uk/pb
2026-03-15T21:20:42.651503Z INFO property_map_server::pocketbase: PocketBase OAuth configured on users collection
2026-03-15T21:20:42.651536Z INFO property_map_server: Gemini configured (model: gemini-3-flash-preview)
2026-03-15T21:20:42.651550Z INFO property_map_server: Loading travel time data from /app/data/travel-times
2026-03-15T21:20:42.658771Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="bicycle" destinations=2780
2026-03-15T21:20:42.658942Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="walking" destinations=350
2026-03-15T21:20:42.659104Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="car" destinations=355
2026-03-15T21:20:42.661000Z INFO property_map_server::data::travel_time: Travel time mode discovered mode="transit" destinations=1849
2026-03-15T21:20:42.661031Z INFO property_map_server: Travel time store loaded modes=4
2026-03-15T21:20:42.661088Z INFO property_map_server: Precomputed AI filters system prompt
2026-03-15T21:20:43.239746Z WARN property_map_server::pocketbase: PocketBase invites count query failed: 400 Bad Request
2026-03-15T21:20:47.481371Z INFO property_map_server: All memory pages locked (mlockall)
2026-03-15T21:20:47.481419Z INFO property_map_server: Server listening on 0.0.0.0:8001

7
server-rs/src/data/postcodes.rs
View file

@ -168,7 +168,12 @@ impl PostcodeData {
local_aabbs.push((aabb_south, aabb_west, aabb_north, aabb_east));
}
Ok::<_, anyhow::Error>((local_postcodes, local_polygons, local_centroids, local_aabbs))
Ok::<_, anyhow::Error>((
local_postcodes,
local_polygons,
local_centroids,
local_aabbs,
))
})
.collect::<Result<Vec<_>, _>>()?;

7
server-rs/src/data/property.rs
View file

@ -543,10 +543,9 @@ impl PropertyData {
listings_buy
.lazy()
.with_column(
(col("Asking price").cast(DataType::Float64)
/ col("Total floor area (sqm)"))
.round(0)
.alias("Asking price per sqm"),
(col("Asking price").cast(DataType::Float64) / col("Total floor area (sqm)"))
.round(0)
.alias("Asking price per sqm"),
)
.collect()
.context("Failed to derive Asking price per sqm")?

3
server-rs/src/pocketbase.rs
View file

@ -820,8 +820,7 @@ async fn poll_pocketbase_counts(state: &AppState) {
("type", "redeemed"),
),
] {
if let Some(total) = pb_count(&state.http_client, pb_url, &token, "invites", filter).await
{
if let Some(total) = pb_count(&state.http_client, pb_url, &token, "invites", filter).await {
gauge!(metric, labels.0 => labels.1.to_string()).set(total as f64);
}
}

2
server-rs/src/routes/ai_filters.rs
View file

@ -3,9 +3,9 @@ use std::sync::Arc;
use axum::http::StatusCode;
use axum::response::Json;
use axum::Extension;
use metrics::counter;
use serde::{Deserialize, Serialize};
use serde_json::{json, Value};
use metrics::counter;
use tracing::{info, warn};
use crate::auth::OptionalUser;

11
server-rs/src/routes/hexagons.rs
View file

@ -4,11 +4,11 @@ use axum::extract::Query;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Json};
use axum::Extension;
use metrics::histogram;
use rayon::prelude::*;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use metrics::histogram;
use tracing::info;
use crate::aggregation::Aggregator;
@ -230,8 +230,13 @@ pub async fn get_hexagons(
) {
continue;
}
let cell_id =
cell_for_row_cached(row, precomputed, h3_res, need_parent, &mut h3_cache);
let cell_id = cell_for_row_cached(
row,
precomputed,
h3_res,
need_parent,
&mut h3_cache,
);
let agg = local_groups
.entry(cell_id)
.or_insert_with(|| Aggregator::new(num_features));

2
server-rs/src/routes/postcodes.rs
View file

@ -4,10 +4,10 @@ use axum::extract::{Path, Query};
use axum::http::StatusCode;
use axum::response::{IntoResponse, Json};
use axum::Extension;
use metrics::histogram;
use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize};
use serde_json::{Map, Value};
use metrics::histogram;
use tracing::info;
use crate::aggregation::Aggregator;

4
server-rs/src/routes/stats.rs
View file

@ -67,9 +67,7 @@ enum FeatureAccum {
global_max: f32,
},
/// Enum: count occurrences per variant index.
Enum {
value_counts: Vec<u64>,
},
Enum { value_counts: Vec<u64> },
/// Feature skipped (not in field_set).
Skip,
}