andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

lgtm

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-25 22:29:52 +01:00
parent 2efa4d9f47
commit 5e73287eaf
99 changed files with 6392 additions and 1462 deletions
Download patch file Download diff file Expand all files Collapse all files

51
Makefile.data
View file

@ -27,7 +27,10 @@ POSTCODES_RAW := $(DATA_DIR)/gb-postcodes-v5
POSTCODES_PQ := $(DATA_DIR)/postcode.parquet
PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
MERGE_STAMP := $(DATA_DIR)/.merge_done
PRICE_INPUTS := $(DATA_DIR)/price_inputs.parquet
POSTCODE_CENTROIDS := $(DATA_DIR)/postcode_centroids.parquet
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
PRICE_ESTIMATES := $(DATA_DIR)/price_estimates.parquet
PRICES_STAMP := $(DATA_DIR)/.prices_done
EPC := $(MANUAL_DATA)/domestic-csv.zip
ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet
@ -38,6 +41,8 @@ TENURE := $(DATA_DIR)/tenure_by_lsoa.parquet
CRIME_DIR := $(DATA_DIR)/crime
CRIME := $(DATA_DIR)/crime_by_postcode.parquet
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet
CRIME_RECORDS := $(DATA_DIR)/crime_records.parquet
AREA_CRIME_AVERAGES := $(DATA_DIR)/area_crime_averages.parquet
POPULATION := $(DATA_DIR)/population_by_postcode.parquet
CRIME_STAMP := $(CRIME_DIR)/.downloaded
NOISE := $(DATA_DIR)/road_noise.parquet
@ -91,8 +96,11 @@ VALIDATE_OUTPUTS := uv run python -m pipeline.validate_outputs
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
MERGE_DEPS := pipeline/transform/merge.py
AREA_CRIME_AVERAGES_DEPS := pipeline/transform/area_crime_averages.py
PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py
PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py
PRICE_JOIN_DEPS := pipeline/transform/join_price_estimates.py pipeline/transform/price_estimation/utils.py
PRICE_INPUTS_DEPS := pipeline/transform/property_base.py pipeline/utils/postcode_mapping.py
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \
pipeline/transform/postcode_boundaries/greenspace.py \
@ -116,11 +124,11 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
download-postcodes download-noise download-inspire download-crime \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \
download-map-assets \
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
transform-pois transform-epc-pp transform-crime transform-poi-proximity transform-area-crime-averages \
transform-school-catchments transform-tree-density \
generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) $(CRIME_RECORDS) $(AREA_CRIME_AVERAGES) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
@ -178,6 +186,7 @@ transform-pois: $(POIS_FILTERED)
transform-epc-pp: $(EPC_PP)
transform-crime: $(CRIME)
transform-poi-proximity: $(POI_PROXIMITY)
transform-area-crime-averages: $(AREA_CRIME_AVERAGES)
transform-school-catchments: $(SCHOOL_CATCH)
transform-tree-density: $(TREE_DENSITY_PC)
generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP)
@ -383,9 +392,9 @@ $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(GIAS) $(OFSTE
$(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
$(CRIME) $(CRIME_BY_YEAR) $(CRIME_RECORDS) &: $(CRIME_STAMP) $(PC_BOUNDARIES_STAMP) pipeline/transform/crime_spatial.py pipeline/transform/postcode_boundaries/loader.py pipeline/transform/crime.py
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR)
uv run python -m pipeline.transform.crime_spatial --input $(CRIME_DIR) --boundaries $(PC_BOUNDARIES)/units --output $(CRIME) --output-by-year $(CRIME_BY_YEAR) --output-records $(CRIME_RECORDS)
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
@ -450,16 +459,42 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
$(POSTCODES_PQ) $(PROPERTIES_PQ) &: $(MERGE_STAMP)
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
$(PRICE_INDEX): $(MERGE_STAMP) $(PRICE_INDEX_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
uv run python -m pipeline.transform.price_estimation.index --input $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --output $@
# Slim price-estimation inputs, built straight from epc_pp + arcgis (NOT the
# merge outputs). This is what decouples the expensive index/kNN from merge:
# adding an area or property column to merge does not change these files, so the
# price index and estimates are reused and only the cheap join re-runs.
$(PRICE_INPUTS) $(POSTCODE_CENTROIDS) &: $(EPC_PP) $(ARCGIS) $(PRICE_INPUTS_DEPS)
uv run python -m pipeline.transform.property_base --epc-pp $(EPC_PP) --arcgis $(ARCGIS) --output $(PRICE_INPUTS) --centroids $(POSTCODE_CENTROIDS)
$(VALIDATE_OUTPUTS) --parquet $(PRICE_INPUTS) --parquet $(POSTCODE_CENTROIDS)
$(PRICE_INDEX): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX_DEPS)
uv run python -m pipeline.transform.price_estimation.index --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --output $@
$(VALIDATE_OUTPUTS) --parquet $@
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ)
# Estimate prices on the slim inputs and write a standalone price_estimates.parquet
# (natural key + the two estimate columns). Never touches properties.parquet.
$(PRICE_ESTIMATES): $(PRICE_INPUTS) $(POSTCODE_CENTROIDS) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS)
uv run python -m pipeline.transform.price_estimation.estimate --input $(PRICE_INPUTS) --postcodes $(POSTCODE_CENTROIDS) --index $(PRICE_INDEX) --output $@
$(VALIDATE_OUTPUTS) --parquet $@
# Join the estimate columns back onto properties.parquet by natural key
# (idempotent, atomic). Re-runs whenever merge or the estimates change.
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_ESTIMATES) $(PRICE_JOIN_DEPS) | $(PROPERTIES_PQ) $(POSTCODES_PQ) $(PRICE_INDEX)
@rm -f $@
uv run python -m pipeline.transform.price_estimation.estimate --properties $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --index $(PRICE_INDEX)
uv run python -m pipeline.transform.join_price_estimates --properties $(PROPERTIES_PQ) --estimates $(PRICE_ESTIMATES)
$(VALIDATE_OUTPUTS) --parquet $(PROPERTIES_PQ) --parquet $(POSTCODES_PQ) --parquet $(PRICE_INDEX) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
@touch $@
# ── Area crime averages (post-merge) ────────────────────
# National / per-outcode / per-sector property-weighted mean headline crime
# rates for the right pane. Depends on $(PRICES_STAMP) so it reads the
# finalised properties.parquet (the price-estimate join rewrites it); only the
# per-postcode property counts and the crime values from postcode.parquet are
# used, both unaffected by that join.
$(AREA_CRIME_AVERAGES): $(PRICES_STAMP) $(AREA_CRIME_AVERAGES_DEPS) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
uv run python -m pipeline.transform.area_crime_averages --postcodes $(POSTCODES_PQ) --properties $(PROPERTIES_PQ) --output $@
$(VALIDATE_OUTPUTS) --parquet $@
$(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
$(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \