andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

changes

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-18 21:22:15 +00:00
parent 524580eb25
commit ffe080adef
82 changed files with 2652 additions and 2956 deletions
Download patch file Download diff file Expand all files Collapse all files

46
Makefile.data
View file

@ -1,7 +1,7 @@
# Data pipeline — download sources and build wide.parquet
# Data pipeline — download sources and build postcode.parquet + properties.parquet
#
# Usage:
# make -f Makefile.data prepare # Build wide.parquet (+ all deps)
# make -f Makefile.data prepare # Build all parquets (+ all deps)
# make -f Makefile.data tiles # Download UK map tiles
#
# Or include from the main Makefile and use targets directly.
@ -22,7 +22,9 @@ POIS_RAW := $(DATA_DIR)/uk_pois.parquet
POIS_FILTERED := $(DATA_DIR)/filtered_uk_pois.parquet
POI_PROXIMITY := $(DATA_DIR)/poi_proximity.parquet
EPC_PP := $(DATA_DIR)/epc_pp.parquet
WIDE := $(DATA_DIR)/wide.parquet
POSTCODES_PQ := $(DATA_DIR)/postcode.parquet
PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
MERGE_STAMP := $(DATA_DIR)/.merge_done
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
PRICES_STAMP := $(DATA_DIR)/.prices_done
EPC := $(MANUAL_DATA)/certificates.csv
@ -48,9 +50,8 @@ TRANSIT_STAMP := $(TRANSIT_DIR)/.done
GREENSPACE := $(DATA_DIR)/greenspace_water.parquet
PBF := $(DATA_DIR)/great-britain-latest.osm.pbf
PLACES := $(DATA_DIR)/places.parquet
RIGHTMOVE_BUY := $(DATA_DIR)/rightmove_buy.parquet
RIGHTMOVE_RENT := $(DATA_DIR)/rightmove_rent.parquet
ONLINE_STAMP := $(DATA_DIR)/.online_done
LISTINGS_BUY := $(DATA_DIR)/online_listings_buy.parquet
LISTINGS_RENT := $(DATA_DIR)/online_listings_rent.parquet
# Sentinel files for directory targets (Make doesn't track directories well)
GEOSURE_STAMP := $(GEOSURE_DIR)/.done
@ -60,7 +61,7 @@ PMTILES_VERSION := 1.22.3
# ── Phony aliases ─────────────────────────────────────────────────────────────
.PHONY: prepare wide tiles \
.PHONY: prepare merge tiles \
download-arcgis download-price-paid download-deprivation download-ethnicity \
download-naptan download-pois download-ofsted download-broadband download-rental-prices \
download-postcodes download-geosure download-noise download-inspire \
@ -70,8 +71,8 @@ PMTILES_VERSION := 1.22.3
generate-postcode-boundaries \
journey-times
prepare: $(DATA_DIR)/.prices_done
wide: $(WIDE)
prepare: $(PRICES_STAMP)
merge: $(MERGE_STAMP)
tiles: $(TILES)
download-arcgis: $(ARCGIS)
download-price-paid: $(PRICE_PAID)
@ -253,10 +254,10 @@ $(PC_BOUNDARIES):
@echo ""
@exit 1
# ── Final merge ───────────────────────────────────────────────────────────────
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
$(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) \
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE) $(RENTAL)
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA) \
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(GEOSURE) $(RENTAL)
uv run python -m pipeline.transform.merge \
--epc-pp $(EPC_PP) \
--arcgis $(ARCGIS) \
@ -271,22 +272,15 @@ $(WIDE): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) $(JT_BANK) $(JT_FITZROVIA)
--broadband $(BROADBAND) \
--geosure $(GEOSURE) \
--rental-prices $(RENTAL) \
--output $@
# ── Online listings (post-merge, pre-pricing) ───────────────────────────────
$(ONLINE_STAMP): $(WIDE) $(RIGHTMOVE_BUY) $(RIGHTMOVE_RENT)
uv run python -m pipeline.transform.add_online_listings \
--input $(WIDE) \
--buy $(RIGHTMOVE_BUY) \
--rent $(RIGHTMOVE_RENT)
--output-postcodes $(POSTCODES_PQ) \
--output-properties $(PROPERTIES_PQ)
@touch $@
# ── Price estimation (post-merge + online) ──────────────────────────────────
# ── Price estimation (post-merge) ───────────────────────────────────────────
$(PRICE_INDEX): $(ONLINE_STAMP)
uv run python -m pipeline.transform.price_estimation.index --input $(WIDE) --output $@
$(PRICE_INDEX): $(MERGE_STAMP)
uv run python -m pipeline.transform.price_estimation.index --input $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --output $@
$(PRICES_STAMP): $(ONLINE_STAMP) $(PRICE_INDEX)
uv run python -m pipeline.transform.price_estimation.estimate --input $(WIDE) --index $(PRICE_INDEX)
$(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX)
uv run python -m pipeline.transform.price_estimation.estimate --properties $(PROPERTIES_PQ) --postcodes $(POSTCODES_PQ) --index $(PRICE_INDEX)
@touch $@