andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

lgtm

Browse source
This commit is contained in:
Andras Schmelczer 2026-05-12 22:00:56 +01:00
parent 8708bf000d
commit 11711c57e6
38 changed files with 5361 additions and 265 deletions
Download patch file Download diff file Expand all files Collapse all files

37
Makefile.data
View file

@ -48,6 +48,10 @@ GREENSPACE := $(DATA_DIR)/greenspace_water.parquet
OS_GREENSPACE := $(DATA_DIR)/os_greenspace.parquet
PBF := $(DATA_DIR)/england-latest.osm.pbf
FR_TOW := $(DATA_DIR)/FR_TOW_V1_ALL.zip
TREE_DENSITY_PC := $(DATA_DIR)/tree_density_by_postcode.parquet
TREE_DENSITY_STREETS := $(DATA_DIR)/tree_density_by_street.parquet
TREE_DENSITY_ADDR := $(DATA_DIR)/tree_density_by_address.parquet
OFS_REGISTER := $(DATA_DIR)/ofs_register.xlsx
PLACES := $(DATA_DIR)/places.parquet
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
MEDIAN_AGE := $(DATA_DIR)/median_age.parquet
@ -62,16 +66,20 @@ MAP_ASSETS_STAMP := $(MAP_ASSETS_DIR)/.done
PMTILES_VERSION := 1.22.3
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
MERGE_DEPS := pipeline/transform/merge.py
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
# ── Phony aliases ─────────────────────────────────────────────────────────────
.PHONY: prepare merge tiles \
download-arcgis download-price-paid download-deprivation download-ethnicity \
download-naptan download-pois download-grocery-retail-points download-ofsted download-broadband download-rental-prices \
download-postcodes download-noise download-inspire download-crime \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-places download-lsoa-population download-median-age download-england-boundary download-rightmove-outcodes \
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-ofs-register download-places download-lsoa-population download-median-age download-england-boundary download-rightmove-outcodes \
download-map-assets \
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
transform-school-proximity \
transform-school-proximity transform-tree-density \
generate-postcode-boundaries generate-travel-times
prepare: $(PRICES_STAMP) download-places tiles generate-postcode-boundaries download-map-assets generate-travel-times
@ -98,6 +106,7 @@ download-greenspace: $(GREENSPACE)
download-os-greenspace: $(OS_GREENSPACE)
download-pbf: $(PBF)
download-fr-tow: $(FR_TOW)
download-ofs-register: $(OFS_REGISTER)
download-places: $(PLACES)
download-lsoa-population: $(LSOA_POP)
download-median-age: $(MEDIAN_AGE)
@ -110,6 +119,7 @@ transform-epc-pp: $(EPC_PP)
transform-crime: $(CRIME)
transform-poi-proximity: $(POI_PROXIMITY)
transform-school-proximity: $(SCHOOL_PROX)
transform-tree-density: $(TREE_DENSITY_ADDR)
generate-postcode-boundaries: $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP)
uv run python -m pipeline.transform.postcode_boundaries \
--uprn $(UPRN_LOOKUP) \
@ -168,6 +178,11 @@ $(FR_TOW):
curl -L -A "Mozilla/5.0" -o $@.tmp "https://www.mediafire.com/file_premium/p5fve6wswwwjqrq/FR_TOW_V1_ALL.zip/file"
mv $@.tmp $@
$(OFS_REGISTER):
@mkdir -p $(DATA_DIR)
curl -fL -A "Mozilla/5.0" -o $@.tmp https://register-api.officeforstudents.org.uk/api/Download/
mv $@.tmp $@
$(POIS_RAW): $(PBF) $(ENGLAND_BOUNDARY)
uv run python -m pipeline.download.pois --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY)
@ -209,8 +224,8 @@ $(GREENSPACE): $(PBF)
$(OS_GREENSPACE):
uv run python -m pipeline.download.os_greenspace --output $@
$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN)
uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN)
$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN) $(OFS_REGISTER) $(ARCGIS)
uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN) --university-register $(OFS_REGISTER) --postcodes $(ARCGIS)
$(LSOA_POP):
uv run python -m pipeline.download.lsoa_population --output $@
@ -243,12 +258,21 @@ $(EPC_PP): $(PRICE_PAID) $(EPC)
$(CRIME): $(CRIME_STAMP)
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE)
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS)
uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
$(TREE_DENSITY_ADDR): $(FR_TOW) $(ARCGIS) $(PRICE_PAID) $(TREE_DENSITY_DEPS)
uv run python -m pipeline.transform.tree_density \
--tow-zip $(FR_TOW) \
--arcgis $(ARCGIS) \
--price-paid $(PRICE_PAID) \
--output-postcodes $(TREE_DENSITY_PC) \
--output-streets $(TREE_DENSITY_STREETS) \
--output-addresses $@
# Postcode boundaries require manual generation — fail with instructions
$(PC_BOUNDARIES):
@echo ""
@ -267,7 +291,7 @@ $(PC_BOUNDARIES):
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION)
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_ADDR) $(MERGE_DEPS)
uv run python -m pipeline.transform.merge \
--epc-pp $(EPC_PP) \
--arcgis $(ARCGIS) \
@ -282,6 +306,7 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
--lsoa-population $(LSOA_POP) \
--median-age $(MEDIAN_AGE) \
--election-results $(ELECTION) \
--tree-density-addresses $(TREE_DENSITY_ADDR) \
--output-postcodes $(POSTCODES_PQ) \
--output-properties $(PROPERTIES_PQ)
@touch $@