has issues
This commit is contained in:
parent
2e112d7398
commit
c645b0f1d4
96 changed files with 2147083 additions and 5787 deletions
|
|
@ -30,17 +30,26 @@ EPC := $(MANUAL_DATA)/domestic-csv.zip
|
|||
ETHNICITY := $(DATA_DIR)/ethnicity_by_la.parquet
|
||||
CRIME_DIR := $(DATA_DIR)/crime
|
||||
CRIME := $(DATA_DIR)/crime_by_lsoa.parquet
|
||||
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_year_by_lsoa.parquet
|
||||
CRIME_STAMP := $(CRIME_DIR)/.downloaded
|
||||
LSOA_LOOKUP := $(DATA_DIR)/lsoa_2011_to_2021.parquet
|
||||
NOISE := $(DATA_DIR)/road_noise.parquet
|
||||
NOISE_OVERLAY_TILES := $(DATA_DIR)/noise_lden_10m.pmtiles
|
||||
CRIME_HOTSPOT_TILES := $(DATA_DIR)/crime_hotspots.pmtiles
|
||||
TREE_OVERLAY_TILES := $(DATA_DIR)/trees_outside_woodlands.pmtiles
|
||||
OFSTED := $(DATA_DIR)/ofsted.parquet
|
||||
GIAS := $(DATA_DIR)/gias.parquet
|
||||
NAPTAN := $(DATA_DIR)/naptan.parquet
|
||||
BROADBAND := $(DATA_DIR)/broadband.parquet
|
||||
CONSERVATION_AREAS := $(DATA_DIR)/conservation_areas.gpkg
|
||||
LISTED_BUILDINGS := $(DATA_DIR)/listed_buildings.gpkg
|
||||
SCHOOL_PROX := $(DATA_DIR)/school_proximity.parquet
|
||||
RENTAL := $(DATA_DIR)/rental_prices.parquet
|
||||
INSPIRE_DIR := $(DATA_DIR)/inspire
|
||||
OA_BOUNDARIES := $(DATA_DIR)/oa_boundaries.gpkg
|
||||
UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet
|
||||
PC_BOUNDARIES := $(DATA_DIR)/postcode_boundaries
|
||||
PC_BOUNDARIES_STAMP := $(PC_BOUNDARIES)/.done
|
||||
TRANSIT_DIR := $(DATA_DIR)/transit
|
||||
TRANSIT_STAMP := $(TRANSIT_DIR)/.done
|
||||
R5_NETWORK_CACHE := $(DATA_DIR)/r5-network/network.dat
|
||||
|
|
@ -73,6 +82,15 @@ MERGE_DEPS := pipeline/transform/merge.py
|
|||
PRICE_INDEX_DEPS := pipeline/transform/price_estimation/index.py pipeline/transform/price_estimation/shrinkage.py pipeline/transform/price_estimation/utils.py
|
||||
PRICE_ESTIMATE_DEPS := pipeline/transform/price_estimation/estimate.py pipeline/transform/price_estimation/knn.py pipeline/transform/price_estimation/utils.py
|
||||
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
|
||||
PC_BOUNDARIES_DEPS := pipeline/transform/postcode_boundaries/__main__.py \
|
||||
pipeline/transform/postcode_boundaries/greenspace.py \
|
||||
pipeline/transform/postcode_boundaries/inspire.py \
|
||||
pipeline/transform/postcode_boundaries/memory.py \
|
||||
pipeline/transform/postcode_boundaries/oa_boundaries.py \
|
||||
pipeline/transform/postcode_boundaries/output.py \
|
||||
pipeline/transform/postcode_boundaries/process_oa.py \
|
||||
pipeline/transform/postcode_boundaries/uprn.py \
|
||||
pipeline/transform/postcode_boundaries/voronoi.py
|
||||
CRIME_DOWNLOAD_DEPS := pipeline/download/crime.py
|
||||
INSPIRE_DOWNLOAD_DEPS := pipeline/download/inspire.py
|
||||
TRANSIT_DOWNLOAD_DEPS := pipeline/download/transit_network.py pipeline/download/transxchange2gtfs_shim.js
|
||||
|
|
@ -80,9 +98,9 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
|
|||
|
||||
# ── Phony aliases ─────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: prepare merge tiles \
|
||||
.PHONY: prepare merge tiles overlay-tiles noise-overlay-tiles crime-hotspot-tiles tree-overlay-tiles \
|
||||
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
||||
download-naptan download-pois download-grocery-retail-points download-ofsted download-broadband download-rental-prices \
|
||||
download-naptan download-pois download-grocery-retail-points download-ofsted download-gias download-broadband download-conservation-areas download-listed-buildings download-rental-prices \
|
||||
download-postcodes download-noise download-inspire download-crime \
|
||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-ofs-register download-places download-lsoa-population download-median-age download-england-boundary download-rightmove-outcodes \
|
||||
download-map-assets \
|
||||
|
|
@ -90,11 +108,15 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
|
|||
transform-school-proximity transform-tree-density \
|
||||
generate-postcode-boundaries generate-travel-times
|
||||
|
||||
prepare: $(PRICES_STAMP) download-places tiles generate-postcode-boundaries download-map-assets generate-travel-times | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
||||
prepare: $(PRICES_STAMP) download-places tiles overlay-tiles generate-postcode-boundaries download-map-assets generate-travel-times | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX)
|
||||
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
|
||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ)
|
||||
tiles: $(TILES)
|
||||
overlay-tiles: noise-overlay-tiles crime-hotspot-tiles tree-overlay-tiles
|
||||
noise-overlay-tiles: $(NOISE_OVERLAY_TILES)
|
||||
crime-hotspot-tiles: $(CRIME_HOTSPOT_TILES)
|
||||
tree-overlay-tiles: $(TREE_OVERLAY_TILES)
|
||||
download-arcgis: $(ARCGIS)
|
||||
download-price-paid: $(PRICE_PAID)
|
||||
download-deprivation: $(IOD)
|
||||
|
|
@ -105,7 +127,10 @@ download-naptan: $(NAPTAN)
|
|||
download-pois: $(POIS_RAW)
|
||||
download-grocery-retail-points: $(GROCERY_RETAIL_POINTS)
|
||||
download-ofsted: $(OFSTED)
|
||||
download-gias: $(GIAS)
|
||||
download-broadband: $(BROADBAND)
|
||||
download-conservation-areas: $(CONSERVATION_AREAS)
|
||||
download-listed-buildings: $(LISTED_BUILDINGS)
|
||||
download-postcodes: $(POSTCODES_RAW)
|
||||
download-rental-prices: $(RENTAL)
|
||||
download-noise: $(NOISE)
|
||||
|
|
@ -134,13 +159,17 @@ transform-crime: $(CRIME)
|
|||
transform-poi-proximity: $(POI_PROXIMITY)
|
||||
transform-school-proximity: $(SCHOOL_PROX)
|
||||
transform-tree-density: $(TREE_DENSITY_PC)
|
||||
generate-postcode-boundaries: $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP)
|
||||
generate-postcode-boundaries: $(PC_BOUNDARIES_STAMP)
|
||||
|
||||
$(PC_BOUNDARIES_STAMP): $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP) $(PC_BOUNDARIES_DEPS)
|
||||
@rm -f $@
|
||||
$(VALIDATE_OUTPUTS) --dir $(INSPIRE_DIR) --zip-glob "$(INSPIRE_DIR)::*.zip"
|
||||
uv run python -m pipeline.transform.postcode_boundaries \
|
||||
--uprn $(UPRN_LOOKUP) \
|
||||
--oa-boundaries $(OA_BOUNDARIES) \
|
||||
--inspire $(INSPIRE_DIR) \
|
||||
--output $(PC_BOUNDARIES)
|
||||
@touch $@
|
||||
generate-travel-times: $(ARCGIS) $(PLACES) $(PBF) download-transit-network
|
||||
@if [ -f "$(R5_NETWORK_CACHE)" ] && { [ "$(PBF)" -nt "$(R5_NETWORK_CACHE)" ] || [ "$(TRANSIT_STAMP)" -nt "$(R5_NETWORK_CACHE)" ]; }; then \
|
||||
echo "R5 inputs are newer than $(R5_NETWORK_CACHE); deleting stale cache"; \
|
||||
|
|
@ -209,15 +238,33 @@ $(GROCERY_RETAIL_POINTS):
|
|||
$(OFSTED):
|
||||
uv run python -m pipeline.download.ofsted --output $@
|
||||
|
||||
$(GIAS): pipeline/download/gias.py
|
||||
uv run python -m pipeline.download.gias --output $@
|
||||
|
||||
$(BROADBAND):
|
||||
uv run python -m pipeline.download.broadband --output $@
|
||||
|
||||
$(CONSERVATION_AREAS): pipeline/download/conservation_areas.py
|
||||
uv run python -m pipeline.download.conservation_areas --output $@
|
||||
|
||||
$(LISTED_BUILDINGS): pipeline/download/listed_buildings.py
|
||||
uv run python -m pipeline.download.listed_buildings --output $@
|
||||
|
||||
$(POSTCODES_RAW):
|
||||
uv run python -m pipeline.download.postcodes --output $@
|
||||
|
||||
$(NOISE): $(ARCGIS) pipeline/download/noise.py
|
||||
uv run python -m pipeline.download.noise --arcgis $(ARCGIS) --output $@
|
||||
|
||||
$(NOISE_OVERLAY_TILES): pipeline/transform/noise_overlay_tiles.py pipeline/download/noise.py pipeline/download/tiles.py
|
||||
uv run python -m pipeline.transform.noise_overlay_tiles --output $@ --raster-dir $(DATA_DIR)/noise_overlay_rasters --pmtiles-bin $(DATA_DIR)/pmtiles --pmtiles-version $(PMTILES_VERSION)
|
||||
|
||||
$(CRIME_HOTSPOT_TILES): $(CRIME_STAMP) pipeline/transform/crime_hotspot_tiles.py pipeline/transform/crime.py
|
||||
uv run python -m pipeline.transform.crime_hotspot_tiles --input $(CRIME_DIR) --output $@
|
||||
|
||||
$(TREE_OVERLAY_TILES): $(FR_TOW) pipeline/transform/tree_overlay_tiles.py pipeline/transform/tree_density.py
|
||||
uv run python -m pipeline.transform.tree_overlay_tiles --tow-zip $(FR_TOW) --output $@
|
||||
|
||||
$(INSPIRE_STAMP): $(INSPIRE_DOWNLOAD_DEPS)
|
||||
@rm -f $@
|
||||
uv run python -m pipeline.download.inspire --output $(INSPIRE_DIR)
|
||||
|
|
@ -272,20 +319,24 @@ $(MAP_ASSETS_STAMP): $(MAP_ASSETS_DEPS)
|
|||
|
||||
# ── Transforms ────────────────────────────────────────────────────────────────
|
||||
|
||||
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(ENGLAND_BOUNDARY)
|
||||
uv run python -m pipeline.transform.transform_poi --input $(POIS_RAW) --naptan $(NAPTAN) --boundary $(ENGLAND_BOUNDARY) --grocery-retail-points $(GROCERY_RETAIL_POINTS) --output $@
|
||||
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(GIAS) $(ENGLAND_BOUNDARY) pipeline/transform/transform_poi.py pipeline/utils/england_geometry.py
|
||||
uv run python -m pipeline.transform.transform_poi --input $(POIS_RAW) --naptan $(NAPTAN) --boundary $(ENGLAND_BOUNDARY) --grocery-retail-points $(GROCERY_RETAIL_POINTS) --gias $(GIAS) --output $@
|
||||
|
||||
$(EPC_PP): $(PRICE_PAID) $(EPC) pipeline/transform/join_epc_pp.py pipeline/utils/fuzzy_join.py
|
||||
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
||||
|
||||
$(CRIME): $(CRIME_STAMP)
|
||||
$(CRIME) $(CRIME_BY_YEAR) &: $(CRIME_STAMP) $(LSOA_LOOKUP) pipeline/transform/crime.py
|
||||
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
|
||||
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@
|
||||
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $(CRIME) --output-by-year $(CRIME_BY_YEAR) --lsoa-lookup $(LSOA_LOOKUP)
|
||||
|
||||
$(LSOA_LOOKUP): pipeline/download/lsoa_2011_to_2021.py
|
||||
uv run python -m pipeline.download.lsoa_2011_to_2021 --output $@
|
||||
$(VALIDATE_OUTPUTS) --parquet $@
|
||||
|
||||
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
|
||||
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
|
||||
|
||||
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS)
|
||||
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS) pipeline/transform/school_proximity.py pipeline/utils/poi_counts.py
|
||||
uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
|
||||
|
||||
$(TREE_DENSITY_PC): $(FR_TOW) $(ARCGIS) $(PRICE_PAID) $(TREE_DENSITY_DEPS)
|
||||
|
|
@ -315,7 +366,7 @@ $(PC_BOUNDARIES):
|
|||
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
|
||||
|
||||
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_PC) $(MERGE_DEPS)
|
||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(CONSERVATION_AREAS) $(LISTED_BUILDINGS) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_PC) $(MERGE_DEPS)
|
||||
@rm -f $@
|
||||
uv run python -m pipeline.transform.merge \
|
||||
--epc-pp $(EPC_PP) \
|
||||
|
|
@ -327,6 +378,8 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
|||
--noise $(NOISE) \
|
||||
--school-proximity $(SCHOOL_PROX) \
|
||||
--broadband $(BROADBAND) \
|
||||
--conservation-areas $(CONSERVATION_AREAS) \
|
||||
--listed-buildings $(LISTED_BUILDINGS) \
|
||||
--rental-prices $(RENTAL) \
|
||||
--lsoa-population $(LSOA_POP) \
|
||||
--median-age $(MEDIAN_AGE) \
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue