New videos
This commit is contained in:
parent
a02c8f7849
commit
6c6780fc60
52 changed files with 414 additions and 74 deletions
|
|
@ -33,15 +33,19 @@ EPC := $(MANUAL_DATA)/domestic-csv.zip
|
|||
ACTUAL_LISTINGS_RAW := $(FINDER_DATA)/online_listings_buy.parquet
|
||||
ACTUAL_LISTINGS_ENRICHED := $(FINDER_DATA)/online_listings_buy_enriched.parquet
|
||||
ETHNICITY := $(DATA_DIR)/ethnicity_by_lsoa.parquet
|
||||
EDUCATION := $(DATA_DIR)/education_by_lsoa.parquet
|
||||
TENURE := $(DATA_DIR)/tenure_by_lsoa.parquet
|
||||
CRIME_DIR := $(DATA_DIR)/crime
|
||||
CRIME := $(DATA_DIR)/crime_by_postcode.parquet
|
||||
CRIME_BY_YEAR := $(DATA_DIR)/crime_by_postcode_by_year.parquet
|
||||
POPULATION := $(DATA_DIR)/population_by_postcode.parquet
|
||||
CRIME_STAMP := $(CRIME_DIR)/.downloaded
|
||||
NOISE := $(DATA_DIR)/road_noise.parquet
|
||||
NOISE_OVERLAY_TILES := $(DATA_DIR)/noise_lden_10m.pmtiles
|
||||
CRIME_HOTSPOT_TILES := $(DATA_DIR)/crime_hotspots.pmtiles
|
||||
TREE_OVERLAY_TILES := $(DATA_DIR)/trees_outside_woodlands.pmtiles
|
||||
PROPERTY_BORDER_TILES := $(DATA_DIR)/property_borders.pmtiles
|
||||
DEVELOPMENT_SITES := $(DATA_DIR)/development_sites.parquet
|
||||
OFSTED := $(DATA_DIR)/ofsted.parquet
|
||||
GIAS := $(DATA_DIR)/gias.parquet
|
||||
NAPTAN := $(DATA_DIR)/naptan.parquet
|
||||
|
|
@ -107,16 +111,16 @@ MAP_ASSETS_DEPS := pipeline/download/map_assets.py pipeline/transform/transform_
|
|||
# ── Phony aliases ─────────────────────────────────────────────────────────────
|
||||
|
||||
.PHONY: prepare merge tiles satellite-tiles satellite-highres-tiles overlay-tiles noise-overlay-tiles crime-hotspot-tiles tree-overlay-tiles property-border-tiles \
|
||||
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
||||
download-arcgis download-price-paid download-deprivation download-ethnicity download-education download-tenure \
|
||||
download-naptan download-pois download-grocery-retail-points download-ofsted download-gias download-lsoa-children download-broadband download-conservation-areas download-listed-buildings download-rental-prices \
|
||||
download-postcodes download-noise download-inspire download-crime \
|
||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-england-boundary download-rightmove-outcodes \
|
||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-nfi download-ofs-register download-places download-median-age download-population download-england-boundary download-rightmove-outcodes \
|
||||
download-map-assets \
|
||||
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
||||
transform-school-catchments transform-tree-density \
|
||||
generate-postcode-boundaries generate-travel-times enrich-actual-listings
|
||||
generate-postcode-boundaries generate-travel-times enrich-actual-listings download-development-sites
|
||||
|
||||
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
||||
prepare: $(PRICES_STAMP) download-places tiles satellite-tiles overlay-tiles property-border-tiles tree-overlay-tiles crime-hotspot-tiles property-border-tiles generate-postcode-boundaries download-map-assets generate-travel-times $(DEVELOPMENT_SITES) $(POPULATION) | $(POSTCODES_PQ) $(PROPERTIES_PQ) $(PRICE_INDEX)
|
||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --parquet $(PRICE_INDEX) --postcode-boundary-match "$(POSTCODES_PQ)::$(PC_BOUNDARIES)" --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)" --price-index $(PRICE_INDEX)
|
||||
merge: $(MERGE_STAMP) | $(POSTCODES_PQ) $(PROPERTIES_PQ)
|
||||
$(VALIDATE_OUTPUTS) --parquet $(POSTCODES_PQ) --parquet $(PROPERTIES_PQ) --postcode-features $(POSTCODES_PQ) --postcode-universe "$(ARCGIS)::$(POSTCODES_PQ)" --properties-subset "$(PROPERTIES_PQ)::$(POSTCODES_PQ)"
|
||||
|
|
@ -133,6 +137,8 @@ download-arcgis: $(ARCGIS)
|
|||
download-price-paid: $(PRICE_PAID)
|
||||
download-deprivation: $(IOD)
|
||||
download-ethnicity: $(ETHNICITY)
|
||||
download-education: $(EDUCATION)
|
||||
download-tenure: $(TENURE)
|
||||
download-crime: $(CRIME_STAMP)
|
||||
$(VALIDATE_OUTPUTS) --file $(CRIME_DIR)/archive_manifest.json --glob "$(CRIME_DIR)::**/*-street.csv"
|
||||
download-naptan: $(NAPTAN)
|
||||
|
|
@ -143,6 +149,7 @@ download-gias: $(GIAS)
|
|||
download-lsoa-children: $(LSOA_CHILDREN)
|
||||
download-broadband: $(BROADBAND)
|
||||
download-conservation-areas: $(CONSERVATION_AREAS)
|
||||
download-development-sites: $(DEVELOPMENT_SITES)
|
||||
download-listed-buildings: $(LISTED_BUILDINGS)
|
||||
download-postcodes: $(POSTCODES_RAW)
|
||||
download-rental-prices: $(RENTAL)
|
||||
|
|
@ -161,6 +168,7 @@ download-nfi: $(NFI)
|
|||
download-ofs-register: $(OFS_REGISTER)
|
||||
download-places: $(PLACES)
|
||||
download-median-age: $(MEDIAN_AGE)
|
||||
download-population: $(POPULATION)
|
||||
download-election-results: $(ELECTION)
|
||||
download-england-boundary: $(ENGLAND_BOUNDARY)
|
||||
download-rightmove-outcodes: $(RM_OUTCODES)
|
||||
|
|
@ -235,6 +243,12 @@ $(IOD):
|
|||
$(ETHNICITY): pipeline/download/ethnicity.py
|
||||
uv run python -m pipeline.download.ethnicity --output $@
|
||||
|
||||
$(EDUCATION): pipeline/download/education.py
|
||||
uv run python -m pipeline.download.education --output $@
|
||||
|
||||
$(TENURE): pipeline/download/tenure.py
|
||||
uv run python -m pipeline.download.tenure --output $@
|
||||
|
||||
$(CRIME_STAMP): $(CRIME_DOWNLOAD_DEPS)
|
||||
@rm -f $@
|
||||
uv run python -m pipeline.download.crime --output $(CRIME_DIR)
|
||||
|
|
@ -285,6 +299,9 @@ $(BROADBAND):
|
|||
$(CONSERVATION_AREAS): pipeline/download/conservation_areas.py
|
||||
uv run python -m pipeline.download.conservation_areas --output $@
|
||||
|
||||
$(DEVELOPMENT_SITES): pipeline/download/development_sites.py
|
||||
uv run python -m pipeline.download.development_sites --output $@
|
||||
|
||||
$(LISTED_BUILDINGS): pipeline/download/listed_buildings.py
|
||||
uv run python -m pipeline.download.listed_buildings --output $@
|
||||
|
||||
|
|
@ -340,6 +357,9 @@ $(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN) $(OFS_REGISTER) $(ARCGIS) $(POIS
|
|||
$(MEDIAN_AGE):
|
||||
uv run python -m pipeline.download.median_age --output $@
|
||||
|
||||
$(POPULATION): pipeline/download/census_population.py
|
||||
uv run python -m pipeline.download.census_population --output $@
|
||||
|
||||
$(ELECTION): pipeline/download/election_results.py
|
||||
uv run python -m pipeline.download.election_results --output $@
|
||||
|
||||
|
|
@ -400,7 +420,7 @@ $(PC_BOUNDARIES):
|
|||
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
|
||||
|
||||
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) $(CONSERVATION_AREAS) $(LISTED_BUILDINGS) $(RENTAL) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_PC) $(MERGE_DEPS)
|
||||
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) $(CONSERVATION_AREAS) $(LISTED_BUILDINGS) $(RENTAL) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_PC) $(MERGE_DEPS)
|
||||
@rm -f $@
|
||||
uv run python -m pipeline.transform.merge \
|
||||
--epc-pp $(EPC_PP) \
|
||||
|
|
@ -408,6 +428,8 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
|||
--iod $(IOD) \
|
||||
--poi-proximity $(POI_PROXIMITY) \
|
||||
--ethnicity $(ETHNICITY) \
|
||||
--education $(EDUCATION) \
|
||||
--tenure $(TENURE) \
|
||||
--crime $(CRIME) \
|
||||
--noise $(NOISE) \
|
||||
--school-catchments $(SCHOOL_CATCH) \
|
||||
|
|
@ -440,7 +462,7 @@ $(PRICES_STAMP): $(MERGE_STAMP) $(PRICE_INDEX) $(PRICE_ESTIMATE_DEPS) | $(PROPER
|
|||
|
||||
$(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
|
||||
$(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \
|
||||
$(ETHNICITY) $(EDUCATION) $(TENURE) $(CRIME) $(NOISE) $(SCHOOL_CATCH) $(BROADBAND) \
|
||||
$(CONSERVATION_AREAS) $(LISTED_BUILDINGS) $(RENTAL) \
|
||||
$(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_PC) \
|
||||
$(MERGE_DEPS) pipeline/utils/fuzzy_join.py
|
||||
|
|
@ -450,6 +472,8 @@ $(ACTUAL_LISTINGS_ENRICHED): $(ACTUAL_LISTINGS_RAW) $(EPC) \
|
|||
--iod $(IOD) \
|
||||
--poi-proximity $(POI_PROXIMITY) \
|
||||
--ethnicity $(ETHNICITY) \
|
||||
--education $(EDUCATION) \
|
||||
--tenure $(TENURE) \
|
||||
--crime $(CRIME) \
|
||||
--noise $(NOISE) \
|
||||
--school-catchments $(SCHOOL_CATCH) \
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue