Compare commits
16 commits
be02fc16bb
...
a08b5d2ae0
| Author | SHA1 | Date | |
|---|---|---|---|
| a08b5d2ae0 | |||
| bd6b511f16 | |||
| 63713c3a2b | |||
| 81a16f543c | |||
| 11711c57e6 | |||
| 8708bf000d | |||
| a9e5a8ad96 | |||
| 1483dc5224 | |||
| 7ca29c2d81 | |||
| b580c51b6d | |||
| f2a2651b8a | |||
| 9248e26af2 | |||
| a5f2e6eabb | |||
| ee231d2ee5 | |||
| 3debacab4f | |||
| fc10381692 |
1036 changed files with 44926 additions and 3079 deletions
|
|
@ -12,3 +12,4 @@ analyses/
|
||||||
property-data
|
property-data
|
||||||
manual-data
|
manual-data
|
||||||
!property-data/arcgis_data.parquet
|
!property-data/arcgis_data.parquet
|
||||||
|
Dockerfile
|
||||||
|
|
|
||||||
|
|
@ -17,12 +17,6 @@ jobs:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Install uv
|
|
||||||
run: curl -LsSf https://astral.sh/uv/install.sh | sh
|
|
||||||
|
|
||||||
- name: Download map assets (fonts, sprites, twemoji)
|
|
||||||
run: uv run python -m pipeline.download.map_assets --output frontend/public/assets
|
|
||||||
|
|
||||||
- name: Install Docker CLI
|
- name: Install Docker CLI
|
||||||
run: |
|
run: |
|
||||||
ARCH=$(uname -m)
|
ARCH=$(uname -m)
|
||||||
|
|
@ -84,4 +78,3 @@ jobs:
|
||||||
${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:sha-${{ steps.tags.outputs.sha_short }}
|
${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:sha-${{ steps.tags.outputs.sha_short }}
|
||||||
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:buildcache
|
cache-from: type=registry,ref=${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:buildcache
|
||||||
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:buildcache,mode=max
|
cache-to: type=registry,ref=${{ env.REGISTRY }}/${{ steps.tags.outputs.repo }}-screenshot:buildcache,mode=max
|
||||||
|
|
||||||
|
|
|
||||||
8
.gitignore
vendored
8
.gitignore
vendored
|
|
@ -6,6 +6,14 @@
|
||||||
server-rs/target
|
server-rs/target
|
||||||
.task
|
.task
|
||||||
frontend/public/assets/*
|
frontend/public/assets/*
|
||||||
|
!frontend/public/assets/fonts/
|
||||||
|
!frontend/public/assets/fonts/**
|
||||||
|
!frontend/public/assets/sprites/
|
||||||
|
!frontend/public/assets/sprites/**
|
||||||
|
!frontend/public/assets/twemoji/
|
||||||
|
!frontend/public/assets/twemoji/**
|
||||||
!frontend/public/assets/poi-icons/
|
!frontend/public/assets/poi-icons/
|
||||||
!frontend/public/assets/poi-icons/**
|
!frontend/public/assets/poi-icons/**
|
||||||
|
frontend/public/assets/.done
|
||||||
server-rs/logs
|
server-rs/logs
|
||||||
|
video/auth.*
|
||||||
|
|
|
||||||
3
.vscode/settings.json
vendored
3
.vscode/settings.json
vendored
|
|
@ -8,5 +8,6 @@
|
||||||
"**/target": true,
|
"**/target": true,
|
||||||
"frontend/dist": true,
|
"frontend/dist": true,
|
||||||
"**/.task": true
|
"**/.task": true
|
||||||
}
|
},
|
||||||
|
"python.terminal.activateEnvironment": false
|
||||||
}
|
}
|
||||||
|
|
@ -1,8 +1,5 @@
|
||||||
# Data pipeline — download sources and build postcode.parquet + properties.parquet
|
|
||||||
#
|
|
||||||
# Usage:
|
# Usage:
|
||||||
# make -f Makefile.data prepare # Build all parquets (+ all deps)
|
# make -f Makefile.data prepare # Build all parquets (+ all deps)
|
||||||
# make -f Makefile.data tiles # Download UK map tiles
|
|
||||||
#
|
#
|
||||||
# Or include from the main Makefile and use targets directly.
|
# Or include from the main Makefile and use targets directly.
|
||||||
|
|
||||||
|
|
@ -29,10 +26,11 @@ PROPERTIES_PQ := $(DATA_DIR)/properties.parquet
|
||||||
MERGE_STAMP := $(DATA_DIR)/.merge_done
|
MERGE_STAMP := $(DATA_DIR)/.merge_done
|
||||||
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
|
PRICE_INDEX := $(DATA_DIR)/price_index.parquet
|
||||||
PRICES_STAMP := $(DATA_DIR)/.prices_done
|
PRICES_STAMP := $(DATA_DIR)/.prices_done
|
||||||
EPC := $(MANUAL_DATA)/certificates.csv
|
EPC := $(MANUAL_DATA)/domestic-csv.zip
|
||||||
ETHNICITY := $(DATA_DIR)/ethnicity_by_la.parquet
|
ETHNICITY := $(DATA_DIR)/ethnicity_by_la.parquet
|
||||||
CRIME_DIR := $(MANUAL_DATA)/crime
|
CRIME_DIR := $(DATA_DIR)/crime
|
||||||
CRIME := $(DATA_DIR)/crime_by_lsoa.parquet
|
CRIME := $(DATA_DIR)/crime_by_lsoa.parquet
|
||||||
|
CRIME_STAMP := $(CRIME_DIR)/.downloaded
|
||||||
NOISE := $(DATA_DIR)/road_noise.parquet
|
NOISE := $(DATA_DIR)/road_noise.parquet
|
||||||
OFSTED := $(DATA_DIR)/ofsted.parquet
|
OFSTED := $(DATA_DIR)/ofsted.parquet
|
||||||
NAPTAN := $(DATA_DIR)/naptan.parquet
|
NAPTAN := $(DATA_DIR)/naptan.parquet
|
||||||
|
|
@ -42,42 +40,56 @@ RENTAL := $(DATA_DIR)/rental_prices.parquet
|
||||||
INSPIRE_DIR := $(DATA_DIR)/inspire
|
INSPIRE_DIR := $(DATA_DIR)/inspire
|
||||||
OA_BOUNDARIES := $(DATA_DIR)/oa_boundaries.gpkg
|
OA_BOUNDARIES := $(DATA_DIR)/oa_boundaries.gpkg
|
||||||
UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet
|
UPRN_LOOKUP := $(DATA_DIR)/uprn_lookup.parquet
|
||||||
PC_BOUNDARIES := $(MANUAL_DATA)/postcode_boundaries
|
PC_BOUNDARIES := $(DATA_DIR)/postcode_boundaries
|
||||||
TRANSIT_DIR := $(DATA_DIR)/transit
|
TRANSIT_DIR := $(DATA_DIR)/transit
|
||||||
TRANSIT_STAMP := $(TRANSIT_DIR)/.done
|
TRANSIT_STAMP := $(TRANSIT_DIR)/.done
|
||||||
|
R5_NETWORK_CACHE := $(DATA_DIR)/r5-network/network.dat
|
||||||
GREENSPACE := $(DATA_DIR)/greenspace_water.parquet
|
GREENSPACE := $(DATA_DIR)/greenspace_water.parquet
|
||||||
OS_GREENSPACE := $(DATA_DIR)/os_greenspace.parquet
|
OS_GREENSPACE := $(DATA_DIR)/os_greenspace.parquet
|
||||||
PBF := $(DATA_DIR)/england-latest.osm.pbf
|
PBF := $(DATA_DIR)/england-latest.osm.pbf
|
||||||
|
FR_TOW := $(DATA_DIR)/FR_TOW_V1_ALL.zip
|
||||||
|
TREE_DENSITY_PC := $(DATA_DIR)/tree_density_by_postcode.parquet
|
||||||
|
TREE_DENSITY_STREETS := $(DATA_DIR)/tree_density_by_street.parquet
|
||||||
|
TREE_DENSITY_ADDR := $(DATA_DIR)/tree_density_by_address.parquet
|
||||||
|
OFS_REGISTER := $(DATA_DIR)/ofs_register.xlsx
|
||||||
PLACES := $(DATA_DIR)/places.parquet
|
PLACES := $(DATA_DIR)/places.parquet
|
||||||
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
|
LSOA_POP := $(DATA_DIR)/lsoa_population.parquet
|
||||||
MEDIAN_AGE := $(DATA_DIR)/median_age.parquet
|
MEDIAN_AGE := $(DATA_DIR)/median_age.parquet
|
||||||
ELECTION := $(DATA_DIR)/election_results.parquet
|
ELECTION := $(DATA_DIR)/election_results.parquet
|
||||||
ENGLAND_BOUNDARY := $(DATA_DIR)/england_boundary.geojson
|
ENGLAND_BOUNDARY := $(DATA_DIR)/england_boundary.geojson
|
||||||
RM_OUTCODES := frontend/src/lib/rightmove-outcodes.json
|
RM_OUTCODES := frontend/src/lib/rightmove-outcodes.json
|
||||||
|
MAP_ASSETS_DIR := frontend/public/assets
|
||||||
|
|
||||||
# Sentinel files for directory targets (Make doesn't track directories well)
|
# Sentinel files for directory targets (Make doesn't track directories well)
|
||||||
INSPIRE_STAMP := $(INSPIRE_DIR)/.done
|
INSPIRE_STAMP := $(INSPIRE_DIR)/.done
|
||||||
|
MAP_ASSETS_STAMP := $(MAP_ASSETS_DIR)/.done
|
||||||
|
|
||||||
PMTILES_VERSION := 1.22.3
|
PMTILES_VERSION := 1.22.3
|
||||||
|
|
||||||
|
POI_PROXIMITY_DEPS := pipeline/transform/poi_proximity.py pipeline/utils/poi_counts.py
|
||||||
|
MERGE_DEPS := pipeline/transform/merge.py
|
||||||
|
TREE_DENSITY_DEPS := pipeline/transform/tree_density.py
|
||||||
|
|
||||||
# ── Phony aliases ─────────────────────────────────────────────────────────────
|
# ── Phony aliases ─────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
.PHONY: prepare merge tiles \
|
.PHONY: prepare merge tiles \
|
||||||
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
download-arcgis download-price-paid download-deprivation download-ethnicity \
|
||||||
download-naptan download-pois download-grocery-retail-points download-ofsted download-broadband download-rental-prices \
|
download-naptan download-pois download-grocery-retail-points download-ofsted download-broadband download-rental-prices \
|
||||||
download-postcodes download-noise download-inspire \
|
download-postcodes download-noise download-inspire download-crime \
|
||||||
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-places download-lsoa-population download-median-age download-england-boundary download-rightmove-outcodes \
|
download-oa-boundaries download-uprn-lookup download-transit-network download-greenspace download-os-greenspace download-pbf download-fr-tow download-ofs-register download-places download-lsoa-population download-median-age download-england-boundary download-rightmove-outcodes \
|
||||||
|
download-map-assets \
|
||||||
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
transform-pois transform-epc-pp transform-crime transform-poi-proximity \
|
||||||
transform-school-proximity transform-postcode-boundaries \
|
transform-school-proximity transform-tree-density \
|
||||||
generate-postcode-boundaries
|
generate-postcode-boundaries generate-travel-times
|
||||||
|
|
||||||
prepare: $(PRICES_STAMP)
|
prepare: $(PRICES_STAMP) download-places tiles generate-postcode-boundaries download-map-assets generate-travel-times
|
||||||
merge: $(MERGE_STAMP)
|
merge: $(MERGE_STAMP)
|
||||||
tiles: $(TILES)
|
tiles: $(TILES)
|
||||||
download-arcgis: $(ARCGIS)
|
download-arcgis: $(ARCGIS)
|
||||||
download-price-paid: $(PRICE_PAID)
|
download-price-paid: $(PRICE_PAID)
|
||||||
download-deprivation: $(IOD)
|
download-deprivation: $(IOD)
|
||||||
download-ethnicity: $(ETHNICITY)
|
download-ethnicity: $(ETHNICITY)
|
||||||
|
download-crime: $(CRIME_STAMP)
|
||||||
download-naptan: $(NAPTAN)
|
download-naptan: $(NAPTAN)
|
||||||
download-pois: $(POIS_RAW)
|
download-pois: $(POIS_RAW)
|
||||||
download-grocery-retail-points: $(GROCERY_RETAIL_POINTS)
|
download-grocery-retail-points: $(GROCERY_RETAIL_POINTS)
|
||||||
|
|
@ -93,24 +105,33 @@ download-transit-network: $(TRANSIT_STAMP)
|
||||||
download-greenspace: $(GREENSPACE)
|
download-greenspace: $(GREENSPACE)
|
||||||
download-os-greenspace: $(OS_GREENSPACE)
|
download-os-greenspace: $(OS_GREENSPACE)
|
||||||
download-pbf: $(PBF)
|
download-pbf: $(PBF)
|
||||||
|
download-fr-tow: $(FR_TOW)
|
||||||
|
download-ofs-register: $(OFS_REGISTER)
|
||||||
download-places: $(PLACES)
|
download-places: $(PLACES)
|
||||||
download-lsoa-population: $(LSOA_POP)
|
download-lsoa-population: $(LSOA_POP)
|
||||||
download-median-age: $(MEDIAN_AGE)
|
download-median-age: $(MEDIAN_AGE)
|
||||||
download-election-results: $(ELECTION)
|
download-election-results: $(ELECTION)
|
||||||
download-england-boundary: $(ENGLAND_BOUNDARY)
|
download-england-boundary: $(ENGLAND_BOUNDARY)
|
||||||
download-rightmove-outcodes: $(RM_OUTCODES)
|
download-rightmove-outcodes: $(RM_OUTCODES)
|
||||||
|
download-map-assets: $(MAP_ASSETS_STAMP)
|
||||||
transform-pois: $(POIS_FILTERED)
|
transform-pois: $(POIS_FILTERED)
|
||||||
transform-epc-pp: $(EPC_PP)
|
transform-epc-pp: $(EPC_PP)
|
||||||
transform-crime: $(CRIME)
|
transform-crime: $(CRIME)
|
||||||
transform-poi-proximity: $(POI_PROXIMITY)
|
transform-poi-proximity: $(POI_PROXIMITY)
|
||||||
transform-school-proximity: $(SCHOOL_PROX)
|
transform-school-proximity: $(SCHOOL_PROX)
|
||||||
transform-postcode-boundaries: $(PC_BOUNDARIES)
|
transform-tree-density: $(TREE_DENSITY_ADDR)
|
||||||
generate-postcode-boundaries: $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP)
|
generate-postcode-boundaries: $(OA_BOUNDARIES) $(INSPIRE_STAMP) $(UPRN_LOOKUP)
|
||||||
uv run python -m pipeline.transform.postcode_boundaries \
|
uv run python -m pipeline.transform.postcode_boundaries \
|
||||||
--uprn $(UPRN_LOOKUP) \
|
--uprn $(UPRN_LOOKUP) \
|
||||||
--oa-boundaries $(OA_BOUNDARIES) \
|
--oa-boundaries $(OA_BOUNDARIES) \
|
||||||
--inspire $(INSPIRE_DIR) \
|
--inspire $(INSPIRE_DIR) \
|
||||||
--output $(PC_BOUNDARIES)
|
--output $(PC_BOUNDARIES)
|
||||||
|
generate-travel-times: $(ARCGIS) $(PLACES) $(PBF) download-transit-network
|
||||||
|
@if [ -f "$(R5_NETWORK_CACHE)" ] && { [ "$(PBF)" -nt "$(R5_NETWORK_CACHE)" ] || [ "$(TRANSIT_STAMP)" -nt "$(R5_NETWORK_CACHE)" ]; }; then \
|
||||||
|
echo "R5 inputs are newer than $(R5_NETWORK_CACHE); deleting stale cache"; \
|
||||||
|
rm -f "$(R5_NETWORK_CACHE)"; \
|
||||||
|
fi
|
||||||
|
./r5-java/run.sh
|
||||||
|
|
||||||
# ── Downloads ─────────────────────────────────────────────────────────────────
|
# ── Downloads ─────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
|
|
@ -121,10 +142,10 @@ $(TILES):
|
||||||
$(EPC):
|
$(EPC):
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "=== EPC dataset not found ==="
|
@echo "=== EPC dataset not found ==="
|
||||||
@echo "The EPC certificates file is required: $@"
|
@echo "The EPC certificates archive is required: $@"
|
||||||
@echo ""
|
@echo ""
|
||||||
@echo "To obtain it, register at https://epc.opendatacommunities.org/login"
|
@echo "To obtain it, register at https://get-energy-performance-data.communities.gov.uk/filter-properties?property_type=domestic"
|
||||||
@echo "and place certificates.csv in manual-data/"
|
@echo "and place domestic-csv.zip in manual-data/"
|
||||||
@echo ""
|
@echo ""
|
||||||
@exit 1
|
@exit 1
|
||||||
|
|
||||||
|
|
@ -140,6 +161,10 @@ $(IOD):
|
||||||
$(ETHNICITY):
|
$(ETHNICITY):
|
||||||
uv run python -m pipeline.download.ethnicity --output $@
|
uv run python -m pipeline.download.ethnicity --output $@
|
||||||
|
|
||||||
|
$(CRIME_STAMP):
|
||||||
|
uv run python -m pipeline.download.crime --output $(CRIME_DIR)
|
||||||
|
@touch $@
|
||||||
|
|
||||||
$(NAPTAN):
|
$(NAPTAN):
|
||||||
uv run python -m pipeline.download.naptan --output $@
|
uv run python -m pipeline.download.naptan --output $@
|
||||||
|
|
||||||
|
|
@ -148,6 +173,16 @@ $(PBF):
|
||||||
curl -L -o $@.tmp https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf
|
curl -L -o $@.tmp https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf
|
||||||
mv $@.tmp $@
|
mv $@.tmp $@
|
||||||
|
|
||||||
|
$(FR_TOW):
|
||||||
|
@mkdir -p $(DATA_DIR)
|
||||||
|
curl -L -A "Mozilla/5.0" -o $@.tmp "https://www.mediafire.com/file_premium/p5fve6wswwwjqrq/FR_TOW_V1_ALL.zip/file"
|
||||||
|
mv $@.tmp $@
|
||||||
|
|
||||||
|
$(OFS_REGISTER):
|
||||||
|
@mkdir -p $(DATA_DIR)
|
||||||
|
curl -fL -A "Mozilla/5.0" -o $@.tmp https://register-api.officeforstudents.org.uk/api/Download/
|
||||||
|
mv $@.tmp $@
|
||||||
|
|
||||||
$(POIS_RAW): $(PBF) $(ENGLAND_BOUNDARY)
|
$(POIS_RAW): $(PBF) $(ENGLAND_BOUNDARY)
|
||||||
uv run python -m pipeline.download.pois --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY)
|
uv run python -m pipeline.download.pois --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY)
|
||||||
|
|
||||||
|
|
@ -189,8 +224,8 @@ $(GREENSPACE): $(PBF)
|
||||||
$(OS_GREENSPACE):
|
$(OS_GREENSPACE):
|
||||||
uv run python -m pipeline.download.os_greenspace --output $@
|
uv run python -m pipeline.download.os_greenspace --output $@
|
||||||
|
|
||||||
$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN)
|
$(PLACES): $(PBF) $(ENGLAND_BOUNDARY) $(NAPTAN) $(OFS_REGISTER) $(ARCGIS)
|
||||||
uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN)
|
uv run python -m pipeline.download.places --output $@ --pbf $(PBF) --boundary $(ENGLAND_BOUNDARY) --naptan $(NAPTAN) --university-register $(OFS_REGISTER) --postcodes $(ARCGIS)
|
||||||
|
|
||||||
$(LSOA_POP):
|
$(LSOA_POP):
|
||||||
uv run python -m pipeline.download.lsoa_population --output $@
|
uv run python -m pipeline.download.lsoa_population --output $@
|
||||||
|
|
@ -208,6 +243,10 @@ $(ENGLAND_BOUNDARY):
|
||||||
$(RM_OUTCODES): $(MERGE_STAMP)
|
$(RM_OUTCODES): $(MERGE_STAMP)
|
||||||
uv run python -m pipeline.download.rightmove_outcodes --postcodes $(POSTCODES_PQ) --output $@
|
uv run python -m pipeline.download.rightmove_outcodes --postcodes $(POSTCODES_PQ) --output $@
|
||||||
|
|
||||||
|
$(MAP_ASSETS_STAMP):
|
||||||
|
uv run python -m pipeline.download.map_assets --output $(MAP_ASSETS_DIR)
|
||||||
|
@touch $@
|
||||||
|
|
||||||
# ── Transforms ────────────────────────────────────────────────────────────────
|
# ── Transforms ────────────────────────────────────────────────────────────────
|
||||||
|
|
||||||
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(ENGLAND_BOUNDARY)
|
$(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(ENGLAND_BOUNDARY)
|
||||||
|
|
@ -216,23 +255,24 @@ $(POIS_FILTERED): $(POIS_RAW) $(NAPTAN) $(GROCERY_RETAIL_POINTS) $(ENGLAND_BOUND
|
||||||
$(EPC_PP): $(PRICE_PAID) $(EPC)
|
$(EPC_PP): $(PRICE_PAID) $(EPC)
|
||||||
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
uv run python -m pipeline.transform.join_epc_pp --epc $(EPC) --price-paid $(PRICE_PAID) --output $@
|
||||||
|
|
||||||
$(CRIME):
|
$(CRIME): $(CRIME_STAMP)
|
||||||
@if [ ! -d "$(CRIME_DIR)" ]; then \
|
|
||||||
echo ""; \
|
|
||||||
echo "=== Crime dataset not found ==="; \
|
|
||||||
echo "Place police.uk crime CSVs in $(CRIME_DIR)/"; \
|
|
||||||
echo "Download from https://data.police.uk/data/"; \
|
|
||||||
echo ""; \
|
|
||||||
exit 1; \
|
|
||||||
fi
|
|
||||||
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@
|
uv run python -m pipeline.transform.crime --input $(CRIME_DIR) --output $@
|
||||||
|
|
||||||
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE)
|
$(POI_PROXIMITY): $(ARCGIS) $(POIS_FILTERED) $(OS_GREENSPACE) $(POI_PROXIMITY_DEPS)
|
||||||
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
|
uv run python -m pipeline.transform.poi_proximity --arcgis $(ARCGIS) --pois $(POIS_FILTERED) --greenspace $(OS_GREENSPACE) --output $@
|
||||||
|
|
||||||
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS)
|
$(SCHOOL_PROX): $(OFSTED) $(ARCGIS)
|
||||||
uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
|
uv run python -m pipeline.transform.school_proximity --ofsted $(OFSTED) --arcgis $(ARCGIS) --output $@
|
||||||
|
|
||||||
|
$(TREE_DENSITY_ADDR): $(FR_TOW) $(ARCGIS) $(PRICE_PAID) $(TREE_DENSITY_DEPS)
|
||||||
|
uv run python -m pipeline.transform.tree_density \
|
||||||
|
--tow-zip $(FR_TOW) \
|
||||||
|
--arcgis $(ARCGIS) \
|
||||||
|
--price-paid $(PRICE_PAID) \
|
||||||
|
--output-postcodes $(TREE_DENSITY_PC) \
|
||||||
|
--output-streets $(TREE_DENSITY_STREETS) \
|
||||||
|
--output-addresses $@
|
||||||
|
|
||||||
# Postcode boundaries require manual generation — fail with instructions
|
# Postcode boundaries require manual generation — fail with instructions
|
||||||
$(PC_BOUNDARIES):
|
$(PC_BOUNDARIES):
|
||||||
@echo ""
|
@echo ""
|
||||||
|
|
@ -251,7 +291,7 @@ $(PC_BOUNDARIES):
|
||||||
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
|
# ── Final merge → postcode.parquet + properties.parquet ──────────────────────
|
||||||
|
|
||||||
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
$(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||||
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION)
|
$(ETHNICITY) $(CRIME) $(NOISE) $(SCHOOL_PROX) $(BROADBAND) $(RENTAL) $(LSOA_POP) $(MEDIAN_AGE) $(ELECTION) $(TREE_DENSITY_ADDR) $(MERGE_DEPS)
|
||||||
uv run python -m pipeline.transform.merge \
|
uv run python -m pipeline.transform.merge \
|
||||||
--epc-pp $(EPC_PP) \
|
--epc-pp $(EPC_PP) \
|
||||||
--arcgis $(ARCGIS) \
|
--arcgis $(ARCGIS) \
|
||||||
|
|
@ -266,6 +306,7 @@ $(MERGE_STAMP): $(EPC_PP) $(ARCGIS) $(IOD) $(POI_PROXIMITY) \
|
||||||
--lsoa-population $(LSOA_POP) \
|
--lsoa-population $(LSOA_POP) \
|
||||||
--median-age $(MEDIAN_AGE) \
|
--median-age $(MEDIAN_AGE) \
|
||||||
--election-results $(ELECTION) \
|
--election-results $(ELECTION) \
|
||||||
|
--tree-density-addresses $(TREE_DENSITY_ADDR) \
|
||||||
--output-postcodes $(POSTCODES_PQ) \
|
--output-postcodes $(POSTCODES_PQ) \
|
||||||
--output-properties $(PROPERTIES_PQ)
|
--output-properties $(PROPERTIES_PQ)
|
||||||
@touch $@
|
@touch $@
|
||||||
|
|
|
||||||
10
README.md
10
README.md
|
|
@ -69,11 +69,10 @@ property-data/postcode_boundaries/
|
||||||
property-data/travel-times/
|
property-data/travel-times/
|
||||||
```
|
```
|
||||||
|
|
||||||
Most data can be downloaded or generated through `Makefile.data`. Some inputs
|
Most data, including police.uk crime archives, can be downloaded or generated
|
||||||
are deliberately manual:
|
through `Makefile.data`. Some inputs are deliberately manual:
|
||||||
|
|
||||||
- `manual-data/certificates.csv` from the EPC register
|
- `manual-data/domestic-csv.zip` from the EPC register
|
||||||
- `manual-data/crime/` CSV exports from police.uk
|
|
||||||
- postcode boundaries, generated from OA boundaries, INSPIRE polygons, and UPRN
|
- postcode boundaries, generated from OA boundaries, INSPIRE polygons, and UPRN
|
||||||
lookup data
|
lookup data
|
||||||
|
|
||||||
|
|
@ -94,8 +93,7 @@ The running server expects the same structure under
|
||||||
Travel times are built separately because they are expensive:
|
Travel times are built separately because they are expensive:
|
||||||
|
|
||||||
```bash
|
```bash
|
||||||
make -f Makefile.data download-transit-network
|
make -f Makefile.data generate-travel-times
|
||||||
./r5-java/run.sh --threads 8 --heap 40g
|
|
||||||
```
|
```
|
||||||
|
|
||||||
For a quick R5 smoke test:
|
For a quick R5 smoke test:
|
||||||
|
|
|
||||||
30541
analyses/tree_density_methodology.ipynb
Normal file
30541
analyses/tree_density_methodology.ipynb
Normal file
File diff suppressed because one or more lines are too long
6
check.sh
6
check.sh
|
|
@ -12,11 +12,7 @@ step() {
|
||||||
|
|
||||||
step "Python lint: ruff" uv run ruff check .
|
step "Python lint: ruff" uv run ruff check .
|
||||||
step "Python dependency lint: deptry" uv run deptry .
|
step "Python dependency lint: deptry" uv run deptry .
|
||||||
step "Python unit tests" uv run pytest \
|
step "Python unit tests" uv run pytest pipeline
|
||||||
pipeline/utils/test_haversine.py \
|
|
||||||
pipeline/utils/test_poi_counts.py \
|
|
||||||
pipeline/download/test_naptan.py \
|
|
||||||
pipeline/transform/postcode_boundaries/test_postcode_boundaries.py
|
|
||||||
|
|
||||||
(
|
(
|
||||||
cd "$ROOT_DIR/frontend"
|
cd "$ROOT_DIR/frontend"
|
||||||
|
|
|
||||||
|
|
@ -6,6 +6,8 @@ services:
|
||||||
server:
|
server:
|
||||||
image: rust:1.84
|
image: rust:1.84
|
||||||
init: true
|
init: true
|
||||||
|
tty: true
|
||||||
|
stdin_open: true
|
||||||
working_dir: /app/server-rs
|
working_dir: /app/server-rs
|
||||||
command: >
|
command: >
|
||||||
bash -c "
|
bash -c "
|
||||||
|
|
|
||||||
BIN
frontend/public/assets/fonts/Noto Sans Italic/0-255.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/0-255.pbf
Normal file
Binary file not shown.
BIN
frontend/public/assets/fonts/Noto Sans Italic/1024-1279.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/1024-1279.pbf
Normal file
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic10240-10495
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic10496-10751
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic10752-11007
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic11008-11263
|
||||||
BIN
frontend/public/assets/fonts/Noto Sans Italic/11264-11519.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/11264-11519.pbf
Normal file
Binary file not shown.
BIN
frontend/public/assets/fonts/Noto Sans Italic/11520-11775.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/11520-11775.pbf
Normal file
Binary file not shown.
BIN
frontend/public/assets/fonts/Noto Sans Italic/11776-12031.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/11776-12031.pbf
Normal file
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic12032-12287
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic12288-12543
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic12544-12799
|
||||||
BIN
frontend/public/assets/fonts/Noto Sans Italic/1280-1535.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/1280-1535.pbf
Normal file
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic12800-13055
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic13056-13311
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic13312-13567
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic13568-13823
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic13824-14079
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic14080-14335
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic14336-14591
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic14592-14847
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic14848-15103
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic15104-15359
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 1536-1791
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic15360-15615
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic15616-15871
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic15872-16127
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic16128-16383
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic16384-16639
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic16640-16895
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic16896-17151
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic17152-17407
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic17408-17663
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic17664-17919
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 1792-2047
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic17920-18175
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic18176-18431
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic18432-18687
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic18688-18943
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic18944-19199
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic19200-19455
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic19456-19711
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic19712-19967
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic19968-20223
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic20224-20479
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 2048-2303
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic20480-20735
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic20736-20991
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic20992-21247
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic21248-21503
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic21504-21759
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic21760-22015
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic22016-22271
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic22272-22527
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic22528-22783
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic22784-23039
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 2304-2559
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic23040-23295
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic23296-23551
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic23552-23807
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic23808-24063
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic24064-24319
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic24320-24575
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic24576-24831
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic24832-25087
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic25088-25343
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic25344-25599
|
||||||
BIN
frontend/public/assets/fonts/Noto Sans Italic/256-511.pbf
Normal file
BIN
frontend/public/assets/fonts/Noto Sans Italic/256-511.pbf
Normal file
Binary file not shown.
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 2560-2815
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic25600-25855
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic25856-26111
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic26112-26367
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic26368-26623
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic26624-26879
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic26880-27135
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic27136-27391
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic27392-27647
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic27648-27903
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic27904-28159
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 2816-3071
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic28160-28415
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic28416-28671
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic28672-28927
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic28928-29183
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic29184-29439
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic29440-29695
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic29696-29951
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic29952-30207
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic30208-30463
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic30464-30719
|
||||||
|
|
@ -0,0 +1,3 @@
|
||||||
|
|
||||||
|
|
||||||
|
Sans Italic 3072-3327
|
||||||
Some files were not shown because too many files have changed in this diff Show more
Loading…
Add table
Add a link
Reference in a new issue