191 lines
6.4 KiB
YAML
191 lines
6.4 KiB
YAML
version: '3'
|
|
|
|
vars:
|
|
DATA_DIR: /bulk/property-data
|
|
ARCGIS_OUTPUT: "{{.DATA_DIR}}/arcgis_data.parquet"
|
|
PRICE_PAID_OUTPUT: "{{.DATA_DIR}}/price-paid-complete.parquet"
|
|
IOD_OUTPUT: "{{.DATA_DIR}}/IoD2025_Scores.parquet"
|
|
POIS_RAW_OUTPUT: "{{.DATA_DIR}}/uk_pois.parquet"
|
|
POIS_FILTERED_OUTPUT: "{{.DATA_DIR}}/filtered_uk_pois.parquet"
|
|
POI_PROXIMITY_OUTPUT: "{{.DATA_DIR}}/poi_proximity.parquet"
|
|
EPC_PP_OUTPUT: "{{.DATA_DIR}}/epc_pp.parquet"
|
|
WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet"
|
|
EPC: "{{.DATA_DIR}}/certificates.csv"
|
|
JOURNEY_TIMES: "./data_sources/processed/journey_times_bank_checkpoint.parquet"
|
|
ETHNICITY_OUTPUT: "{{.DATA_DIR}}/ethnicity_by_la.parquet"
|
|
CRIME_DIR: "{{.DATA_DIR}}/crime"
|
|
CRIME_OUTPUT: "{{.DATA_DIR}}/crime_by_lsoa.parquet"
|
|
NOISE_OUTPUT: "{{.DATA_DIR}}/road_noise.parquet"
|
|
OFSTED_OUTPUT: "{{.DATA_DIR}}/ofsted.parquet"
|
|
NAPTAN_OUTPUT: "{{.DATA_DIR}}/naptan.parquet"
|
|
BROADBAND_OUTPUT: "{{.DATA_DIR}}/broadband.parquet"
|
|
SCHOOL_PROXIMITY_OUTPUT: "{{.DATA_DIR}}/school_proximity.parquet"
|
|
|
|
tasks:
|
|
prompt:epc:
|
|
desc: Prompt user to download EPC dataset (requires registration)
|
|
status:
|
|
- test -f {{.EPC}}
|
|
cmds:
|
|
- |
|
|
echo ""
|
|
echo "=== EPC dataset not found ==="
|
|
echo "The EPC certificates file is required: {{.EPC}}"
|
|
echo ""
|
|
echo "To obtain it, register at https://epc.opendatacommunities.org/login"
|
|
echo ""
|
|
exit 1
|
|
|
|
prompt:journey-times:
|
|
desc: Download TFL journey times if missing (requires API key registration)
|
|
status:
|
|
- test -f {{.JOURNEY_TIMES}}
|
|
deps:
|
|
- download:arcgis
|
|
cmds:
|
|
- |
|
|
echo ""
|
|
echo "=== TFL journey times not found ==="
|
|
echo "Register for a TFL API key at https://api-portal.tfl.gov.uk/signin"
|
|
echo "Then set the TFL_API_KEY environment variable and re-run this task."
|
|
echo ""
|
|
exit 1
|
|
|
|
download:arcgis:
|
|
desc: Download and convert ArcGIS postcode data
|
|
status:
|
|
- test -f {{.ARCGIS_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.arcgis --output {{.ARCGIS_OUTPUT}}
|
|
|
|
download:price-paid:
|
|
desc: Download and convert Land Registry price-paid data
|
|
status:
|
|
- test -f {{.PRICE_PAID_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.price_paid --output {{.PRICE_PAID_OUTPUT}}
|
|
|
|
download:deprivation:
|
|
desc: Download and convert Index of Deprivation data
|
|
status:
|
|
- test -f {{.IOD_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.deprivation_data --output {{.IOD_OUTPUT}}
|
|
|
|
download:ethnicity:
|
|
desc: Download ethnicity by local authority data
|
|
status:
|
|
- test -f {{.ETHNICITY_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.ethnicity --output {{.ETHNICITY_OUTPUT}}
|
|
|
|
download:naptan:
|
|
desc: Download NaPTAN station data
|
|
status:
|
|
- test -f {{.NAPTAN_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.naptan --output {{.NAPTAN_OUTPUT}}
|
|
|
|
download:pois:
|
|
desc: Download and extract POIs from OpenStreetMap
|
|
status:
|
|
- test -f {{.POIS_RAW_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.pois --output {{.POIS_RAW_OUTPUT}}
|
|
|
|
download:ofsted:
|
|
desc: Download Ofsted school inspection outcomes
|
|
status:
|
|
- test -f {{.OFSTED_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.ofsted --output {{.OFSTED_OUTPUT}}
|
|
|
|
download:broadband:
|
|
desc: Download Ofcom broadband performance data
|
|
status:
|
|
- test -f {{.BROADBAND_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.broadband --output {{.BROADBAND_OUTPUT}}
|
|
|
|
download:noise:
|
|
desc: Download Defra noise data (road, rail, airport) sampled at postcode centroids
|
|
deps:
|
|
- download:arcgis
|
|
status:
|
|
- test -f {{.NOISE_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.download.noise --arcgis {{.ARCGIS_OUTPUT}} --output {{.NOISE_OUTPUT}}
|
|
|
|
transform:pois:
|
|
desc: Transform raw POIs to filtered version with friendly names
|
|
deps:
|
|
- download:pois
|
|
- download:naptan
|
|
status:
|
|
- test -f {{.POIS_FILTERED_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.transform_poi --input {{.POIS_RAW_OUTPUT}} --naptan {{.NAPTAN_OUTPUT}} --output {{.POIS_FILTERED_OUTPUT}}
|
|
|
|
transform:epc-pp:
|
|
desc: Fuzzy join EPC and Price Paid data
|
|
deps:
|
|
- download:price-paid
|
|
- prompt:epc
|
|
status:
|
|
- test -f {{.EPC_PP_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.join_epc_pp --epc {{.EPC}} --price-paid {{.PRICE_PAID_OUTPUT}} --output {{.EPC_PP_OUTPUT}}
|
|
|
|
transform:crime:
|
|
desc: Transform crime CSVs into yearly averages by LSOA
|
|
status:
|
|
- test -f {{.CRIME_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.crime --input {{.CRIME_DIR}} --output {{.CRIME_OUTPUT}}
|
|
|
|
transform:poi-proximity:
|
|
desc: Compute POI proximity counts per postcode
|
|
deps:
|
|
- download:arcgis
|
|
- transform:pois
|
|
status:
|
|
- test -f {{.POI_PROXIMITY_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.poi_proximity --arcgis {{.ARCGIS_OUTPUT}} --pois {{.POIS_FILTERED_OUTPUT}} --output {{.POI_PROXIMITY_OUTPUT}}
|
|
|
|
transform:school-proximity:
|
|
desc: Compute good+ school proximity counts per postcode
|
|
deps:
|
|
- download:ofsted
|
|
- download:arcgis
|
|
status:
|
|
- test -f {{.SCHOOL_PROXIMITY_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.school_proximity --ofsted {{.OFSTED_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --output {{.SCHOOL_PROXIMITY_OUTPUT}}
|
|
|
|
download:journey-times:
|
|
desc: Fetch TfL journey times for all postcodes
|
|
deps:
|
|
- download:arcgis
|
|
status:
|
|
- test -f {{.JOURNEY_TIMES}}
|
|
cmds:
|
|
- uv run python -m pipeline.journey_times
|
|
|
|
prepare:
|
|
desc: Build wide property dataframe with all joins
|
|
deps:
|
|
- transform:epc-pp
|
|
- download:arcgis
|
|
- download:deprivation
|
|
- download:ethnicity
|
|
- download:broadband
|
|
- download:noise
|
|
- transform:crime
|
|
- transform:poi-proximity
|
|
- transform:school-proximity
|
|
- prompt:journey-times
|
|
status:
|
|
- test -f {{.WIDE_OUTPUT}}
|
|
cmds:
|
|
- uv run python -m pipeline.transform.merge --epc-pp {{.EPC_PP_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --iod {{.IOD_OUTPUT}} --poi-proximity {{.POI_PROXIMITY_OUTPUT}} --journey-times {{.JOURNEY_TIMES}} --ethnicity {{.ETHNICITY_OUTPUT}} --crime {{.CRIME_OUTPUT}} --noise {{.NOISE_OUTPUT}} --school-proximity {{.SCHOOL_PROXIMITY_OUTPUT}} --broadband {{.BROADBAND_OUTPUT}} --output {{.WIDE_OUTPUT}}
|