Update data deps

This commit is contained in:
Andras Schmelczer 2026-02-03 20:24:51 +00:00
parent 7a2ff48201
commit 1f148b2185

View file

@ -12,7 +12,7 @@ vars:
WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet"
EPC: "{{.DATA_DIR}}/certificates.csv"
JOURNEY_TIMES_BANK: "{{.DATA_DIR}}/journey_times_bank.parquet"
JOURNEY_TIMES_FITZROVIA: "{{.DATA_DIR}}/journey_times_fitzrovia.parquet"
JOURNEY_TIMES_FITZROVIA: "{{.DATA_DIR}}/journey_times_fitzrovia_checkpoint.parquet"
ETHNICITY_OUTPUT: "{{.DATA_DIR}}/ethnicity_by_la.parquet"
CRIME_DIR: "{{.DATA_DIR}}/crime"
CRIME_OUTPUT: "{{.DATA_DIR}}/crime_by_lsoa.parquet"
@ -168,29 +168,27 @@ tasks:
- uv run python -m pipeline.transform.school_proximity --ofsted {{.OFSTED_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --output {{.SCHOOL_PROXIMITY_OUTPUT}}
download:journey-times:
desc: "Fetch TfL journey times: task download:journey-times -- <destination>"
desc: "Fetch TfL journey times: task download:journey-times"
deps:
- download:arcgis
requires:
vars: [CLI_ARGS]
status:
- test -f {{.DATA_DIR}}/journey_times_{{.CLI_ARGS}}.parquet
- test -f {{.DATA_DIR}}/journey_times_*.parquet
cmds:
- uv run python -m pipeline.journey_times --destination {{.CLI_ARGS}} --output-dir {{.DATA_DIR}} --postcodes {{.ARCGIS_OUTPUT}}
prepare:
desc: Build wide property dataframe with all joins
deps:
- transform:epc-pp
- download:arcgis
- download:deprivation
- download:ethnicity
- download:broadband
- download:noise
- transform:crime
- transform:poi-proximity
- transform:school-proximity
- prompt:journey-times
# - transform:epc-pp
# - download:arcgis
# - download:deprivation
# - download:ethnicity
# - download:broadband
# - download:noise
# - transform:crime
# - transform:poi-proximity
# - transform:school-proximity
# - prompt:journey-times
status:
- test -f {{.WIDE_OUTPUT}}
cmds: