Update data deps

This commit is contained in:
Andras Schmelczer 2026-02-03 20:24:51 +00:00
parent 7a2ff48201
commit 1f148b2185

View file

@ -12,7 +12,7 @@ vars:
WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet" WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet"
EPC: "{{.DATA_DIR}}/certificates.csv" EPC: "{{.DATA_DIR}}/certificates.csv"
JOURNEY_TIMES_BANK: "{{.DATA_DIR}}/journey_times_bank.parquet" JOURNEY_TIMES_BANK: "{{.DATA_DIR}}/journey_times_bank.parquet"
JOURNEY_TIMES_FITZROVIA: "{{.DATA_DIR}}/journey_times_fitzrovia.parquet" JOURNEY_TIMES_FITZROVIA: "{{.DATA_DIR}}/journey_times_fitzrovia_checkpoint.parquet"
ETHNICITY_OUTPUT: "{{.DATA_DIR}}/ethnicity_by_la.parquet" ETHNICITY_OUTPUT: "{{.DATA_DIR}}/ethnicity_by_la.parquet"
CRIME_DIR: "{{.DATA_DIR}}/crime" CRIME_DIR: "{{.DATA_DIR}}/crime"
CRIME_OUTPUT: "{{.DATA_DIR}}/crime_by_lsoa.parquet" CRIME_OUTPUT: "{{.DATA_DIR}}/crime_by_lsoa.parquet"
@ -168,29 +168,27 @@ tasks:
- uv run python -m pipeline.transform.school_proximity --ofsted {{.OFSTED_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --output {{.SCHOOL_PROXIMITY_OUTPUT}} - uv run python -m pipeline.transform.school_proximity --ofsted {{.OFSTED_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --output {{.SCHOOL_PROXIMITY_OUTPUT}}
download:journey-times: download:journey-times:
desc: "Fetch TfL journey times: task download:journey-times -- <destination>" desc: "Fetch TfL journey times: task download:journey-times"
deps: deps:
- download:arcgis - download:arcgis
requires:
vars: [CLI_ARGS]
status: status:
- test -f {{.DATA_DIR}}/journey_times_{{.CLI_ARGS}}.parquet - test -f {{.DATA_DIR}}/journey_times_*.parquet
cmds: cmds:
- uv run python -m pipeline.journey_times --destination {{.CLI_ARGS}} --output-dir {{.DATA_DIR}} --postcodes {{.ARCGIS_OUTPUT}} - uv run python -m pipeline.journey_times --destination {{.CLI_ARGS}} --output-dir {{.DATA_DIR}} --postcodes {{.ARCGIS_OUTPUT}}
prepare: prepare:
desc: Build wide property dataframe with all joins desc: Build wide property dataframe with all joins
deps: deps:
- transform:epc-pp # - transform:epc-pp
- download:arcgis # - download:arcgis
- download:deprivation # - download:deprivation
- download:ethnicity # - download:ethnicity
- download:broadband # - download:broadband
- download:noise # - download:noise
- transform:crime # - transform:crime
- transform:poi-proximity # - transform:poi-proximity
- transform:school-proximity # - transform:school-proximity
- prompt:journey-times # - prompt:journey-times
status: status:
- test -f {{.WIDE_OUTPUT}} - test -f {{.WIDE_OUTPUT}}
cmds: cmds: