perfect-postcode/Taskfile.yml

version: '3'

vars:
  DATA_DIR: data
  ARCGIS_OUTPUT: "{{.DATA_DIR}}/arcgis_data.parquet"
  PRICE_PAID_OUTPUT: "{{.DATA_DIR}}/price-paid-complete.parquet"
  IOD_OUTPUT: "{{.DATA_DIR}}/IoD2025_Scores.parquet"
  POIS_RAW_OUTPUT: "{{.DATA_DIR}}/uk_pois.parquet"
  POIS_FILTERED_OUTPUT: "{{.DATA_DIR}}/filtered_uk_pois.parquet"
  POI_PROXIMITY_OUTPUT: "{{.DATA_DIR}}/poi_proximity.parquet"
  EPC_PP_OUTPUT: "{{.DATA_DIR}}/epc_pp.parquet"
  WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet"
  EPC_CSV: "{{.DATA_DIR}}/epc/certificates.csv"
  JOURNEY_TIMES: "{{.DATA_DIR}}/journey_times_bank_checkpoint.parquet"

tasks:
  install:
    desc: Install dependencies
    cmds:
      - uv sync
      - cd frontend && npm install

  download:arcgis:
    desc: Download and convert ArcGIS postcode data
    sources:
      - pipeline/download/arcgis.py
    generates:
      - "{{.ARCGIS_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.download.arcgis --output {{.ARCGIS_OUTPUT}}

  download:price-paid:
    desc: Download and convert Land Registry price-paid data
    sources:
      - pipeline/download/price_paid.py
    generates:
      - "{{.PRICE_PAID_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.download.price_paid --output {{.PRICE_PAID_OUTPUT}}

  download:deprivation:
    desc: Download and convert Index of Deprivation data
    sources:
      - pipeline/download/deprivation_data.py
    generates:
      - "{{.IOD_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.download.deprivation_data --output {{.IOD_OUTPUT}}

  download:pois:
    desc: Download and extract POIs from OpenStreetMap
    sources:
      - pipeline/download/pois.py
    generates:
      - "{{.POIS_RAW_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.download.pois --output {{.POIS_RAW_OUTPUT}}

  transform:pois:
    desc: Transform raw POIs to filtered version with friendly names
    deps:
      - download:pois
    sources:
      - pipeline/transform/transform_poi.py
      - "{{.POIS_RAW_OUTPUT}}"
    generates:
      - "{{.POIS_FILTERED_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.transform.transform_poi --input {{.POIS_RAW_OUTPUT}} --output {{.POIS_FILTERED_OUTPUT}}

  transform:epc-pp:
    desc: Fuzzy join EPC and Price Paid data
    deps:
      - download:price-paid
    sources:
      - pipeline/transform/join_epc_pp.py
      - pipeline/utils/fuzzy_join.py
      - "{{.PRICE_PAID_OUTPUT}}"
      - "{{.EPC_CSV}}"
    generates:
      - "{{.EPC_PP_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.transform.join_epc_pp --epc {{.EPC_CSV}} --price-paid {{.PRICE_PAID_OUTPUT}} --output {{.EPC_PP_OUTPUT}}

  transform:poi-proximity:
    desc: Compute POI proximity counts per postcode
    deps:
      - download:arcgis
      - transform:pois
    sources:
      - pipeline/transform/poi_proximity.py
      - pipeline/utils/poi_counts.py
      - "{{.ARCGIS_OUTPUT}}"
      - "{{.POIS_FILTERED_OUTPUT}}"
    generates:
      - "{{.POI_PROXIMITY_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.transform.poi_proximity --arcgis {{.ARCGIS_OUTPUT}} --pois {{.POIS_FILTERED_OUTPUT}} --output {{.POI_PROXIMITY_OUTPUT}}

  transform:wide:
    desc: Build wide property dataframe with all joins
    deps:
      - join:epc-pp
      - download:arcgis
      - download:deprivation
      - transform:poi-proximity
    sources:
      - pipeline/transform/merge.py
      - "{{.EPC_PP_OUTPUT}}"
      - "{{.ARCGIS_OUTPUT}}"
      - "{{.IOD_OUTPUT}}"
      - "{{.POI_PROXIMITY_OUTPUT}}"
    generates:
      - "{{.WIDE_OUTPUT}}"
    cmds:
      - uv run python -m pipeline.transform.merge --epc-pp {{.EPC_PP_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --iod {{.IOD_OUTPUT}} --poi-proximity {{.POI_PROXIMITY_OUTPUT}} --journey-times {{.JOURNEY_TIMES}} --output {{.WIDE_OUTPUT}}

  prepare:
    desc: Prepare the application (install, download data, run pipeline)
    deps:
      - transform:wide

  test:
    cmds:
      - uv run -m pipeline.utils.test_fuzzy_join
      - uv run pytest pipeline/utils/test_haversine.py
      - uv run pytest pipeline/utils/test_poi_counts.py

  server:
    desc: Run Rust backend on port 8001
    dir: server-rs
    cmds:
      - cargo run --release -- {{.WIDE_OUTPUT}}

  frontend:
    desc: Run frontend dev server on port 3030 (proxies /api to :8001)
    dir: frontend
    cmds:
      - npm run dev

  build:
    desc: Build frontend for production
    dir: frontend
    cmds:
      - npm run build


  lint:
    desc: Lint all code (Python, TypeScript, and Rust)
    cmds:
      - task: lint:python
      - task: lint:frontend
      - task: lint:rust

  lint:python:
    desc: Lint Python code with ruff
    cmds:
      - uv run ruff check .

  lint:frontend:
    desc: Lint frontend TypeScript code
    dir: frontend
    cmds:
      - npm run lint
      - npm run format:check

  lint:rust:
    desc: Lint Rust code with clippy and check formatting
    dir: server-rs
    cmds:
      - cargo clippy -- -D warnings
      - cargo fmt --check

  format:
    desc: Format all code (Python, TypeScript, and Rust)
    cmds:
      - task: format:python
      - task: format:frontend
      - task: format:rust

  format:python:
    desc: Format Python code with ruff
    cmds:
      - uv run ruff check --fix .
      - uv run ruff format .

  format:frontend:
    desc: Format frontend TypeScript code
    dir: frontend
    cmds:
      - npm run lint:fix
      - npm run format

  format:rust:
    desc: Format Rust code with cargo fmt
    dir: server-rs
    cmds:
      - cargo fmt

  check:
    desc: Run all checks (lint, typecheck, build)
    cmds:
      - task: lint
      - task: typecheck
      - task: build

  typecheck:
    desc: Type check frontend TypeScript code
    dir: frontend
    cmds:
      - npm run typecheck