version: '3'

vars:
  DATA_DIR: /bulk/property-data
  ARCGIS_OUTPUT: "{{.DATA_DIR}}/arcgis_data.parquet"
  PRICE_PAID_OUTPUT: "{{.DATA_DIR}}/price-paid-complete.parquet"
  IOD_OUTPUT: "{{.DATA_DIR}}/IoD2025_Scores.parquet"
  POIS_RAW_OUTPUT: "{{.DATA_DIR}}/uk_pois.parquet"
  POIS_FILTERED_OUTPUT: "{{.DATA_DIR}}/filtered_uk_pois.parquet"
  POI_PROXIMITY_OUTPUT: "{{.DATA_DIR}}/poi_proximity.parquet"
  EPC_PP_OUTPUT: "{{.DATA_DIR}}/epc_pp.parquet"
  WIDE_OUTPUT: "{{.DATA_DIR}}/wide.parquet"
  EPC: "{{.DATA_DIR}}/certificates.csv"
  JOURNEY_TIMES_BANK: "{{.DATA_DIR}}/journey_times_bank.parquet"
  JOURNEY_TIMES_FITZROVIA: "{{.DATA_DIR}}/journey_times_fitzrovia.parquet"
  ETHNICITY_OUTPUT: "{{.DATA_DIR}}/ethnicity_by_la.parquet"
  CRIME_DIR: "{{.DATA_DIR}}/crime"
  CRIME_OUTPUT: "{{.DATA_DIR}}/crime_by_lsoa.parquet"
  NOISE_OUTPUT: "{{.DATA_DIR}}/road_noise.parquet"
  OFSTED_OUTPUT: "{{.DATA_DIR}}/ofsted.parquet"
  NAPTAN_OUTPUT: "{{.DATA_DIR}}/naptan.parquet"
  BROADBAND_OUTPUT: "{{.DATA_DIR}}/broadband.parquet"
  SCHOOL_PROXIMITY_OUTPUT: "{{.DATA_DIR}}/school_proximity.parquet"

tasks:
  prompt:epc:
    desc: Prompt user to download EPC dataset (requires registration)
    status:
      - test -f {{.EPC}}
    cmds:
      - |
        echo ""
        echo "=== EPC dataset not found ==="
        echo "The EPC certificates file is required: {{.EPC}}"
        echo ""
        echo "To obtain it, register at https://epc.opendatacommunities.org/login"
        echo ""
        exit 1

  prompt:journey-times:
    desc: Download TFL journey times if missing (requires API key registration)
    status:
      - test -f {{.JOURNEY_TIMES_BANK}} || test -f {{.JOURNEY_TIMES_FITZROVIA}}
    deps:
      - download:arcgis
    cmds:
      - |
        echo ""
        echo "=== TFL journey times not found ==="
        echo "Register for a TFL API key at https://api-portal.tfl.gov.uk/signin"
        echo "Then set the TFL_API_KEY environment variable and run:"
        echo "  task download:journey-times -- bank"
        echo "  task download:journey-times -- fitzrovia"
        echo ""
        exit 1

  download:arcgis:
    desc: Download and convert ArcGIS postcode data
    status:
      - test -f {{.ARCGIS_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.arcgis --output {{.ARCGIS_OUTPUT}}

  download:price-paid:
    desc: Download and convert Land Registry price-paid data
    status:
      - test -f {{.PRICE_PAID_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.price_paid --output {{.PRICE_PAID_OUTPUT}}

  download:deprivation:
    desc: Download and convert Index of Deprivation data
    status:
      - test -f {{.IOD_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.deprivation_data --output {{.IOD_OUTPUT}}

  download:ethnicity:
    desc: Download ethnicity by local authority data
    status:
      - test -f {{.ETHNICITY_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.ethnicity --output {{.ETHNICITY_OUTPUT}}

  download:naptan:
    desc: Download NaPTAN station data
    status:
      - test -f {{.NAPTAN_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.naptan --output {{.NAPTAN_OUTPUT}}

  download:pois:
    desc: Download and extract POIs from OpenStreetMap
    status:
      - test -f {{.POIS_RAW_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.pois --output {{.POIS_RAW_OUTPUT}}

  download:ofsted:
    desc: Download Ofsted school inspection outcomes
    status:
      - test -f {{.OFSTED_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.ofsted --output {{.OFSTED_OUTPUT}}

  download:broadband:
    desc: Download Ofcom broadband performance data
    status:
      - test -f {{.BROADBAND_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.broadband --output {{.BROADBAND_OUTPUT}}


  download:noise:
    desc: Download Defra noise data (road, rail, airport) sampled at postcode centroids
    deps:
      - download:arcgis
    status:
      - test -f {{.NOISE_OUTPUT}}
    cmds:
      - uv run python -m pipeline.download.noise --arcgis {{.ARCGIS_OUTPUT}} --output {{.NOISE_OUTPUT}}

  transform:pois:
    desc: Transform raw POIs to filtered version with friendly names
    deps:
      - download:pois
      - download:naptan
    status:
      - test -f {{.POIS_FILTERED_OUTPUT}}
    cmds:
      - uv run python -m pipeline.transform.transform_poi --input {{.POIS_RAW_OUTPUT}} --naptan {{.NAPTAN_OUTPUT}} --output {{.POIS_FILTERED_OUTPUT}}

  transform:epc-pp:
    desc: Fuzzy join EPC and Price Paid data
    deps:
      - download:price-paid
      - prompt:epc
    status:
      - test -f {{.EPC_PP_OUTPUT}}
    cmds:
      - uv run python -m pipeline.transform.join_epc_pp --epc {{.EPC}} --price-paid {{.PRICE_PAID_OUTPUT}} --output {{.EPC_PP_OUTPUT}}

  transform:crime:
    desc: Transform crime CSVs into yearly averages by LSOA
    status:
      - test -f {{.CRIME_OUTPUT}}
    cmds:
      - uv run python -m pipeline.transform.crime --input {{.CRIME_DIR}} --output {{.CRIME_OUTPUT}}

  transform:poi-proximity:
    desc: Compute POI proximity counts per postcode
    deps:
      - download:arcgis
      - transform:pois
    status:
      - test -f {{.POI_PROXIMITY_OUTPUT}}
    cmds:
      - uv run python -m pipeline.transform.poi_proximity --arcgis {{.ARCGIS_OUTPUT}} --pois {{.POIS_FILTERED_OUTPUT}} --output {{.POI_PROXIMITY_OUTPUT}}

  transform:school-proximity:
    desc: Compute good+ school proximity counts per postcode
    deps:
      - download:ofsted
      - download:arcgis
    status:
      - test -f {{.SCHOOL_PROXIMITY_OUTPUT}}
    cmds:
      - uv run python -m pipeline.transform.school_proximity --ofsted {{.OFSTED_OUTPUT}} --arcgis {{.ARCGIS_OUTPUT}} --output {{.SCHOOL_PROXIMITY_OUTPUT}}

  download:journey-times:
    desc: "Fetch TfL journey times: task download:journey-times -- <destination>"
    deps:
      - download:arcgis
    requires:
      vars: [CLI_ARGS]
    status:
      - test -f {{.DATA_DIR}}/journey_times_{{.CLI_ARGS}}.parquet
    cmds:
      - uv run python -m pipeline.journey_times --destination {{.CLI_ARGS}} --output-dir {{.DATA_DIR}} --postcodes {{.ARCGIS_OUTPUT}}

  prepare:
    desc: Build wide property dataframe with all joins
    deps:
      - transform:epc-pp
      - download:arcgis
      - download:deprivation
      - download:ethnicity
      - download:broadband
      - download:noise
      - transform:crime
      - transform:poi-proximity
      - transform:school-proximity
      - prompt:journey-times
    status:
      - test -f {{.WIDE_OUTPUT}}
    cmds:
      - >-
        uv run python -m pipeline.transform.merge
        --epc-pp {{.EPC_PP_OUTPUT}}
        --arcgis {{.ARCGIS_OUTPUT}}
        --iod {{.IOD_OUTPUT}}
        --poi-proximity {{.POI_PROXIMITY_OUTPUT}}
        --journey-times-bank {{.JOURNEY_TIMES_BANK}}
        --journey-times-fitzrovia {{.JOURNEY_TIMES_FITZROVIA}}
        --ethnicity {{.ETHNICITY_OUTPUT}}
        --crime {{.CRIME_OUTPUT}}
        --noise {{.NOISE_OUTPUT}}
        --school-proximity {{.SCHOOL_PROXIMITY_OUTPUT}}
        --broadband {{.BROADBAND_OUTPUT}}
        --output {{.WIDE_OUTPUT}}
