#!/usr/bin/env bash
# Every 12 hours: scrape rightmove + onthemarket via the finder container,
# then enrich the actual listings. No step is fatal; failures are logged
# loudly and the loop carries on.
#
# Run it in the background with:
#   nohup scripts/scrape-loop.sh >> logs/scrape-loop.log 2>&1 &

REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
INTERVAL_SECONDS=$((12 * 3600))

log() {
    printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
}

fail_loudly() {
    log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
    log "!!! STEP FAILED: $*"
    log '!!! continuing anyway'
    log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
}

run_step() {
    local desc="$1"
    shift
    log "step: $desc"
    if "$@"; then
        log "done: $desc"
        return 0
    else
        fail_loudly "$desc (exit code $?)"
        return 1
    fi
}

while true; do
    log '=== starting scrape + enrich cycle ==='

    cd "$REPO_DIR/finder" || fail_loudly "cd $REPO_DIR/finder"

    run_step 'docker compose up' docker compose up -d
    # A rejected scrape leaves the previous parquet in place, so enriching would
    # just republish the data that is already live, and on a real failure it
    # would hide it. Skip straight to the next cycle instead.
    if run_step 'finder scrape (rightmove, onthemarket)' \
        docker compose exec -T finder uv run python main.py --source rightmove,onthemarket
    then
        cd "$REPO_DIR" || fail_loudly "cd $REPO_DIR"
        run_step 'enrich actual listings' make -f Makefile.data enrich-actual-listings
    else
        log 'skipping enrich: the scrape was rejected and wrote nothing'
    fi

    cd "$REPO_DIR" || fail_loudly "cd $REPO_DIR"

    log "=== cycle finished, sleeping 12h ==="
    sleep "$INTERVAL_SECONDS"
done