diff --git a/scripts/scrape-loop.sh b/scripts/scrape-loop.sh new file mode 100755 index 0000000..65605e3 --- /dev/null +++ b/scripts/scrape-loop.sh @@ -0,0 +1,49 @@ +#!/usr/bin/env bash +# Every 12 hours: scrape rightmove + onthemarket via the finder container, +# then enrich the actual listings. No step is fatal; failures are logged +# loudly and the loop carries on. +# +# Run it in the background with: +# nohup scripts/scrape-loop.sh >> logs/scrape-loop.log 2>&1 & + +REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)" +INTERVAL_SECONDS=$((12 * 3600)) + +log() { + printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*" +} + +fail_loudly() { + log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' + log "!!! STEP FAILED: $*" + log '!!! continuing anyway' + log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!' +} + +run_step() { + local desc="$1" + shift + log "step: $desc" + if "$@"; then + log "done: $desc" + else + fail_loudly "$desc (exit code $?)" + fi +} + +while true; do + log '=== starting scrape + enrich cycle ===' + + cd "$REPO_DIR/finder" || fail_loudly "cd $REPO_DIR/finder" + + run_step 'docker compose up' docker compose up -d + run_step 'finder scrape (rightmove, onthemarket)' \ + docker compose exec -T finder uv run python main.py --source rightmove,onthemarket + + cd "$REPO_DIR" || fail_loudly "cd $REPO_DIR" + + run_step 'enrich actual listings' make -f Makefile.data enrich-actual-listings + + log "=== cycle finished, sleeping 12h ===" + sleep "$INTERVAL_SECONDS" +done