scrape loop

scripts/scrape-loop.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+# Every 12 hours: scrape rightmove + onthemarket via the finder container,
+# then enrich the actual listings. No step is fatal; failures are logged
+# loudly and the loop carries on.
+#
+# Run it in the background with:
+#   nohup scripts/scrape-loop.sh >> logs/scrape-loop.log 2>&1 &
+
+REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
+INTERVAL_SECONDS=$((12 * 3600))
+
+log() {
+    printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
+}
+
+fail_loudly() {
+    log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
+    log "!!! STEP FAILED: $*"
+    log '!!! continuing anyway'
+    log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
+}
+
+run_step() {
+    local desc="$1"
+    shift
+    log "step: $desc"
+    if "$@"; then
+        log "done: $desc"
+    else
+        fail_loudly "$desc (exit code $?)"
+    fi
+}
+
+while true; do
+    log '=== starting scrape + enrich cycle ==='
+
+    cd "$REPO_DIR/finder" || fail_loudly "cd $REPO_DIR/finder"
+
+    run_step 'docker compose up' docker compose up -d
+    run_step 'finder scrape (rightmove, onthemarket)' \
+        docker compose exec -T finder uv run python main.py --source rightmove,onthemarket
+
+    cd "$REPO_DIR" || fail_loudly "cd $REPO_DIR"
+
+    run_step 'enrich actual listings' make -f Makefile.data enrich-actual-listings
+
+    log "=== cycle finished, sleeping 12h ==="
+    sleep "$INTERVAL_SECONDS"
+done