49 lines
1.3 KiB
Bash
Executable file
49 lines
1.3 KiB
Bash
Executable file
#!/usr/bin/env bash
|
|
# Every 12 hours: scrape rightmove + onthemarket via the finder container,
|
|
# then enrich the actual listings. No step is fatal; failures are logged
|
|
# loudly and the loop carries on.
|
|
#
|
|
# Run it in the background with:
|
|
# nohup scripts/scrape-loop.sh >> logs/scrape-loop.log 2>&1 &
|
|
|
|
REPO_DIR="$(cd "$(dirname "$0")/.." && pwd)"
|
|
INTERVAL_SECONDS=$((12 * 3600))
|
|
|
|
log() {
|
|
printf '[%s] %s\n' "$(date '+%Y-%m-%d %H:%M:%S')" "$*"
|
|
}
|
|
|
|
fail_loudly() {
|
|
log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
|
|
log "!!! STEP FAILED: $*"
|
|
log '!!! continuing anyway'
|
|
log '!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!!'
|
|
}
|
|
|
|
run_step() {
|
|
local desc="$1"
|
|
shift
|
|
log "step: $desc"
|
|
if "$@"; then
|
|
log "done: $desc"
|
|
else
|
|
fail_loudly "$desc (exit code $?)"
|
|
fi
|
|
}
|
|
|
|
while true; do
|
|
log '=== starting scrape + enrich cycle ==='
|
|
|
|
cd "$REPO_DIR/finder" || fail_loudly "cd $REPO_DIR/finder"
|
|
|
|
run_step 'docker compose up' docker compose up -d
|
|
run_step 'finder scrape (rightmove, onthemarket)' \
|
|
docker compose exec -T finder uv run python main.py --source rightmove,onthemarket
|
|
|
|
cd "$REPO_DIR" || fail_loudly "cd $REPO_DIR"
|
|
|
|
run_step 'enrich actual listings' make -f Makefile.data enrich-actual-listings
|
|
|
|
log "=== cycle finished, sleeping 12h ==="
|
|
sleep "$INTERVAL_SECONDS"
|
|
done
|