andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Cache postcodes

Browse source
This commit is contained in:
Andras Schmelczer 2026-06-14 14:50:38 +01:00
parent f59d01227b
commit 8e4c56bb0d
7 changed files with 502 additions and 141 deletions
Download patch file Download diff file Expand all files Collapse all files

25
finder/constants.py
View file

@ -15,6 +15,16 @@ DELAY_BETWEEN_PAGES = 0.3
DELAY_BETWEEN_OUTCODES = 0.5
MAX_RETRIES = 3
RETRY_BASE_DELAY = 2.0
# Concurrency + global rate limiting for the HTTP-based scrapers (Rightmove and
# OnTheMarket). Detail-page fetches dominate runtime and are independent,
# idempotent GETs, so they are fetched concurrently rather than one-at-a-time.
# A single shared token-bucket limiter (http_client.RATE_LIMITER) caps the
# COMBINED request rate across every worker thread and both providers, so the
# VPN egress IP stays polite no matter how high the concurrency is. Tune both
# down if the portals start returning 429/403.
DETAIL_FETCH_CONCURRENCY = int(os.environ.get("DETAIL_FETCH_CONCURRENCY", "8"))
REQUESTS_PER_SECOND = float(os.environ.get("REQUESTS_PER_SECOND", "10"))
GRID_CELL_SIZE = 0.01 # degrees for postcode spatial index
MAX_BEDROOMS = 20 # sanity cap — values above this are almost certainly parsing errors
@ -35,6 +45,21 @@ RIGHTMOVE_DETAIL_URL = "https://www.rightmove.co.uk/properties/{id}"
RIGHTMOVE_FETCH_DETAILS = True # fetch detail pages for true per-listing postcodes
RIGHTMOVE_MAX_DETAILS_PER_OUTCODE = 4000 # max detail-page fetches per outcode
# Skip the detail fetch when the search result already pins the property
# precisely. Each search-result `location` carries a `pinType`: for
# "ACCURATE_POINT" listings the coordinates are rooftop-exact, so the
# coordinate-nearest postcode already has the right outcode (and almost always
# the right unit) and the detail page adds little. The detail fetch earns its
# keep on APPROXIMATE pins (new-builds/developments) where Rightmove
# deliberately fuzzes the coordinates. Degrades safely: when `pinType` is absent
# from the search payload, nothing is skipped (behaviour is unchanged), so this
# is only a speed-up to the extent the field is present — verify against a live
# search response before relying on the saving.
RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS = (
os.environ.get("RIGHTMOVE_SKIP_DETAILS_FOR_ACCURATE_PINS", "1") != "0"
)
RIGHTMOVE_ACCURATE_PIN_TYPE = "ACCURATE_POINT"
# OnTheMarket
ONTHEMARKET_BASE = "https://www.onthemarket.com"