121 lines
3.8 KiB
Python
121 lines
3.8 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
ARCGIS_PATH = os.environ.get("ARCGIS_PATH", "/data/arcgis_data.parquet")
|
|
DATA_DIR = Path("/app/data")
|
|
PAGE_SIZE = 24
|
|
DELAY_BETWEEN_PAGES = 0.3
|
|
DELAY_BETWEEN_OUTCODES = 0.5
|
|
MAX_RETRIES = 3
|
|
RETRY_BASE_DELAY = 2.0
|
|
GRID_CELL_SIZE = 0.01 # degrees for postcode spatial index
|
|
MAX_BEDROOMS = 20 # sanity cap — values above this are almost certainly parsing errors
|
|
# Rent sanity bounds (monthly). Rents outside this range are nulled out — they are
|
|
# almost always total-stay pricing (e.g. "Golf Open 2026" short lets), annual rents
|
|
# mislabelled as monthly, or data errors.
|
|
MIN_RENT_MONTHLY = 50 # below £50/month is implausible for any UK property
|
|
MAX_RENT_MONTHLY = 25_000 # above £25k/month covers ultra-prime London; higher is suspect
|
|
SEED = 42
|
|
CHECKPOINT_INTERVAL = int(os.environ.get("CHECKPOINT_INTERVAL", "900")) # seconds
|
|
|
|
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
|
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
|
# Whether to run a scrape immediately on startup
|
|
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
|
|
# Enable/disable individual sources
|
|
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in (
|
|
"1",
|
|
"true",
|
|
"yes",
|
|
)
|
|
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in (
|
|
"1",
|
|
"true",
|
|
"yes",
|
|
)
|
|
SCRAPE_OPENRENT = os.environ.get("SCRAPE_OPENRENT", "true").lower() in (
|
|
"1",
|
|
"true",
|
|
"yes",
|
|
)
|
|
SCRAPE_ZOOPLA = os.environ.get("SCRAPE_ZOOPLA", "true").lower() in (
|
|
"1",
|
|
"true",
|
|
"yes",
|
|
)
|
|
|
|
# URL to trigger server data reload after scrape (e.g. http://server:8001/api/reload)
|
|
RELOAD_URL = os.environ.get("RELOAD_URL", "")
|
|
|
|
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
|
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
|
RIGHTMOVE_BASE = "https://www.rightmove.co.uk"
|
|
|
|
# home.co.uk
|
|
HOMECOUK_BASE = "https://home.co.uk"
|
|
HOMECOUK_API_BASE = f"{HOMECOUK_BASE}/api"
|
|
HOMECOUK_PER_PAGE = 30 # max supported by the API
|
|
|
|
# OpenRent
|
|
OPENRENT_BASE = "https://www.openrent.co.uk"
|
|
|
|
# Zoopla
|
|
ZOOPLA_BASE = "https://www.zoopla.co.uk"
|
|
|
|
PROPERTY_TYPE_MAP = {
|
|
"Detached": "Detached",
|
|
"Semi-Detached": "Semi-Detached",
|
|
"Terraced": "Terraced",
|
|
"End of Terrace": "Terraced",
|
|
"Mid Terrace": "Terraced",
|
|
"Flat": "Flats/Maisonettes",
|
|
"Maisonette": "Flats/Maisonettes",
|
|
"Studio": "Flats/Maisonettes",
|
|
"Apartment": "Flats/Maisonettes",
|
|
"Penthouse": "Flats/Maisonettes",
|
|
"Ground Flat": "Flats/Maisonettes",
|
|
"Duplex": "Flats/Maisonettes",
|
|
"Detached Bungalow": "Detached",
|
|
"Semi-Detached Bungalow": "Semi-Detached",
|
|
"Town House": "Terraced",
|
|
"Link Detached": "Detached",
|
|
"Link Detached House": "Detached",
|
|
"Bungalow": "Other",
|
|
"Cottage": "Other",
|
|
"Park Home": "Other",
|
|
"Mobile Home": "Other",
|
|
"Caravan": "Other",
|
|
"Lodge": "Other",
|
|
"Land": "Other",
|
|
"Farm / Barn": "Other",
|
|
"Farm House": "Other",
|
|
"House": "Detached",
|
|
"House of Multiple Occupation": "Flats/Maisonettes",
|
|
"House Share": "Other",
|
|
"Not Specified": "Other",
|
|
"Chalet": "Other",
|
|
"Barn Conversion": "Other",
|
|
"Coach House": "Other",
|
|
"Character Property": "Other",
|
|
"Cluster House": "Other",
|
|
"Retirement Property": "Flats/Maisonettes",
|
|
"Parking": "Other",
|
|
"Plot": "Other",
|
|
"Garages": "Other",
|
|
"Mews": "Terraced",
|
|
"Property": "Other",
|
|
# Lowercase variants (from home.co.uk / Rightmove APIs)
|
|
"house": "Detached",
|
|
"bungalow": "Other",
|
|
"townhouse": "Terraced",
|
|
"land": "Other",
|
|
"other": "Other",
|
|
"not-specified": "Other",
|
|
"retirement-property": "Flats/Maisonettes",
|
|
"equestrian-facility": "Other",
|
|
}
|
|
|
|
CHANNELS = [
|
|
{"channel": "BUY", "transactionType": "BUY", "sortType": "2"},
|
|
{"channel": "RENT", "transactionType": "LETTING", "sortType": "6"},
|
|
]
|