69 lines
2.3 KiB
Python
69 lines
2.3 KiB
Python
import os
|
|
from pathlib import Path
|
|
|
|
ARCGIS_PATH = os.environ.get("ARCGIS_PATH", "/data/arcgis_data.parquet")
|
|
DATA_DIR = Path("/app/data")
|
|
PAGE_SIZE = 24
|
|
DELAY_BETWEEN_PAGES = 1.0
|
|
DELAY_BETWEEN_OUTCODES = 2.0
|
|
MAX_RETRIES = 3
|
|
RETRY_BASE_DELAY = 2.0
|
|
GRID_CELL_SIZE = 0.01 # degrees for postcode spatial index
|
|
SEED = 42
|
|
|
|
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
|
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
|
# Whether to run a scrape immediately on startup
|
|
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
|
|
# Enable/disable individual sources
|
|
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes")
|
|
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes")
|
|
|
|
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
|
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
|
RIGHTMOVE_BASE = "https://www.rightmove.co.uk"
|
|
|
|
# home.co.uk
|
|
HOMECOUK_BASE = "https://home.co.uk"
|
|
HOMECOUK_API_BASE = f"{HOMECOUK_BASE}/api"
|
|
HOMECOUK_PER_PAGE = 30 # max supported by the API
|
|
|
|
PROPERTY_TYPE_MAP = {
|
|
"Detached": "Detached",
|
|
"Semi-Detached": "Semi-Detached",
|
|
"Terraced": "Terraced",
|
|
"End of Terrace": "Terraced",
|
|
"Mid Terrace": "Terraced",
|
|
"Flat": "Flats/Maisonettes",
|
|
"Maisonette": "Flats/Maisonettes",
|
|
"Studio": "Flats/Maisonettes",
|
|
"Apartment": "Flats/Maisonettes",
|
|
"Penthouse": "Flats/Maisonettes",
|
|
"Ground Flat": "Flats/Maisonettes",
|
|
"Detached Bungalow": "Detached",
|
|
"Semi-Detached Bungalow": "Semi-Detached",
|
|
"Town House": "Terraced",
|
|
"Link Detached": "Detached",
|
|
"Link Detached House": "Detached",
|
|
"Bungalow": "Other",
|
|
"Cottage": "Other",
|
|
"Park Home": "Other",
|
|
"Land": "Other",
|
|
"Farm / Barn": "Other",
|
|
"House": "Detached",
|
|
"Not Specified": "Other",
|
|
"Chalet": "Other",
|
|
"Barn Conversion": "Other",
|
|
"Coach House": "Other",
|
|
"Character Property": "Other",
|
|
"Cluster House": "Other",
|
|
"Retirement Property": "Flats/Maisonettes",
|
|
"Plot": "Other",
|
|
"Garages": "Other",
|
|
"Mews": "Terraced",
|
|
}
|
|
|
|
CHANNELS = [
|
|
{"channel": "BUY", "transactionType": "BUY", "sortType": "2"},
|
|
{"channel": "RENT", "transactionType": "LETTING", "sortType": "6"},
|
|
]
|