Remove finder
This commit is contained in:
parent
55238f59aa
commit
cd778dd088
26 changed files with 0 additions and 57826 deletions
|
|
@ -1,154 +0,0 @@
|
|||
import os
|
||||
from pathlib import Path
|
||||
|
||||
ARCGIS_PATH = os.environ.get("ARCGIS_PATH", "/data/arcgis_data.parquet")
|
||||
DATA_DIR = Path("/app/data")
|
||||
PAGE_SIZE = 24
|
||||
DELAY_BETWEEN_PAGES = 0.3
|
||||
DELAY_BETWEEN_OUTCODES = 0.5
|
||||
MAX_RETRIES = 3
|
||||
RETRY_BASE_DELAY = 2.0
|
||||
GRID_CELL_SIZE = 0.01 # degrees for postcode spatial index
|
||||
MAX_BEDROOMS = 20 # sanity cap — values above this are almost certainly parsing errors
|
||||
# Rent sanity bounds (monthly). Rents outside this range are nulled out — they are
|
||||
# almost always total-stay pricing (e.g. "Golf Open 2026" short lets), annual rents
|
||||
# mislabelled as monthly, or data errors.
|
||||
MIN_RENT_MONTHLY = 50 # below £50/month is implausible for any UK property
|
||||
MAX_RENT_MONTHLY = 25_000 # above £25k/month covers ultra-prime London; higher is suspect
|
||||
SEED = 42
|
||||
CHECKPOINT_INTERVAL = int(os.environ.get("CHECKPOINT_INTERVAL", "900")) # seconds
|
||||
|
||||
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
||||
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
||||
# Whether to run a scrape immediately on startup
|
||||
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
|
||||
# Enable/disable individual sources
|
||||
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
SCRAPE_OPENRENT = os.environ.get("SCRAPE_OPENRENT", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
SCRAPE_ZOOPLA = os.environ.get("SCRAPE_ZOOPLA", "true").lower() in (
|
||||
"1",
|
||||
"true",
|
||||
"yes",
|
||||
)
|
||||
|
||||
# URL to trigger server data reload after scrape (e.g. http://server:8001/api/reload)
|
||||
RELOAD_URL = os.environ.get("RELOAD_URL", "")
|
||||
|
||||
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
||||
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
||||
RIGHTMOVE_BASE = "https://www.rightmove.co.uk"
|
||||
|
||||
# home.co.uk
|
||||
HOMECOUK_BASE = "https://home.co.uk"
|
||||
HOMECOUK_API_BASE = f"{HOMECOUK_BASE}/api"
|
||||
HOMECOUK_PER_PAGE = 30 # max supported by the API
|
||||
HOMECOUK_CONCURRENCY = int(os.environ.get("HOMECOUK_CONCURRENCY", "4"))
|
||||
|
||||
# OpenRent
|
||||
OPENRENT_BASE = "https://www.openrent.co.uk"
|
||||
|
||||
# Zoopla
|
||||
ZOOPLA_BASE = "https://www.zoopla.co.uk"
|
||||
|
||||
PROPERTY_TYPE_MAP = {
|
||||
"Detached": "Detached",
|
||||
"Semi-Detached": "Semi-Detached",
|
||||
"Terraced": "Terraced",
|
||||
"End of Terrace": "Terraced",
|
||||
"Mid Terrace": "Terraced",
|
||||
"Flat": "Flats/Maisonettes",
|
||||
"Maisonette": "Flats/Maisonettes",
|
||||
"Studio": "Flats/Maisonettes",
|
||||
"Apartment": "Flats/Maisonettes",
|
||||
"Penthouse": "Flats/Maisonettes",
|
||||
"Ground Flat": "Flats/Maisonettes",
|
||||
"Duplex": "Flats/Maisonettes",
|
||||
"Detached Bungalow": "Detached",
|
||||
"Semi-Detached Bungalow": "Semi-Detached",
|
||||
"Town House": "Terraced",
|
||||
"Link Detached": "Detached",
|
||||
"Link Detached House": "Detached",
|
||||
"Bungalow": "Other",
|
||||
"Cottage": "Other",
|
||||
"Park Home": "Other",
|
||||
"Mobile Home": "Other",
|
||||
"Caravan": "Other",
|
||||
"Lodge": "Other",
|
||||
"Land": "Other",
|
||||
"Farm / Barn": "Other",
|
||||
"Farm House": "Other",
|
||||
"House": "Detached",
|
||||
"House of Multiple Occupation": "Flats/Maisonettes",
|
||||
"House Share": "Other",
|
||||
"Not Specified": "Other",
|
||||
"Chalet": "Other",
|
||||
"Barn Conversion": "Other",
|
||||
"Coach House": "Other",
|
||||
"Character Property": "Other",
|
||||
"Cluster House": "Other",
|
||||
"Retirement Property": "Flats/Maisonettes",
|
||||
"Parking": "Other",
|
||||
"Plot": "Other",
|
||||
"Garages": "Other",
|
||||
"Mews": "Terraced",
|
||||
"Property": "Other",
|
||||
"Flat Share": "Other",
|
||||
"Block of Apartments": "Flats/Maisonettes",
|
||||
"Private Halls": "Flats/Maisonettes",
|
||||
"Terraced Bungalow": "Terraced",
|
||||
"Equestrian Facility": "Other",
|
||||
"Ground Maisonette": "Flats/Maisonettes",
|
||||
"Country House": "Detached",
|
||||
"Village House": "Detached",
|
||||
"Farm Land": "Other",
|
||||
"House Boat": "Other",
|
||||
"Barn": "Other",
|
||||
"Serviced Apartments": "Flats/Maisonettes",
|
||||
# Space-separated variants (from home.co.uk underscore/hyphen normalization)
|
||||
"Semi Detached": "Semi-Detached",
|
||||
"Semi Detached Bungalow": "Semi-Detached",
|
||||
"End Of Terrace": "Terraced",
|
||||
"End Terrace": "Terraced",
|
||||
"Block Of Apartments": "Flats/Maisonettes",
|
||||
"Farm / Barn": "Other",
|
||||
# Lowercase variants (from home.co.uk / Rightmove APIs)
|
||||
"house": "Detached",
|
||||
"bungalow": "Other",
|
||||
"townhouse": "Terraced",
|
||||
"land": "Other",
|
||||
"other": "Other",
|
||||
"not-specified": "Other",
|
||||
"retirement-property": "Flats/Maisonettes",
|
||||
"equestrian-facility": "Other",
|
||||
"flat": "Flats/Maisonettes",
|
||||
"detached": "Detached",
|
||||
"semi-detached": "Semi-Detached",
|
||||
"terraced": "Terraced",
|
||||
"maisonette": "Flats/Maisonettes",
|
||||
"apartment": "Flats/Maisonettes",
|
||||
"studio": "Flats/Maisonettes",
|
||||
"penthouse": "Flats/Maisonettes",
|
||||
"cottage": "Other",
|
||||
"chalet": "Other",
|
||||
"farm_house": "Detached",
|
||||
"country house": "Detached",
|
||||
"village house": "Detached",
|
||||
}
|
||||
|
||||
CHANNELS = [
|
||||
{"channel": "BUY", "transactionType": "BUY", "sortType": "2"},
|
||||
{"channel": "RENT", "transactionType": "LETTING", "sortType": "6"},
|
||||
]
|
||||
Loading…
Add table
Add a link
Reference in a new issue