andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

More fixes

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-12 20:27:04 +00:00
parent 791bc6976b
commit 14a3555cf1
21 changed files with 549 additions and 99 deletions
Download patch file Download diff file Expand all files Collapse all files

5
finder/constants.py
View file

@ -14,7 +14,10 @@ SEED = 42
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
# Whether to run a scrape immediately on startup
RUN_ON_STARTUP = False
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "").lower() in ("1", "true", "yes")
# Enable/disable individual sources
SCRAPE_RIGHTMOVE = os.environ.get("SCRAPE_RIGHTMOVE", "true").lower() in ("1", "true", "yes")
SCRAPE_HOMECOUK = os.environ.get("SCRAPE_HOMECOUK", "true").lower() in ("1", "true", "yes")
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"

4
finder/storage.py
View file

@ -56,7 +56,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
"Address per Property Register": [
p["Address per Property Register"] for p in properties
],
"Leashold/Freehold": [p["Leashold/Freehold"] for p in properties],
"Leasehold/Freehold": [p["Leasehold/Freehold"] for p in properties],
"Property type": [p["Property type"] for p in properties],
"Property sub-type": [p["Property sub-type"] for p in properties],
"Price qualifier": [p["Price qualifier"] for p in properties],
@ -78,7 +78,7 @@ def write_parquet(properties: list[dict], path: Path, channel: str) -> None:
"lat": pl.Float64,
"Postcode": pl.Utf8,
"Address per Property Register": pl.Utf8,
"Leashold/Freehold": pl.Utf8,
"Leasehold/Freehold": pl.Utf8,
"Property type": pl.Utf8,
"Property sub-type": pl.Utf8,
"Price qualifier": pl.Utf8,