andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Faster scraping

Browse source
This commit is contained in:
Andras Schmelczer 2026-03-20 07:52:22 +00:00
parent 05b8ee06c1
commit 852bb3f3a7
4 changed files with 437 additions and 324 deletions
Download patch file Download diff file Expand all files Collapse all files

19
finder/openrent.py
View file

@ -788,7 +788,24 @@ def search_outcode(
for search_data in search_results:
detail_data = None
if fetch_details and search_data.get("url"):
# Skip detail page if we already have coordinates or a resolvable postcode
has_coords = (
search_data.get("lat") is not None
and search_data.get("lng") is not None
)
has_resolvable_pc = (
search_data.get("postcode")
and pc_coords
and search_data["postcode"] in pc_coords
)
needs_detail = (
fetch_details
and search_data.get("url")
and not has_coords
and not has_resolvable_pc
)
if needs_detail:
detail_html = fetch_page(client, search_data["url"])
if detail_html:
detail_data = parse_property_detail(detail_html)