This commit is contained in:
Andras Schmelczer 2026-07-03 18:39:34 +01:00
parent 1ee796b282
commit ab688243d7
36 changed files with 307 additions and 135 deletions

View file

@ -63,7 +63,7 @@ def _fetch_csv(
"""Download a CSV, defending against silent truncation.
The NOMIS bulk files are served with chunked transfer encoding and no
Content-Length, so a dropped connection ends the stream without raising
Content-Length, so a dropped connection ends the stream without raising,
yielding a short file. We retry until the parsed row count reaches the
file's known approximate size, keeping the largest parse seen.
"""
@ -72,7 +72,7 @@ def _fetch_csv(
try:
download(url, dest)
df = pl.read_csv(dest)
except Exception as exc: # noqa: BLE001 retry any transport/parse error
except Exception as exc: # noqa: BLE001: retry any transport/parse error
print(f" {url} attempt {attempt}: error {exc}")
time.sleep(3)
continue