..
This commit is contained in:
parent
1ee796b282
commit
ab688243d7
36 changed files with 307 additions and 135 deletions
|
|
@ -63,7 +63,7 @@ def _fetch_csv(
|
|||
"""Download a CSV, defending against silent truncation.
|
||||
|
||||
The NOMIS bulk files are served with chunked transfer encoding and no
|
||||
Content-Length, so a dropped connection ends the stream without raising —
|
||||
Content-Length, so a dropped connection ends the stream without raising,
|
||||
yielding a short file. We retry until the parsed row count reaches the
|
||||
file's known approximate size, keeping the largest parse seen.
|
||||
"""
|
||||
|
|
@ -72,7 +72,7 @@ def _fetch_csv(
|
|||
try:
|
||||
download(url, dest)
|
||||
df = pl.read_csv(dest)
|
||||
except Exception as exc: # noqa: BLE001 — retry any transport/parse error
|
||||
except Exception as exc: # noqa: BLE001: retry any transport/parse error
|
||||
print(f" {url} attempt {attempt}: error {exc}")
|
||||
time.sleep(3)
|
||||
continue
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue