fix flaky crime download

pipeline/download/test_crime.py

@@ -1,7 +1,13 @@
+import hashlib
+import shutil
 from zipfile import ZipFile
 
+import pytest
+
+from pipeline.download import crime
 from pipeline.download.crime import (
     CrimeArchive,
+    download_archive,
     extract_csvs,
     prepare_archive_dir,
     prune_unused_csvs,
@@ -10,6 +16,19 @@ from pipeline.download.crime import (
 )
 
 
+def _make_archive(md5: str | None = None) -> CrimeArchive:
+    return CrimeArchive(
+        month="2023-05",
+        label="May 2023",
+        url="https://data.police.uk/data/archive/2023-05.zip",
+        filename="2023-05.zip",
+        size="1.6 GB",
+        contained_range="",
+        md5=md5,
+        raw_md5=md5 or "",
+    )
+
+
 def test_parse_archives_reads_monthly_zip_links_only():
     html = """
     <p><a href="/data/archive/latest.zip">latest.zip</a></p>
@@ -194,3 +213,47 @@ def test_prune_unused_csvs_removes_non_street_csvs(tmp_path):
     assert street.exists()
     assert not outcomes.exists()
     assert not stop_search.exists()
+
+
+def test_download_archive_recovers_when_workdir_wiped_mid_download(
+    tmp_path, monkeypatch
+):
+    # Reproduces the crash where a completed .part vanished before the rename
+    # (a concurrent run wiped _download_tmp), so partial.replace raised ENOENT
+    # even though the download had reached 100%.
+    monkeypatch.setattr(crime.time, "sleep", lambda *_: None)
+    archive_dir = tmp_path / "_download_tmp"
+    archive_dir.mkdir()
+    payload = b"real-zip-bytes" * 4096
+    archive = _make_archive(md5=hashlib.md5(payload).hexdigest())
+
+    calls = {"n": 0}
+
+    def fake_stream(_archive, partial, *, timeout):
+        calls["n"] += 1
+        partial.write_bytes(payload)
+        if calls["n"] == 1:
+            # Simulate the concurrent cleanup: the rename target disappears.
+            shutil.rmtree(partial.parent)
+
+    monkeypatch.setattr(crime, "_stream_to_partial", fake_stream)
+
+    dest = download_archive(archive, archive_dir, verify=True, force=False, timeout=1.0)
+
+    assert calls["n"] == 2
+    assert dest.read_bytes() == payload
+
+
+def test_download_archive_gives_up_after_repeated_failures(tmp_path, monkeypatch):
+    monkeypatch.setattr(crime.time, "sleep", lambda *_: None)
+    archive_dir = tmp_path / "_download_tmp"
+
+    def always_fail(_archive, _partial, *, timeout):
+        raise OSError("connection reset")
+
+    monkeypatch.setattr(crime, "_stream_to_partial", always_fail)
+
+    with pytest.raises(RuntimeError, match=f"after {crime.DOWNLOAD_ATTEMPTS} attempts"):
+        download_archive(
+            _make_archive(), archive_dir, verify=False, force=False, timeout=1.0
+        )