perfect-postcode/pipeline/download/test_transit_network.py
2026-07-12 20:37:39 +01:00

536 lines
20 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters

This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.

"""Tests for transit_network GTFS processing."""
import datetime as dt
import zipfile
from pathlib import Path
import pytest
from pipeline.download.transit_network import (
STATION_COORD_OVERRIDES,
_repair_stop_coordinate,
clean_national_rail_gtfs,
convert_high_freq_to_frequency_based,
validate_gtfs_feed,
validate_london_coverage,
validate_stop_geometry,
)
def _write_gtfs(path: Path, *, stop_times: str) -> None:
"""Write a minimal GTFS zip with one metro route and several trips."""
routes = "route_id,route_type\nR1,1\n"
trips = "trip_id,route_id,direction_id,service_id\n" + "".join(
f"T{i},R1,0,S1\n" for i in range(1, 7)
)
with zipfile.ZipFile(path, "w") as z:
z.writestr("routes.txt", routes)
z.writestr("trips.txt", trips)
z.writestr("stop_times.txt", stop_times)
def _one_based_stop_times() -> str:
"""Six trips, 1-based stop_sequence (1,2,...), 5-minute headway."""
header = "trip_id,stop_sequence,departure_time,stop_id\n"
rows = []
# First departures 06:00, 06:05, ... (300s = 5 min headway, well under 15 min)
for i in range(6):
trip = f"T{i + 1}"
first_dep = 6 * 3600 + i * 300
h, m = divmod(first_dep, 3600)
m, s = divmod(m, 60)
# First stop has stop_sequence 1 (NOT 0); second stop sequence 2.
rows.append(f"{trip},1,{h:02d}:{m:02d}:{s:02d},STOP_A\n")
later = first_dep + 120
h2, m2 = divmod(later, 3600)
m2, s2 = divmod(m2, 60)
rows.append(f"{trip},2,{h2:02d}:{m2:02d}:{s2:02d},STOP_B\n")
return header + "".join(rows)
def test_one_based_stop_sequence_is_converted(tmp_path: Path) -> None:
"""First stop selection must use the minimum stop_sequence, not literal "0".
With 1-based stop_sequence the old code (keyed on stop_sequence == "0") found
zero first stops and produced an empty frequencies.txt. The fix selects the
minimum stop_sequence per trip, so the high-frequency group is converted.
"""
src = tmp_path / "in.zip"
dst = tmp_path / "out.zip"
_write_gtfs(src, stop_times=_one_based_stop_times())
convert_high_freq_to_frequency_based(src, dst)
with zipfile.ZipFile(dst, "r") as z:
freq = z.read("frequencies.txt").decode("utf-8")
freq_rows = [r for r in freq.splitlines()[1:] if r.strip()]
# The single high-frequency group must produce exactly one frequency entry.
assert len(freq_rows) == 1, freq
trip_id, start_time, end_time, headway_secs, _exact = freq_rows[0].split(",")
# Template trip is the earliest departure (T1 at 06:00) starting at first stop.
assert start_time == "06:00:00"
# Median headway of 300s rounds to a 300s headway entry.
assert headway_secs == "300"
def test_clean_national_rail_gtfs_orders_by_stop_sequence_not_file_order(
tmp_path: Path,
) -> None:
"""dtd2mysql exports happen to be ordered by stop_sequence within each
trip, but nothing guarantees it. Rows arriving out of order must be sorted
by their original stop_sequence before the backwards-time check and the
0-based renumbering. File order would flag the trip as backwards and drop
it (or scramble the stop order)."""
src = tmp_path / "in.zip"
dst = tmp_path / "out.zip"
with zipfile.ZipFile(src, "w") as z:
z.writestr(
"stops.txt",
"stop_id,stop_lat,stop_lon\nSTOP_A,51.5,-0.1\nSTOP_B,51.6,-0.1\n",
)
z.writestr("routes.txt", "route_id,route_type\nR1,2\n")
z.writestr("trips.txt", "trip_id,route_id,service_id\nT1,R1,S1\n")
# File order is seq 2 then seq 1: in file order departures look
# backwards (07:00 then 06:00); in sequence order they are fine.
z.writestr(
"stop_times.txt",
"trip_id,stop_id,stop_sequence,departure_time\n"
"T1,STOP_B,2,07:00:00\n"
"T1,STOP_A,1,06:00:00\n",
)
clean_national_rail_gtfs(src, dst)
with zipfile.ZipFile(dst, "r") as z:
stop_times = z.read("stop_times.txt").decode("utf-8").splitlines()
trips = z.read("trips.txt").decode("utf-8").splitlines()
assert trips == ["trip_id,route_id,service_id", "T1,R1,S1"]
assert stop_times == [
"trip_id,stop_id,stop_sequence,departure_time",
"T1,STOP_A,0,06:00:00",
"T1,STOP_B,1,07:00:00",
]
def test_raises_when_no_first_stops_found(tmp_path: Path) -> None:
"""A non-empty target trip set with unparseable stop_sequence is loud, not silent."""
src = tmp_path / "in.zip"
dst = tmp_path / "out.zip"
bad = (
"trip_id,stop_sequence,departure_time,stop_id\n"
"T1,not_a_number,06:00:00,STOP_A\n"
)
_write_gtfs(src, stop_times=bad)
with pytest.raises(RuntimeError, match="no first stops"):
convert_high_freq_to_frequency_based(src, dst)
# ── validate_gtfs_feed ────────────────────────────────────────────────────────
TODAY = dt.date(2026, 6, 10)
def _make_gtfs(
path: Path,
*,
calendar: str | None = (
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,0,0,20260101,20271231\n"
),
calendar_dates: str | None = None,
stops: str = (
"stop_id,stop_name,stop_lat,stop_lon\n"
"STOP_A,Bank,51.5133,-0.0886\n"
"STOP_B,Liverpool Street,51.5178,-0.0823\n"
),
routes: str = "route_id,agency_id,route_short_name,route_type\nR1,OP1,Central,1\n",
trips: str = "trip_id,route_id,service_id\nT1,R1,S1\n",
stop_times: str = (
"trip_id,stop_sequence,departure_time,stop_id\n"
"T1,0,06:00:00,STOP_A\n"
"T1,1,06:02:00,STOP_B\n"
),
) -> Path:
"""Write a tiny synthetic GTFS zip; defaults form a valid current feed."""
with zipfile.ZipFile(path, "w") as z:
if calendar is not None:
z.writestr("calendar.txt", calendar)
if calendar_dates is not None:
z.writestr("calendar_dates.txt", calendar_dates)
z.writestr("stops.txt", stops)
z.writestr("routes.txt", routes)
z.writestr("trips.txt", trips)
z.writestr("stop_times.txt", stop_times)
return path
def test_validate_gtfs_feed_happy_path(tmp_path: Path) -> None:
feed = _make_gtfs(tmp_path / "feed.zip")
validate_gtfs_feed(feed, "test feed", today=TODAY) # must not raise
def test_validate_gtfs_feed_expired_calendar(tmp_path: Path) -> None:
"""The 2010 TfL snapshot failure mode: all calendars ended years ago."""
feed = _make_gtfs(
tmp_path / "feed.zip",
calendar=(
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,0,0,20091201,20101224\n"
),
)
with pytest.raises(RuntimeError, match=r"'stale tfl'.*no service active"):
validate_gtfs_feed(feed, "stale tfl", today=TODAY)
def test_validate_gtfs_feed_calendar_starting_after_window_fails(
tmp_path: Path,
) -> None:
feed = _make_gtfs(
tmp_path / "feed.zip",
calendar=(
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,0,0,20270101,20271231\n"
),
)
with pytest.raises(RuntimeError, match="no service active"):
validate_gtfs_feed(feed, "future feed", today=TODAY)
def test_validate_gtfs_feed_calendar_dates_rescues_expired_calendar(
tmp_path: Path,
) -> None:
"""An expired calendar.txt passes if calendar_dates.txt adds service now."""
feed = _make_gtfs(
tmp_path / "feed.zip",
calendar=(
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,0,0,20091201,20101224\n"
),
calendar_dates="service_id,date,exception_type\nS1,20260615,1\n",
)
validate_gtfs_feed(feed, "rescued feed", today=TODAY) # must not raise
def test_validate_gtfs_feed_removed_service_exception_does_not_count(
tmp_path: Path,
) -> None:
feed = _make_gtfs(
tmp_path / "feed.zip",
calendar=None,
calendar_dates="service_id,date,exception_type\nS1,20260615,2\n",
)
with pytest.raises(RuntimeError, match="no service active"):
validate_gtfs_feed(feed, "removed-only feed", today=TODAY)
def test_validate_gtfs_feed_zero_and_empty_coords(tmp_path: Path) -> None:
"""The 2010 TfL snapshot's other failure mode: empty or 0,0 stop coords."""
feed = _make_gtfs(
tmp_path / "feed.zip",
stops=(
"stop_id,stop_name,stop_lat,stop_lon\nSTOP_A,Nowhere,0,0\nSTOP_B,Blank,,\n"
),
)
with pytest.raises(RuntimeError, match=r"plausible UK coordinates"):
validate_gtfs_feed(feed, "coordless feed", today=TODAY)
def test_validate_gtfs_feed_non_uk_coords_fail(tmp_path: Path) -> None:
feed = _make_gtfs(
tmp_path / "feed.zip",
stops=(
"stop_id,stop_name,stop_lat,stop_lon\n"
"STOP_A,New York,40.71,-74.0\n"
"STOP_B,Sydney,-33.87,151.21\n"
),
)
with pytest.raises(RuntimeError, match="plausible UK coordinates"):
validate_gtfs_feed(feed, "abroad feed", today=TODAY)
def test_validate_gtfs_feed_minority_bad_coords_pass(tmp_path: Path) -> None:
"""One bad stop out of 30 (3.3%) stays under the 5% tolerance."""
rows = [f"STOP_{i},Stop {i},51.5,{-0.1 + i * 0.001}\n" for i in range(29)]
rows.append("STOP_BAD,Broken,0,0\n")
feed = _make_gtfs(
tmp_path / "feed.zip",
stops="stop_id,stop_name,stop_lat,stop_lon\n" + "".join(rows),
)
validate_gtfs_feed(feed, "mostly good feed", today=TODAY) # must not raise
def test_validate_gtfs_feed_empty_trips(tmp_path: Path) -> None:
feed = _make_gtfs(tmp_path / "feed.zip", trips="trip_id,route_id,service_id\n")
with pytest.raises(RuntimeError, match="trips.txt has no data rows"):
validate_gtfs_feed(feed, "tripless feed", today=TODAY)
def test_validate_gtfs_feed_missing_calendar_files(tmp_path: Path) -> None:
feed = _make_gtfs(tmp_path / "feed.zip", calendar=None)
with pytest.raises(RuntimeError, match="neither calendar.txt nor calendar_dates"):
validate_gtfs_feed(feed, "calendarless feed", today=TODAY)
def test_validate_gtfs_feed_not_a_zip(tmp_path: Path) -> None:
bogus = tmp_path / "feed.zip"
bogus.write_text("not a zip")
with pytest.raises(RuntimeError, match="not a valid zip"):
validate_gtfs_feed(bogus, "bogus feed", today=TODAY)
# ── _repair_stop_coordinate ───────────────────────────────────────────────────
@pytest.mark.parametrize(
"stop_id,lat,lon,expected",
[
# Known-bad stations get an authoritative override (TCR ships transposed,
# BDS ships a wrong-signed longitude; both are in STATION_COORD_OVERRIDES).
("TCR", -0.1306, 51.5163, (*STATION_COORD_OVERRIDES["TCR"], "override")),
("BDS", 51.514, 0.15, (*STATION_COORD_OVERRIDES["BDS"], "override")),
# A plausible UK coordinate is left untouched.
("ZFD", 51.5205, -0.1050, (51.5205, -0.1050, "keep")),
# An unknown station with lat/lon transposed is swapped back generically.
("ZZZ", -0.13, 51.51, (51.51, -0.13, "transpose")),
# Genuinely out-of-area garbage (Irish CIE South Atlantic) is neutralised.
("IEP", -4.172, -14.5154, (54.0, -2.0, "dump")),
# Missing coordinates are neutralised too.
("NUL", None, None, (54.0, -2.0, "dump")),
],
)
def test_repair_stop_coordinate(stop_id, lat, lon, expected) -> None:
assert _repair_stop_coordinate(stop_id, lat, lon) == expected
def test_clean_national_rail_repairs_broken_station_coords(tmp_path: Path) -> None:
"""End-to-end: the cleaner repairs the exact TCR/BDS failure modes.
TCR (transposed) and BDS (wrong-signed lon) are corrected to their override
coordinates; an unknown transposed stop is swapped back; genuine out-of-area
garbage is dumped; a good coordinate is preserved.
"""
src = tmp_path / "in.zip"
dst = tmp_path / "out.zip"
stops = (
"stop_id,stop_name,stop_lat,stop_lon\n"
"TCR,Tottenham Court Road (Elizabeth line),-0.1306,51.5163\n"
"BDS,Bond Street (Elizabeth line),51.514,0.15\n"
"ZZZ,Transposed Halt,-0.20,51.40\n"
"IEP,Cork (CIE),-4.172,-14.5154\n"
"GUD,Good Station,51.50,-0.10\n"
)
with zipfile.ZipFile(src, "w") as z:
z.writestr("stops.txt", stops)
z.writestr("routes.txt", "route_id,route_type\nR1,2\n")
z.writestr("trips.txt", "trip_id,route_id,service_id\nT1,R1,S1\n")
z.writestr(
"stop_times.txt",
"trip_id,stop_id,stop_sequence,departure_time\n"
"T1,TCR,1,06:00:00\n"
"T1,BDS,2,06:03:00\n"
"T1,GUD,3,06:06:00\n",
)
clean_national_rail_gtfs(src, dst)
with zipfile.ZipFile(dst, "r") as z:
rows = z.read("stops.txt").decode("utf-8").splitlines()
coords = {r.split(",")[0]: r.split(",")[-2:] for r in rows[1:]}
assert (
float(coords["TCR"][0]),
float(coords["TCR"][1]),
) == STATION_COORD_OVERRIDES["TCR"]
assert (
float(coords["BDS"][0]),
float(coords["BDS"][1]),
) == STATION_COORD_OVERRIDES["BDS"]
assert (float(coords["ZZZ"][0]), float(coords["ZZZ"][1])) == (51.40, -0.20)
assert (float(coords["IEP"][0]), float(coords["IEP"][1])) == (54.0, -2.0)
assert (float(coords["GUD"][0]), float(coords["GUD"][1])) == (51.50, -0.10)
# ── validate_stop_geometry ────────────────────────────────────────────────────
def _geometry_gtfs(path: Path, *, n_trips: int, b_lat: float, b_lon: float) -> Path:
"""A metro line ABC (5-minute hops) repeated across n_trips.
Displacing B far from A and C makes it a displacement outlier; n_trips sets
its service level (the hard-fail vs warn tier).
"""
routes = "route_id,route_type\nR1,1\n"
trips = "trip_id,route_id,service_id\n" + "".join(
f"T{i},R1,S1\n" for i in range(n_trips)
)
stops = (
"stop_id,stop_name,stop_lat,stop_lon\n"
"A,Aaa,51.50,-0.10\n"
f"B,Bbb,{b_lat},{b_lon}\n"
"C,Ccc,51.52,-0.10\n"
)
header = "trip_id,stop_id,stop_sequence,arrival_time,departure_time\n"
body = "".join(
f"T{i},A,0,06:00:00,06:00:00\n"
f"T{i},B,1,06:05:00,06:05:00\n"
f"T{i},C,2,06:10:00,06:10:00\n"
for i in range(n_trips)
)
with zipfile.ZipFile(path, "w") as z:
z.writestr("routes.txt", routes)
z.writestr("trips.txt", trips)
z.writestr("stops.txt", stops)
z.writestr("stop_times.txt", header + body)
return path
def test_validate_stop_geometry_fails_on_high_service_displacement(
tmp_path: Path,
) -> None:
"""A busy stop whose trains imply teleportation fails the build (TCR mode)."""
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=120, b_lat=58.0, b_lon=-2.0)
with pytest.raises(RuntimeError, match="stop-geometry validation failed"):
validate_stop_geometry(feed, "displaced feed")
def test_validate_stop_geometry_passes_when_stops_are_coherent(
tmp_path: Path,
) -> None:
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=120, b_lat=51.51, b_lon=-0.10)
validate_stop_geometry(feed, "coherent feed") # must not raise
def test_validate_stop_geometry_only_warns_on_low_service_displacement(
tmp_path: Path,
) -> None:
"""Heritage-line quirks (few trips) warn but do not block a build."""
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=6, b_lat=58.0, b_lon=-2.0)
validate_stop_geometry(feed, "heritage feed") # must not raise
# ── validate_london_coverage ──────────────────────────────────────────────────
_ALL_LU = (
"Bakerloo",
"Central",
"Circle",
"District",
"Hammersmith & City",
"Jubilee",
"Metropolitan",
"Northern",
"Piccadilly",
"Victoria",
"Waterloo & City",
)
_COVERAGE_CALENDAR = (
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,1,1,20260101,20271231\n"
)
def _bods_coverage(
path: Path,
*,
lu_lines: tuple[str, ...] = _ALL_LU,
include_dlr: bool = True,
include_tramlink: bool = True,
) -> Path:
routes = ["route_id,agency_id,route_short_name,route_long_name,route_type"]
trips = ["trip_id,route_id,service_id"]
n = 0
for line in lu_lines:
routes.append(f"LU{n},LU,{line},,1")
trips.append(f"T{n},LU{n},S1")
n += 1
if include_dlr:
routes.append(f"DLR{n},DLRA,DLR,Docklands Light Railway,2")
trips.append(f"T{n},DLR{n},S1")
n += 1
if include_tramlink:
routes.append(f"TR{n},TRAM,Tram,London Tramlink,0")
trips.append(f"T{n},TR{n},S1")
n += 1
with zipfile.ZipFile(path, "w") as z:
z.writestr("calendar.txt", _COVERAGE_CALENDAR)
z.writestr("routes.txt", "\n".join(routes) + "\n")
z.writestr("trips.txt", "\n".join(trips) + "\n")
return path
def _nr_coverage(
path: Path, *, include_elizabeth: bool = True, include_overground: bool = True
) -> Path:
routes = ["route_id,agency_id,route_short_name,route_long_name,route_type"]
trips = ["trip_id,route_id,service_id"]
if include_elizabeth:
routes.append("XR1,XR,XR:PAD->ABW,Elizabeth line,2")
trips.append("TX,XR1,S1")
if include_overground:
routes.append("LO1,LO,LO,London Overground,2")
trips.append("TL,LO1,S1")
with zipfile.ZipFile(path, "w") as z:
z.writestr("calendar.txt", _COVERAGE_CALENDAR)
z.writestr("routes.txt", "\n".join(routes) + "\n")
z.writestr("trips.txt", "\n".join(trips) + "\n")
return path
def test_validate_london_coverage_happy_path(tmp_path: Path) -> None:
bods = _bods_coverage(tmp_path / "bods.zip")
nr = _nr_coverage(tmp_path / "nr.zip")
validate_london_coverage(bods, nr, today=TODAY) # must not raise
def test_validate_london_coverage_missing_tube_line_fails(tmp_path: Path) -> None:
bods = _bods_coverage(
tmp_path / "bods.zip",
lu_lines=tuple(name for name in _ALL_LU if name != "Victoria"),
)
nr = _nr_coverage(tmp_path / "nr.zip")
with pytest.raises(RuntimeError, match="Victoria"):
validate_london_coverage(bods, nr, today=TODAY)
def test_validate_london_coverage_missing_dlr_fails(tmp_path: Path) -> None:
bods = _bods_coverage(tmp_path / "bods.zip", include_dlr=False)
nr = _nr_coverage(tmp_path / "nr.zip")
with pytest.raises(RuntimeError, match="DLR"):
validate_london_coverage(bods, nr, today=TODAY)
def test_validate_london_coverage_missing_elizabeth_fails(tmp_path: Path) -> None:
bods = _bods_coverage(tmp_path / "bods.zip")
nr = _nr_coverage(tmp_path / "nr.zip", include_elizabeth=False)
with pytest.raises(RuntimeError, match="Elizabeth line"):
validate_london_coverage(bods, nr, today=TODAY)
def test_validate_london_coverage_expired_service_fails(tmp_path: Path) -> None:
"""A line whose only calendar expired years ago counts as missing (present in
routes.txt but with no active service — the retired-TfL-feed failure mode)."""
bods = _bods_coverage(tmp_path / "bods.zip")
nr = tmp_path / "nr.zip"
expired = (
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
"start_date,end_date\n"
"S1,1,1,1,1,1,1,1,20091201,20101224\n"
)
with zipfile.ZipFile(nr, "w") as z:
z.writestr("calendar.txt", expired)
z.writestr(
"routes.txt",
"route_id,agency_id,route_short_name,route_long_name,route_type\n"
"XR1,XR,XR:PAD->ABW,Elizabeth line,2\nLO1,LO,LO,London Overground,2\n",
)
z.writestr("trips.txt", "trip_id,route_id,service_id\nTX,XR1,S1\nTL,LO1,S1\n")
with pytest.raises(RuntimeError, match="Elizabeth line|Overground"):
validate_london_coverage(bods, nr, today=TODAY)