fine 2
This commit is contained in:
parent
9e4e65fa2a
commit
ca771a7edf
32 changed files with 1467 additions and 109 deletions
|
|
@ -7,9 +7,13 @@ from pathlib import Path
|
|||
import pytest
|
||||
|
||||
from pipeline.download.transit_network import (
|
||||
STATION_COORD_OVERRIDES,
|
||||
_repair_stop_coordinate,
|
||||
clean_national_rail_gtfs,
|
||||
convert_high_freq_to_frequency_based,
|
||||
validate_gtfs_feed,
|
||||
validate_london_coverage,
|
||||
validate_stop_geometry,
|
||||
)
|
||||
|
||||
|
||||
|
|
@ -231,9 +235,7 @@ def test_validate_gtfs_feed_zero_and_empty_coords(tmp_path: Path) -> None:
|
|||
feed = _make_gtfs(
|
||||
tmp_path / "feed.zip",
|
||||
stops=(
|
||||
"stop_id,stop_name,stop_lat,stop_lon\n"
|
||||
"STOP_A,Nowhere,0,0\n"
|
||||
"STOP_B,Blank,,\n"
|
||||
"stop_id,stop_name,stop_lat,stop_lon\nSTOP_A,Nowhere,0,0\nSTOP_B,Blank,,\n"
|
||||
),
|
||||
)
|
||||
with pytest.raises(RuntimeError, match=r"plausible UK coordinates"):
|
||||
|
|
@ -281,3 +283,254 @@ def test_validate_gtfs_feed_not_a_zip(tmp_path: Path) -> None:
|
|||
bogus.write_text("not a zip")
|
||||
with pytest.raises(RuntimeError, match="not a valid zip"):
|
||||
validate_gtfs_feed(bogus, "bogus feed", today=TODAY)
|
||||
|
||||
|
||||
# ── _repair_stop_coordinate ───────────────────────────────────────────────────
|
||||
|
||||
|
||||
@pytest.mark.parametrize(
|
||||
"stop_id,lat,lon,expected",
|
||||
[
|
||||
# Known-bad stations get an authoritative override (TCR ships transposed,
|
||||
# BDS ships a wrong-signed longitude; both are in STATION_COORD_OVERRIDES).
|
||||
("TCR", -0.1306, 51.5163, (*STATION_COORD_OVERRIDES["TCR"], "override")),
|
||||
("BDS", 51.514, 0.15, (*STATION_COORD_OVERRIDES["BDS"], "override")),
|
||||
# A plausible UK coordinate is left untouched.
|
||||
("ZFD", 51.5205, -0.1050, (51.5205, -0.1050, "keep")),
|
||||
# An unknown station with lat/lon transposed is swapped back generically.
|
||||
("ZZZ", -0.13, 51.51, (51.51, -0.13, "transpose")),
|
||||
# Genuinely out-of-area garbage (Irish CIE South Atlantic) is neutralised.
|
||||
("IEP", -4.172, -14.5154, (54.0, -2.0, "dump")),
|
||||
# Missing coordinates are neutralised too.
|
||||
("NUL", None, None, (54.0, -2.0, "dump")),
|
||||
],
|
||||
)
|
||||
def test_repair_stop_coordinate(stop_id, lat, lon, expected) -> None:
|
||||
assert _repair_stop_coordinate(stop_id, lat, lon) == expected
|
||||
|
||||
|
||||
def test_clean_national_rail_repairs_broken_station_coords(tmp_path: Path) -> None:
|
||||
"""End-to-end: the cleaner repairs the exact TCR/BDS failure modes.
|
||||
|
||||
TCR (transposed) and BDS (wrong-signed lon) are corrected to their override
|
||||
coordinates; an unknown transposed stop is swapped back; genuine out-of-area
|
||||
garbage is dumped; a good coordinate is preserved.
|
||||
"""
|
||||
src = tmp_path / "in.zip"
|
||||
dst = tmp_path / "out.zip"
|
||||
stops = (
|
||||
"stop_id,stop_name,stop_lat,stop_lon\n"
|
||||
"TCR,Tottenham Court Road (Elizabeth line),-0.1306,51.5163\n"
|
||||
"BDS,Bond Street (Elizabeth line),51.514,0.15\n"
|
||||
"ZZZ,Transposed Halt,-0.20,51.40\n"
|
||||
"IEP,Cork (CIE),-4.172,-14.5154\n"
|
||||
"GUD,Good Station,51.50,-0.10\n"
|
||||
)
|
||||
with zipfile.ZipFile(src, "w") as z:
|
||||
z.writestr("stops.txt", stops)
|
||||
z.writestr("routes.txt", "route_id,route_type\nR1,2\n")
|
||||
z.writestr("trips.txt", "trip_id,route_id,service_id\nT1,R1,S1\n")
|
||||
z.writestr(
|
||||
"stop_times.txt",
|
||||
"trip_id,stop_id,stop_sequence,departure_time\n"
|
||||
"T1,TCR,1,06:00:00\n"
|
||||
"T1,BDS,2,06:03:00\n"
|
||||
"T1,GUD,3,06:06:00\n",
|
||||
)
|
||||
|
||||
clean_national_rail_gtfs(src, dst)
|
||||
|
||||
with zipfile.ZipFile(dst, "r") as z:
|
||||
rows = z.read("stops.txt").decode("utf-8").splitlines()
|
||||
coords = {r.split(",")[0]: r.split(",")[-2:] for r in rows[1:]}
|
||||
assert (
|
||||
float(coords["TCR"][0]),
|
||||
float(coords["TCR"][1]),
|
||||
) == STATION_COORD_OVERRIDES["TCR"]
|
||||
assert (
|
||||
float(coords["BDS"][0]),
|
||||
float(coords["BDS"][1]),
|
||||
) == STATION_COORD_OVERRIDES["BDS"]
|
||||
assert (float(coords["ZZZ"][0]), float(coords["ZZZ"][1])) == (51.40, -0.20)
|
||||
assert (float(coords["IEP"][0]), float(coords["IEP"][1])) == (54.0, -2.0)
|
||||
assert (float(coords["GUD"][0]), float(coords["GUD"][1])) == (51.50, -0.10)
|
||||
|
||||
|
||||
# ── validate_stop_geometry ────────────────────────────────────────────────────
|
||||
|
||||
|
||||
def _geometry_gtfs(path: Path, *, n_trips: int, b_lat: float, b_lon: float) -> Path:
|
||||
"""A metro line A–B–C (5-minute hops) repeated across n_trips.
|
||||
|
||||
Displacing B far from A and C makes it a displacement outlier; n_trips sets
|
||||
its service level (the hard-fail vs warn tier).
|
||||
"""
|
||||
routes = "route_id,route_type\nR1,1\n"
|
||||
trips = "trip_id,route_id,service_id\n" + "".join(
|
||||
f"T{i},R1,S1\n" for i in range(n_trips)
|
||||
)
|
||||
stops = (
|
||||
"stop_id,stop_name,stop_lat,stop_lon\n"
|
||||
"A,Aaa,51.50,-0.10\n"
|
||||
f"B,Bbb,{b_lat},{b_lon}\n"
|
||||
"C,Ccc,51.52,-0.10\n"
|
||||
)
|
||||
header = "trip_id,stop_id,stop_sequence,arrival_time,departure_time\n"
|
||||
body = "".join(
|
||||
f"T{i},A,0,06:00:00,06:00:00\n"
|
||||
f"T{i},B,1,06:05:00,06:05:00\n"
|
||||
f"T{i},C,2,06:10:00,06:10:00\n"
|
||||
for i in range(n_trips)
|
||||
)
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("routes.txt", routes)
|
||||
z.writestr("trips.txt", trips)
|
||||
z.writestr("stops.txt", stops)
|
||||
z.writestr("stop_times.txt", header + body)
|
||||
return path
|
||||
|
||||
|
||||
def test_validate_stop_geometry_fails_on_high_service_displacement(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""A busy stop whose trains imply teleportation fails the build (TCR mode)."""
|
||||
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=120, b_lat=58.0, b_lon=-2.0)
|
||||
with pytest.raises(RuntimeError, match="stop-geometry validation failed"):
|
||||
validate_stop_geometry(feed, "displaced feed")
|
||||
|
||||
|
||||
def test_validate_stop_geometry_passes_when_stops_are_coherent(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=120, b_lat=51.51, b_lon=-0.10)
|
||||
validate_stop_geometry(feed, "coherent feed") # must not raise
|
||||
|
||||
|
||||
def test_validate_stop_geometry_only_warns_on_low_service_displacement(
|
||||
tmp_path: Path,
|
||||
) -> None:
|
||||
"""Heritage-line quirks (few trips) warn but do not block a build."""
|
||||
feed = _geometry_gtfs(tmp_path / "feed.zip", n_trips=6, b_lat=58.0, b_lon=-2.0)
|
||||
validate_stop_geometry(feed, "heritage feed") # must not raise
|
||||
|
||||
|
||||
# ── validate_london_coverage ──────────────────────────────────────────────────
|
||||
|
||||
_ALL_LU = (
|
||||
"Bakerloo",
|
||||
"Central",
|
||||
"Circle",
|
||||
"District",
|
||||
"Hammersmith & City",
|
||||
"Jubilee",
|
||||
"Metropolitan",
|
||||
"Northern",
|
||||
"Piccadilly",
|
||||
"Victoria",
|
||||
"Waterloo & City",
|
||||
)
|
||||
|
||||
_COVERAGE_CALENDAR = (
|
||||
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
|
||||
"start_date,end_date\n"
|
||||
"S1,1,1,1,1,1,1,1,20260101,20271231\n"
|
||||
)
|
||||
|
||||
|
||||
def _bods_coverage(
|
||||
path: Path,
|
||||
*,
|
||||
lu_lines: tuple[str, ...] = _ALL_LU,
|
||||
include_dlr: bool = True,
|
||||
include_tramlink: bool = True,
|
||||
) -> Path:
|
||||
routes = ["route_id,agency_id,route_short_name,route_long_name,route_type"]
|
||||
trips = ["trip_id,route_id,service_id"]
|
||||
n = 0
|
||||
for line in lu_lines:
|
||||
routes.append(f"LU{n},LU,{line},,1")
|
||||
trips.append(f"T{n},LU{n},S1")
|
||||
n += 1
|
||||
if include_dlr:
|
||||
routes.append(f"DLR{n},DLRA,DLR,Docklands Light Railway,2")
|
||||
trips.append(f"T{n},DLR{n},S1")
|
||||
n += 1
|
||||
if include_tramlink:
|
||||
routes.append(f"TR{n},TRAM,Tram,London Tramlink,0")
|
||||
trips.append(f"T{n},TR{n},S1")
|
||||
n += 1
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("calendar.txt", _COVERAGE_CALENDAR)
|
||||
z.writestr("routes.txt", "\n".join(routes) + "\n")
|
||||
z.writestr("trips.txt", "\n".join(trips) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def _nr_coverage(
|
||||
path: Path, *, include_elizabeth: bool = True, include_overground: bool = True
|
||||
) -> Path:
|
||||
routes = ["route_id,agency_id,route_short_name,route_long_name,route_type"]
|
||||
trips = ["trip_id,route_id,service_id"]
|
||||
if include_elizabeth:
|
||||
routes.append("XR1,XR,XR:PAD->ABW,Elizabeth line,2")
|
||||
trips.append("TX,XR1,S1")
|
||||
if include_overground:
|
||||
routes.append("LO1,LO,LO,London Overground,2")
|
||||
trips.append("TL,LO1,S1")
|
||||
with zipfile.ZipFile(path, "w") as z:
|
||||
z.writestr("calendar.txt", _COVERAGE_CALENDAR)
|
||||
z.writestr("routes.txt", "\n".join(routes) + "\n")
|
||||
z.writestr("trips.txt", "\n".join(trips) + "\n")
|
||||
return path
|
||||
|
||||
|
||||
def test_validate_london_coverage_happy_path(tmp_path: Path) -> None:
|
||||
bods = _bods_coverage(tmp_path / "bods.zip")
|
||||
nr = _nr_coverage(tmp_path / "nr.zip")
|
||||
validate_london_coverage(bods, nr, today=TODAY) # must not raise
|
||||
|
||||
|
||||
def test_validate_london_coverage_missing_tube_line_fails(tmp_path: Path) -> None:
|
||||
bods = _bods_coverage(
|
||||
tmp_path / "bods.zip",
|
||||
lu_lines=tuple(name for name in _ALL_LU if name != "Victoria"),
|
||||
)
|
||||
nr = _nr_coverage(tmp_path / "nr.zip")
|
||||
with pytest.raises(RuntimeError, match="Victoria"):
|
||||
validate_london_coverage(bods, nr, today=TODAY)
|
||||
|
||||
|
||||
def test_validate_london_coverage_missing_dlr_fails(tmp_path: Path) -> None:
|
||||
bods = _bods_coverage(tmp_path / "bods.zip", include_dlr=False)
|
||||
nr = _nr_coverage(tmp_path / "nr.zip")
|
||||
with pytest.raises(RuntimeError, match="DLR"):
|
||||
validate_london_coverage(bods, nr, today=TODAY)
|
||||
|
||||
|
||||
def test_validate_london_coverage_missing_elizabeth_fails(tmp_path: Path) -> None:
|
||||
bods = _bods_coverage(tmp_path / "bods.zip")
|
||||
nr = _nr_coverage(tmp_path / "nr.zip", include_elizabeth=False)
|
||||
with pytest.raises(RuntimeError, match="Elizabeth line"):
|
||||
validate_london_coverage(bods, nr, today=TODAY)
|
||||
|
||||
|
||||
def test_validate_london_coverage_expired_service_fails(tmp_path: Path) -> None:
|
||||
"""A line whose only calendar expired years ago counts as missing (present in
|
||||
routes.txt but with no active service — the retired-TfL-feed failure mode)."""
|
||||
bods = _bods_coverage(tmp_path / "bods.zip")
|
||||
nr = tmp_path / "nr.zip"
|
||||
expired = (
|
||||
"service_id,monday,tuesday,wednesday,thursday,friday,saturday,sunday,"
|
||||
"start_date,end_date\n"
|
||||
"S1,1,1,1,1,1,1,1,20091201,20101224\n"
|
||||
)
|
||||
with zipfile.ZipFile(nr, "w") as z:
|
||||
z.writestr("calendar.txt", expired)
|
||||
z.writestr(
|
||||
"routes.txt",
|
||||
"route_id,agency_id,route_short_name,route_long_name,route_type\n"
|
||||
"XR1,XR,XR:PAD->ABW,Elizabeth line,2\nLO1,LO,LO,London Overground,2\n",
|
||||
)
|
||||
z.writestr("trips.txt", "trip_id,route_id,service_id\nTX,XR1,S1\nTL,LO1,S1\n")
|
||||
with pytest.raises(RuntimeError, match="Elizabeth line|Overground"):
|
||||
validate_london_coverage(bods, nr, today=TODAY)
|
||||
|
|
|
|||
|
|
@ -42,6 +42,7 @@ import zipfile
|
|||
from collections import defaultdict
|
||||
from pathlib import Path
|
||||
|
||||
import polars as pl
|
||||
from tqdm import tqdm
|
||||
|
||||
from pipeline.local_temp import local_tmp_dir
|
||||
|
|
@ -66,6 +67,56 @@ GTFS_MIN_VALID_STOP_FRACTION = 0.95
|
|||
UK_LAT_RANGE = (49.0, 61.0)
|
||||
UK_LON_RANGE = (-9.0, 2.5)
|
||||
|
||||
# Authoritative (lat, lon) for stops that upstream feeds ship mislocated, keyed
|
||||
# by GTFS stop_id (National Rail CRS code). The National Rail CIF → GTFS export
|
||||
# ships the Elizabeth-line-only core stations broken: Tottenham Court Road (TCR)
|
||||
# has its lat/lon transposed and Bond Street (BDS) has a wrong-signed longitude,
|
||||
# leaving them ~300km and ~20km from reality. Both then fail to link to the
|
||||
# street network, so nobody can board/alight the Elizabeth line there and every
|
||||
# journey through them silently reroutes (e.g. TCR→Heathrow via the Central line
|
||||
# + Heathrow Express instead of one seat on the Elizabeth line). Coordinates are
|
||||
# the TfL/BODS station nodes for the same physical stations. New breakages that
|
||||
# these overrides don't cover are caught by validate_stop_geometry() below, which
|
||||
# fails the build rather than shipping a phantom stop.
|
||||
STATION_COORD_OVERRIDES = {
|
||||
"TCR": (51.51643, -0.13041), # Tottenham Court Road (Elizabeth line)
|
||||
"BDS": (51.51430, -0.14972), # Bond Street (Elizabeth line)
|
||||
}
|
||||
|
||||
# validate_stop_geometry(): a served tram/metro/rail stop is a "displacement
|
||||
# outlier" when, over real timetabled hops (>= GEOMETRY_MIN_HOP_SECONDS, so that
|
||||
# coarse timetables where adjacent stops share a minute don't count), the implied
|
||||
# in-vehicle speed to a MAJORITY of its distinct trip-neighbours exceeds
|
||||
# GEOMETRY_MAX_KMH. Such a stop sits nowhere near where its trains actually run
|
||||
# (the TCR failure mode). Outliers with >= GEOMETRY_HARDFAIL_MIN_TRIPS timetabled
|
||||
# trips FAIL the build; rarer ones (heritage lines) only warn. Scoped to
|
||||
# rail/metro/tram route_types: buses carry demand-responsive "area" stops and
|
||||
# same-minute urban hops that are noisy without being real geometry errors, and
|
||||
# the R5-side unlinked-stop check covers gross bus displacement anyway.
|
||||
GEOMETRY_RAIL_ROUTE_TYPES = frozenset({"0", "1", "2"})
|
||||
GEOMETRY_MIN_HOP_SECONDS = 120
|
||||
GEOMETRY_MAX_KMH = 300.0
|
||||
GEOMETRY_HARDFAIL_MIN_TRIPS = 100
|
||||
|
||||
# London modes/lines that must be present with active service in the combined
|
||||
# network. A feed regression that silently drops one (as the retired TfL
|
||||
# TransXChange feed did — see module docstring) FAILS the build instead of
|
||||
# quietly degrading every affected journey. Underground lines and Tramlink/DLR
|
||||
# come from BODS; the Elizabeth line and Overground come from National Rail.
|
||||
LONDON_UNDERGROUND_LINES = (
|
||||
"Bakerloo",
|
||||
"Central",
|
||||
"Circle",
|
||||
"District",
|
||||
"Hammersmith & City",
|
||||
"Jubilee",
|
||||
"Metropolitan",
|
||||
"Northern",
|
||||
"Piccadilly",
|
||||
"Victoria",
|
||||
"Waterloo & City",
|
||||
)
|
||||
|
||||
|
||||
def _download_http(
|
||||
url: str, dest: Path, *, desc: str, headers: dict | None = None
|
||||
|
|
@ -804,6 +855,8 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
|
|||
bad_trips_removed = 0
|
||||
seqs_renumbered = 0
|
||||
coords_fixed = 0
|
||||
coords_overridden = 0
|
||||
coords_transposed = 0
|
||||
route_types_fixed = 0
|
||||
|
||||
with (
|
||||
|
|
@ -890,6 +943,7 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
|
|||
with zin.open(info) as f:
|
||||
header = f.readline()
|
||||
cols = _parse_csv_line(header)
|
||||
stop_id_idx = cols.index("stop_id")
|
||||
lat_idx = cols.index("stop_lat")
|
||||
lon_idx = cols.index("stop_lon")
|
||||
|
||||
|
|
@ -905,16 +959,24 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
|
|||
parts = _parse_csv_line(line)
|
||||
if not parts:
|
||||
continue
|
||||
stop_id = parts[stop_id_idx].strip('"')
|
||||
try:
|
||||
lat = float(parts[lat_idx])
|
||||
# Fix bogus Irish CIE coordinates (South Atlantic)
|
||||
if lat < 0:
|
||||
# Set to a neutral UK coordinate that won't be routed to
|
||||
parts[lat_idx] = "54.0"
|
||||
parts[lon_idx] = "-2.0"
|
||||
lon = float(parts[lon_idx])
|
||||
except (ValueError, IndexError):
|
||||
lat = lon = None
|
||||
new_lat, new_lon, action = _repair_stop_coordinate(
|
||||
stop_id, lat, lon
|
||||
)
|
||||
if action != "keep":
|
||||
parts[lat_idx] = repr(new_lat)
|
||||
parts[lon_idx] = repr(new_lon)
|
||||
if action == "override":
|
||||
coords_overridden += 1
|
||||
elif action == "transpose":
|
||||
coords_transposed += 1
|
||||
else: # "dump"
|
||||
coords_fixed += 1
|
||||
except ValueError:
|
||||
pass
|
||||
tmp.write(_format_csv_row(parts))
|
||||
|
||||
tmp.close()
|
||||
|
|
@ -1014,7 +1076,9 @@ def clean_national_rail_gtfs(src: Path, dst: Path) -> None:
|
|||
print(f" Orphan stop references removed: {orphan_stops_removed}")
|
||||
print(f" Bad trip stop_times removed: {bad_trips_removed}")
|
||||
print(f" Stop sequences renumbered: {seqs_renumbered}")
|
||||
print(f" Bogus coordinates fixed: {coords_fixed}")
|
||||
print(f" Coordinates overridden (known-bad stations): {coords_overridden}")
|
||||
print(f" Coordinates de-transposed (lat/lon swapped): {coords_transposed}")
|
||||
print(f" Bogus coordinates dumped (out-of-area): {coords_fixed}")
|
||||
print(f" Route types 714→3 fixed: {route_types_fixed}")
|
||||
print(f" Saved to {dst}")
|
||||
|
||||
|
|
@ -1139,6 +1203,410 @@ def convert_national_rail_to_gtfs(raw_dir: Path, output_dir: Path) -> Path:
|
|||
return dest
|
||||
|
||||
|
||||
def _in_uk(lat: float, lon: float) -> bool:
|
||||
"""True if (lat, lon) falls inside the coarse UK routing bounding box."""
|
||||
return (
|
||||
UK_LAT_RANGE[0] <= lat <= UK_LAT_RANGE[1]
|
||||
and UK_LON_RANGE[0] <= lon <= UK_LON_RANGE[1]
|
||||
)
|
||||
|
||||
|
||||
def _repair_stop_coordinate(
|
||||
stop_id: str, lat: float | None, lon: float | None
|
||||
) -> tuple[float, float, str]:
|
||||
"""Repair an obviously-broken stop coordinate. Returns (lat, lon, action).
|
||||
|
||||
action is one of:
|
||||
"override" - an authoritative coordinate was substituted for a known-bad
|
||||
station (see STATION_COORD_OVERRIDES).
|
||||
"transpose" - the feed shipped lat/lon swapped (the coordinate is outside
|
||||
the UK but swapping lands inside it), so they are swapped
|
||||
back. This is the Tottenham Court Road failure mode and is
|
||||
handled generically, not just for the hard-coded stations.
|
||||
"dump" - the coordinate is genuinely out of area (Irish CIE stations
|
||||
at 0,0-ish South Atlantic garbage, missing coordinates) and
|
||||
is moved to a neutral inland point that will not be routed
|
||||
to, preserving the historical behaviour for those stops.
|
||||
"keep" - already a plausible UK coordinate; left unchanged.
|
||||
"""
|
||||
if stop_id in STATION_COORD_OVERRIDES:
|
||||
return (*STATION_COORD_OVERRIDES[stop_id], "override")
|
||||
if lat is None or lon is None:
|
||||
return 54.0, -2.0, "dump"
|
||||
if _in_uk(lat, lon):
|
||||
return lat, lon, "keep"
|
||||
if _in_uk(lon, lat):
|
||||
return lon, lat, "transpose"
|
||||
return 54.0, -2.0, "dump"
|
||||
|
||||
|
||||
def _secs_expr(col: str) -> pl.Expr:
|
||||
"""Polars expression parsing an HH:MM:SS GTFS time to seconds since midnight."""
|
||||
parts = pl.col(col).str.split(":")
|
||||
return (
|
||||
parts.list.get(0).cast(pl.Int64, strict=False) * 3600
|
||||
+ parts.list.get(1).cast(pl.Int64, strict=False) * 60
|
||||
+ parts.list.get(2).cast(pl.Int64, strict=False)
|
||||
)
|
||||
|
||||
|
||||
def _extract_member(path: Path, member: str, dest_dir: str) -> Path:
|
||||
"""Stream one file out of a GTFS zip to dest_dir (avoids holding it in RAM)."""
|
||||
out = Path(dest_dir) / member
|
||||
with zipfile.ZipFile(path) as z, z.open(member) as src, open(out, "wb") as dst:
|
||||
shutil.copyfileobj(src, dst)
|
||||
return out
|
||||
|
||||
|
||||
def validate_stop_geometry(path: Path, feed_name: str) -> None:
|
||||
"""Fail if a served rail/metro/tram stop is a coordinate displacement outlier.
|
||||
|
||||
Guards against the Tottenham Court Road failure mode: a stop whose timetabled
|
||||
trains imply teleportation (>{GEOMETRY_MAX_KMH:.0f} km/h over a real hop) to a
|
||||
MAJORITY of its distinct trip-neighbours is not where the feed places it, so
|
||||
it cannot link to the street network and every journey through it silently
|
||||
reroutes. High-service outliers (>= {GEOMETRY_HARDFAIL_MIN_TRIPS} trips) raise;
|
||||
negligible-service ones (heritage lines) only warn. Scoped to tram/metro/rail
|
||||
route types; see GEOMETRY_* constants.
|
||||
"""
|
||||
print(f"Validating stop geometry for feed '{feed_name}'...")
|
||||
with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as td:
|
||||
routes = pl.read_csv(
|
||||
_extract_member(path, "routes.txt", td), infer_schema_length=0
|
||||
).select(
|
||||
pl.col("route_id").cast(pl.Utf8),
|
||||
pl.col("route_type").cast(pl.Utf8),
|
||||
)
|
||||
rail_route_ids = routes.filter(
|
||||
pl.col("route_type").is_in(list(GEOMETRY_RAIL_ROUTE_TYPES))
|
||||
).select("route_id")
|
||||
trips = pl.read_csv(
|
||||
_extract_member(path, "trips.txt", td), infer_schema_length=0
|
||||
).select(
|
||||
pl.col("trip_id").cast(pl.Utf8),
|
||||
pl.col("route_id").cast(pl.Utf8),
|
||||
)
|
||||
rail_trips = trips.join(rail_route_ids, on="route_id", how="inner").select(
|
||||
"trip_id"
|
||||
)
|
||||
if rail_trips.height == 0:
|
||||
print(" no rail/metro/tram trips in feed; nothing to check")
|
||||
return
|
||||
stops = pl.read_csv(
|
||||
_extract_member(path, "stops.txt", td), infer_schema_length=0
|
||||
).select(
|
||||
pl.col("stop_id").cast(pl.Utf8),
|
||||
pl.col("stop_name").cast(pl.Utf8).alias("name"),
|
||||
pl.col("stop_lat").cast(pl.Float64, strict=False).alias("lat"),
|
||||
pl.col("stop_lon").cast(pl.Float64, strict=False).alias("lon"),
|
||||
)
|
||||
|
||||
# Only rail/metro/tram stop_times, ordered within each trip.
|
||||
st = (
|
||||
pl.scan_csv(
|
||||
_extract_member(path, "stop_times.txt", td), infer_schema_length=0
|
||||
)
|
||||
.select(
|
||||
pl.col("trip_id").cast(pl.Utf8),
|
||||
pl.col("stop_id").cast(pl.Utf8),
|
||||
pl.col("stop_sequence").cast(pl.Int64, strict=False).alias("seq"),
|
||||
_secs_expr("departure_time").alias("dep"),
|
||||
_secs_expr("arrival_time").alias("arr"),
|
||||
)
|
||||
.join(rail_trips.lazy(), on="trip_id", how="inner")
|
||||
.join(
|
||||
stops.lazy().select(["stop_id", "lat", "lon"]), on="stop_id", how="left"
|
||||
)
|
||||
.collect()
|
||||
.sort(["trip_id", "seq"])
|
||||
)
|
||||
|
||||
# Service level: distinct trips serving each stop (the hard-fail tier).
|
||||
svc = st.group_by("stop_id").agg(pl.col("trip_id").n_unique().alias("trips"))
|
||||
|
||||
# Consecutive-stop hops within a trip.
|
||||
st = st.with_columns(
|
||||
pl.col("lat").shift(1).over("trip_id").alias("plat"),
|
||||
pl.col("lon").shift(1).over("trip_id").alias("plon"),
|
||||
pl.col("stop_id").shift(1).over("trip_id").alias("pid"),
|
||||
pl.col("dep").shift(1).over("trip_id").alias("pdep"),
|
||||
)
|
||||
earth_km = 6371.0
|
||||
dlat = (pl.col("lat") - pl.col("plat")).radians()
|
||||
dlon = (pl.col("lon") - pl.col("plon")).radians()
|
||||
hav = (dlat / 2).sin() ** 2 + pl.col("plat").radians().cos() * pl.col(
|
||||
"lat"
|
||||
).radians().cos() * (dlon / 2).sin() ** 2
|
||||
hops = (
|
||||
st.with_columns(
|
||||
(2 * earth_km * hav.sqrt().arcsin()).alias("dist_km"),
|
||||
(pl.col("arr") - pl.col("pdep")).alias("dt_s"),
|
||||
)
|
||||
.filter(
|
||||
pl.col("plat").is_not_null()
|
||||
& pl.col("dist_km").is_not_null()
|
||||
& (pl.col("dt_s") >= GEOMETRY_MIN_HOP_SECONDS)
|
||||
)
|
||||
.with_columns((pl.col("dist_km") / (pl.col("dt_s") / 3600.0)).alias("kmh"))
|
||||
)
|
||||
if hops.height == 0:
|
||||
print(" no timetabled hops long enough to assess; skipping")
|
||||
return
|
||||
|
||||
# Undirected stop-neighbour edges, flagged if any hop teleports.
|
||||
fwd = hops.select(
|
||||
pl.col("stop_id").alias("a"),
|
||||
pl.col("pid").alias("b"),
|
||||
(pl.col("kmh") > GEOMETRY_MAX_KMH).alias("tp"),
|
||||
)
|
||||
rev = fwd.select(pl.col("b").alias("a"), pl.col("a").alias("b"), "tp")
|
||||
edges = (
|
||||
pl.concat([fwd, rev])
|
||||
.group_by(["a", "b"])
|
||||
.agg(pl.col("tp").max().alias("tp"))
|
||||
)
|
||||
per_stop = edges.group_by("a").agg(
|
||||
pl.len().alias("nbrs"), pl.col("tp").sum().alias("tp_nbrs")
|
||||
)
|
||||
outliers = (
|
||||
per_stop.filter(
|
||||
(pl.col("nbrs") >= 2) & (pl.col("tp_nbrs") / pl.col("nbrs") >= 0.5)
|
||||
)
|
||||
.join(stops, left_on="a", right_on="stop_id", how="left")
|
||||
.join(svc, left_on="a", right_on="stop_id", how="left")
|
||||
.with_columns(pl.col("trips").fill_null(0))
|
||||
.sort("trips", descending=True)
|
||||
)
|
||||
|
||||
hard = outliers.filter(pl.col("trips") >= GEOMETRY_HARDFAIL_MIN_TRIPS)
|
||||
soft = outliers.filter(pl.col("trips") < GEOMETRY_HARDFAIL_MIN_TRIPS)
|
||||
for r in soft.iter_rows(named=True):
|
||||
print(
|
||||
f" WARN low-service displacement outlier: {r['a']} "
|
||||
f"'{r['name']}' ({r['trips']} trips) at ({r['lat']}, {r['lon']})"
|
||||
)
|
||||
if hard.height > 0:
|
||||
lines = [
|
||||
f" {r['a']} '{r['name']}' ({r['trips']} trips) "
|
||||
f"at ({r['lat']}, {r['lon']})"
|
||||
for r in hard.iter_rows(named=True)
|
||||
]
|
||||
raise RuntimeError(
|
||||
f"stop-geometry validation failed for feed '{feed_name}': "
|
||||
f"{hard.height} high-service rail/metro/tram stop(s) sit nowhere "
|
||||
f"near where their trains run (implied speed > {GEOMETRY_MAX_KMH:.0f} "
|
||||
f"km/h to most neighbours). These cannot link to the street network "
|
||||
f"and every journey through them silently reroutes. Add an entry to "
|
||||
f"STATION_COORD_OVERRIDES or fix the upstream feed:\n"
|
||||
+ "\n".join(lines)
|
||||
)
|
||||
print(
|
||||
f" OK: {outliers.height} displacement outlier(s), none at or above "
|
||||
f"{GEOMETRY_HARDFAIL_MIN_TRIPS} trips"
|
||||
)
|
||||
|
||||
|
||||
def _active_service_ids(path: Path, window_start: int, window_end: int) -> set[str]:
|
||||
"""Service ids with at least one running day in [window_start, window_end]."""
|
||||
active: set[str] = set()
|
||||
weekdays = (
|
||||
"monday",
|
||||
"tuesday",
|
||||
"wednesday",
|
||||
"thursday",
|
||||
"friday",
|
||||
"saturday",
|
||||
"sunday",
|
||||
)
|
||||
with zipfile.ZipFile(path) as z:
|
||||
names = set(z.namelist())
|
||||
if "calendar.txt" in names:
|
||||
with z.open("calendar.txt") as f:
|
||||
cols = _parse_csv_line(f.readline())
|
||||
sid_i = cols.index("service_id")
|
||||
start_i = cols.index("start_date")
|
||||
end_i = cols.index("end_date")
|
||||
day_i = [cols.index(d) for d in weekdays if d in cols]
|
||||
for line in f:
|
||||
parts = _parse_csv_line(line)
|
||||
if not parts:
|
||||
continue
|
||||
try:
|
||||
start = int(parts[start_i].strip('"'))
|
||||
end = int(parts[end_i].strip('"'))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
if start > window_end or end < window_start:
|
||||
continue
|
||||
if day_i and not any(
|
||||
parts[i].strip('"') == "1" for i in day_i if i < len(parts)
|
||||
):
|
||||
continue
|
||||
active.add(parts[sid_i].strip('"'))
|
||||
if "calendar_dates.txt" in names:
|
||||
with z.open("calendar_dates.txt") as f:
|
||||
cols = _parse_csv_line(f.readline())
|
||||
sid_i = cols.index("service_id")
|
||||
date_i = cols.index("date")
|
||||
exc_i = cols.index("exception_type")
|
||||
for line in f:
|
||||
parts = _parse_csv_line(line)
|
||||
if not parts:
|
||||
continue
|
||||
try:
|
||||
date = int(parts[date_i].strip('"'))
|
||||
except (ValueError, IndexError):
|
||||
continue
|
||||
if exc_i < len(parts) and parts[exc_i].strip('"') != "1":
|
||||
continue
|
||||
if window_start <= date <= window_end:
|
||||
active.add(parts[sid_i].strip('"'))
|
||||
return active
|
||||
|
||||
|
||||
def _lines_with_active_service(
|
||||
path: Path,
|
||||
active_services: set[str],
|
||||
*,
|
||||
route_predicate,
|
||||
) -> set[str]:
|
||||
"""Return the set of route labels (agency_id::short_name matched by
|
||||
route_predicate) that have at least one trip on an active service.
|
||||
|
||||
route_predicate((agency_id, route_type, short_name, long_name)) -> label|None.
|
||||
A returned label marks the route as one we care about; None ignores it.
|
||||
"""
|
||||
with tempfile.TemporaryDirectory(dir=local_tmp_dir()) as td:
|
||||
routes = pl.read_csv(
|
||||
_extract_member(path, "routes.txt", td), infer_schema_length=0
|
||||
)
|
||||
route_label: dict[str, str] = {}
|
||||
for r in routes.iter_rows(named=True):
|
||||
label = route_predicate(
|
||||
(
|
||||
str(r.get("agency_id", "")),
|
||||
str(r.get("route_type", "")),
|
||||
str(r.get("route_short_name", "")),
|
||||
str(r.get("route_long_name", "")),
|
||||
)
|
||||
)
|
||||
if label is not None:
|
||||
route_label[str(r["route_id"])] = label
|
||||
|
||||
if not route_label:
|
||||
return set()
|
||||
|
||||
trips = pl.read_csv(
|
||||
_extract_member(path, "trips.txt", td), infer_schema_length=0
|
||||
).select(
|
||||
pl.col("route_id").cast(pl.Utf8),
|
||||
pl.col("service_id").cast(pl.Utf8),
|
||||
)
|
||||
present: set[str] = set()
|
||||
for row in trips.iter_rows():
|
||||
route_id, service_id = str(row[0]), str(row[1])
|
||||
label = route_label.get(route_id)
|
||||
if label is not None and service_id in active_services:
|
||||
present.add(label)
|
||||
return present
|
||||
|
||||
|
||||
def validate_london_coverage(
|
||||
bods_path: Path, nr_path: Path, *, today: dt.date | None = None
|
||||
) -> None:
|
||||
"""Fail if any must-have London line/mode lacks active service in the window.
|
||||
|
||||
A silent regression that drops the Underground, DLR, Tramlink, the Elizabeth
|
||||
line or the Overground (as the retired TfL TransXChange feed did) would leave
|
||||
the map quietly under-serving huge swaths of journeys. This turns that into a
|
||||
hard build failure. See LONDON_UNDERGROUND_LINES.
|
||||
"""
|
||||
if today is None:
|
||||
today = dt.date.today()
|
||||
window_start = int(today.strftime("%Y%m%d"))
|
||||
window_end = int(
|
||||
(today + dt.timedelta(days=GTFS_CALENDAR_LOOKAHEAD_DAYS)).strftime("%Y%m%d")
|
||||
)
|
||||
print("Validating London mode/line coverage...")
|
||||
|
||||
bods_active = _active_service_ids(bods_path, window_start, window_end)
|
||||
nr_active = _active_service_ids(nr_path, window_start, window_end)
|
||||
|
||||
# BODS underground lines: agency 'London Underground (TfL)', route_type=1.
|
||||
def lu_pred(row):
|
||||
agency_id, route_type, short, _long = row
|
||||
return (
|
||||
short if route_type == "1" and short in LONDON_UNDERGROUND_LINES else None
|
||||
)
|
||||
|
||||
# DLR: metro/light-rail named DLR (agency 'London Docklands Light Railway').
|
||||
def dlr_pred(row):
|
||||
_agency_id, _route_type, short, long = row
|
||||
text = f"{short} {long}".lower()
|
||||
return "DLR" if ("dlr" in text or "docklands light" in text) else None
|
||||
|
||||
# London Tramlink tram service.
|
||||
def tramlink_pred(row):
|
||||
_agency_id, route_type, short, long = row
|
||||
text = f"{short} {long}".lower()
|
||||
return "Tramlink" if (route_type == "0" and "tram" in text) else None
|
||||
|
||||
lu_present = _lines_with_active_service(
|
||||
bods_path, bods_active, route_predicate=lu_pred
|
||||
)
|
||||
dlr_present = _lines_with_active_service(
|
||||
bods_path, bods_active, route_predicate=dlr_pred
|
||||
)
|
||||
tramlink_present = _lines_with_active_service(
|
||||
bods_path, bods_active, route_predicate=tramlink_pred
|
||||
)
|
||||
|
||||
# National Rail: Elizabeth line (agency_id XR), Overground (agency_id LO).
|
||||
def nr_agency_pred(target_id, label):
|
||||
def pred(row):
|
||||
agency_id, _route_type, _short, _long = row
|
||||
return label if agency_id == target_id else None
|
||||
|
||||
return pred
|
||||
|
||||
elizabeth_present = _lines_with_active_service(
|
||||
nr_path, nr_active, route_predicate=nr_agency_pred("XR", "Elizabeth line")
|
||||
)
|
||||
overground_present = _lines_with_active_service(
|
||||
nr_path, nr_active, route_predicate=nr_agency_pred("LO", "London Overground")
|
||||
)
|
||||
|
||||
problems: list[str] = []
|
||||
missing_lu = [line for line in LONDON_UNDERGROUND_LINES if line not in lu_present]
|
||||
if missing_lu:
|
||||
problems.append(
|
||||
"London Underground lines missing/without active service: "
|
||||
+ ", ".join(missing_lu)
|
||||
)
|
||||
if not dlr_present:
|
||||
problems.append("DLR missing or without active service (BODS)")
|
||||
if not tramlink_present:
|
||||
problems.append("London Tramlink missing or without active service (BODS)")
|
||||
if not elizabeth_present:
|
||||
problems.append(
|
||||
"Elizabeth line missing or without active service (National Rail, XR)"
|
||||
)
|
||||
if not overground_present:
|
||||
problems.append(
|
||||
"London Overground missing or without active service (National Rail, LO)"
|
||||
)
|
||||
|
||||
if problems:
|
||||
raise RuntimeError(
|
||||
"London coverage validation failed (window "
|
||||
f"{window_start}-{window_end}):\n " + "\n ".join(problems)
|
||||
)
|
||||
print(
|
||||
f" OK: {len(lu_present)}/{len(LONDON_UNDERGROUND_LINES)} Underground lines, "
|
||||
"DLR, Tramlink, Elizabeth line and Overground all present with active service"
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Download and prepare transit network data for R5 routing engine"
|
||||
|
|
@ -1167,6 +1635,7 @@ def main() -> None:
|
|||
bods_final = output_dir / "bods_gtfs.zip"
|
||||
convert_high_freq_to_frequency_based(bods_cleaned, bods_final)
|
||||
validate_gtfs_feed(bods_final, "BODS GTFS")
|
||||
validate_stop_geometry(bods_final, "BODS GTFS")
|
||||
|
||||
# 2. National Rail CIF → GTFS. Heavy rail is mandatory: trains are how people
|
||||
# reach the ~2,725 railway-station destinations, so a bus/metro-only network
|
||||
|
|
@ -1183,6 +1652,11 @@ def main() -> None:
|
|||
)
|
||||
nr_final = convert_national_rail_to_gtfs(raw_dir, output_dir)
|
||||
validate_gtfs_feed(nr_final, "National Rail GTFS")
|
||||
validate_stop_geometry(nr_final, "National Rail GTFS")
|
||||
|
||||
# 3. Cross-feed check: every must-have London mode/line is present with
|
||||
# active service. Catches a feed regression that silently drops a whole mode.
|
||||
validate_london_coverage(bods_final, nr_final)
|
||||
|
||||
# Summary
|
||||
print()
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue