lgtm
Some checks failed
CI / Check (push) Failing after 2m47s
Build and publish Docker image / build-and-push (push) Successful in 5m37s

This commit is contained in:
Andras Schmelczer 2026-06-26 16:48:20 +01:00
parent 5e73287eaf
commit e2b85fe819
73 changed files with 1180 additions and 2028 deletions

View file

@ -12,6 +12,7 @@ import polars as pl
NAPTAN_CSV_URL = "https://naptan.api.dft.gov.uk/v1/access-nodes?dataFormat=csv"
TUBE_STATION_CATEGORY = "Tube station"
DLR_STATION_CATEGORY = "DLR station"
TRAM_METRO_CATEGORY = "Tram & Metro stop"
TUBE_STATION_MERGE_RADIUS_DEGREES = 0.01
@ -24,6 +25,13 @@ TUBE_STATION_MERGE_RADIUS_DEGREES = 0.01
# WM Metro, Blackpool Tramway, heritage railways, ...).
LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU"
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
# to its own "DLR station" category — restoring DLR to the train/tube station
# list and giving it the DLR roundel rather than a generic tram icon.
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
STOP_TYPES = {
"AIR": "Airport",
@ -48,10 +56,11 @@ STOP_TYPES = {
"TXR": "Taxi rank",
# Tram/Metro/Underground: TMU is an entrance node, MET the station access
# area. Both start as "Tram & Metro stop"; merged stations whose ATCO codes
# mark them as London Underground (ZZLU) are reclassified to "Tube station"
# after dedup (see _deduplicate_station_areas). Heritage railways (RHDR,
# Severn Valley, ...) are TMU/MET in NaPTAN with no machine-readable
# "heritage" flag, so they remain in "Tram & Metro stop".
# mark them as London Underground (ZZLU) or Docklands Light Railway (ZZDL)
# are reclassified to "Tube station" / "DLR station" after dedup (see
# _deduplicate_station_areas). Heritage railways (RHDR, Severn Valley, ...)
# are TMU/MET in NaPTAN with no machine-readable "heritage" flag, so they
# remain in "Tram & Metro stop".
"TMU": TRAM_METRO_CATEGORY,
"MET": TRAM_METRO_CATEGORY,
}
@ -68,6 +77,7 @@ ENTRANCE_STOP_TYPES = {"RSE", "FTD", "TMU", "BCE"}
STATION_MERGE_CATEGORIES = {
TRAM_METRO_CATEGORY,
TUBE_STATION_CATEGORY,
DLR_STATION_CATEGORY,
"Rail station",
"Ferry",
"Bus station",
@ -266,6 +276,7 @@ class StationAccumulator:
lng_sum: float
entrance: bool = False
is_lu: bool = False
is_dlr: bool = False
count: int = 1
qualifier: str = ""
@ -292,6 +303,7 @@ class StationAccumulator:
self.lng_sum += float(row["lng"])
self.count += 1
self.is_lu = self.is_lu or bool(row.get("is_lu"))
self.is_dlr = self.is_dlr or bool(row.get("is_dlr"))
name = str(row["name"] or "")
row_qualifier = station_name_qualifier(name)
@ -318,12 +330,19 @@ class StationAccumulator:
@property
def output_category(self) -> str:
# A merged tram/metro station is a genuine Tube station when ANY of its
# constituent nodes carries a London Underground ATCO code. Checking
# the whole group (not just the winning node) matters because LU
# entrance nodes often carry non-ZZLU codes (e.g. 4900VICT...).
# A merged tram/metro station is a genuine Tube/DLR station when ANY of
# its constituent nodes carries the matching ATCO system code. Checking
# the whole group (not just the winning node) matters because LU/DLR
# entrance nodes often carry non-ZZLU/ZZDL codes (e.g. 4900VICT...).
# A single node is never both (ZZLU vs ZZDL), but a co-located
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
# and DLR halves into one group carrying both flags; Tube is checked
# first so these resolve to "Tube station" — their primary identity —
# leaving "DLR station" for the DLR-only stops the fix targets.
if self.category == TRAM_METRO_CATEGORY and self.is_lu:
return TUBE_STATION_CATEGORY
if self.category == TRAM_METRO_CATEGORY and self.is_dlr:
return DLR_STATION_CATEGORY
return self.category
@ -336,6 +355,7 @@ def _station_from_row(row: dict[str, object]) -> StationAccumulator:
lng_sum=float(row["lng"]),
entrance=bool(row.get("entrance")),
is_lu=bool(row.get("is_lu")),
is_dlr=bool(row.get("is_dlr")),
qualifier=station_name_qualifier(str(row["name"] or "")),
)
@ -423,7 +443,8 @@ def deduplicate_naptan(df: pl.DataFrame) -> pl.DataFrame:
station by normalized name + area, with the primary station/terminal node
(e.g. RLY, FER, MET, BST) winning over an entrance node (RSE, FTD, TMU,
BCE). Merged tram/metro stations with a London Underground ATCO code in
the group become "Tube station". Other stops are deduplicated by exact
the group become "Tube station"; those with a Docklands Light Railway code
become "DLR station". Other stops are deduplicated by exact
name+category+locality.
"""
station = df.filter(pl.col("category").is_in(list(STATION_MERGE_CATEGORIES)))
@ -490,6 +511,10 @@ def download_naptan(output: Path) -> None:
.str.contains(LONDON_UNDERGROUND_ATCO_PATTERN)
.fill_null(False)
.alias("is_lu"),
pl.col("ATCOCode")
.str.contains(LONDON_DLR_ATCO_PATTERN)
.fill_null(False)
.alias("is_dlr"),
)
before = len(df)

View file

@ -651,7 +651,12 @@ def _naptan_dlr_stations(naptan_path: Path) -> list[dict]:
match = _DLR_CODE_RE.search(atco_id)
if not match:
continue
if row["category"] not in {"Tube station", "Tram & Metro stop", "Rail station"}:
if row["category"] not in {
"Tube station",
"DLR station",
"Tram & Metro stop",
"Rail station",
}:
continue
code = match.group(1)

View file

@ -2,6 +2,7 @@ import polars as pl
import pytest
from pipeline.download.naptan import (
DLR_STATION_CATEGORY,
TRAM_METRO_CATEGORY,
TUBE_STATION_CATEGORY,
canonical_station_name,
@ -131,6 +132,48 @@ def test_deduplicate_naptan_splits_london_underground_from_tram_metro():
assert tram["id"][0] == "9400ZZMAWST"
def test_deduplicate_naptan_splits_dlr_from_tram_metro():
# DLR stations arrive as the tram/metro family (MET station node + TMU
# entrance). The Beckton group carries a 9400ZZDL station node, so the
# merged POI is reclassified to "DLR station" even though its entrance
# carries a non-ZZDL ATCO code; the Metrolink group stays "Tram & Metro
# stop".
df = pl.DataFrame(
{
"id": [
"9400ZZDLBEC",
"490000254S",
"9400ZZMAWST",
],
"name": [
"Beckton DLR Station",
"Beckton",
"Weaste (Manchester Metrolink)",
],
"category": [TRAM_METRO_CATEGORY] * 3,
"lat": [51.5148, 51.5148, 53.4826],
"lng": [0.0613, 0.0614, -2.3087],
"locality": [None, None, None],
"entrance": [False, True, False],
"is_lu": [False, False, False],
"is_dlr": [True, False, False],
}
)
result = deduplicate_naptan(df).sort("category")
assert len(result) == 2
assert result["category"].to_list() == [
DLR_STATION_CATEGORY,
TRAM_METRO_CATEGORY,
]
dlr = result.filter(pl.col("category") == DLR_STATION_CATEGORY)
# The station node (not the entrance) represents the merged POI.
assert dlr["id"][0] == "9400ZZDLBEC"
tram = result.filter(pl.col("category") == TRAM_METRO_CATEGORY)
assert tram["id"][0] == "9400ZZMAWST"
def test_deduplicate_naptan_merges_bus_station_bays_and_entrances():
# BCS bays and a BCE entrance of one bus station collapse to a single POI
# represented by a non-entrance node; a different bus station in another