This commit is contained in:
Andras Schmelczer 2026-07-03 18:28:56 +01:00
parent 909e241907
commit 1ee796b282
29 changed files with 250 additions and 126 deletions

View file

@ -3,7 +3,7 @@
Downloads five-year age band counts (TS007A) from the NOMIS API, then computes
the median age per LSOA using linear interpolation within the median class.
Source: NOMIS (ONS Census 2021 TS007A dataset, NM_2020_1)
Source: NOMIS (ONS Census 2021, TS007A dataset, NM_2020_1)
License: Open Government Licence v3.0
"""
@ -20,7 +20,7 @@ from pipeline.utils import ENGLAND_LSOA_COUNT_2021, download_nomis_csv
BASE_URL = "https://www.nomisweb.co.uk/api/v01/dataset/NM_2020_1.data.csv?date=latest&geography=TYPE151&c2021_age_19=1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18&measures=20100&select=GEOGRAPHY_CODE,C2021_AGE_19_NAME,OBS_VALUE"
# Five-year age bands in order, with lower bounds for interpolation.
# The last band (85+) is open-ended — we treat it as 85-89 for median purposes.
# The last band (85+) is open-ended. We treat it as 85-89 for median purposes.
AGE_BANDS = [
(0, 5), # Aged 0 to 4 years
(5, 5), # Aged 5 to 9 years

View file

@ -28,7 +28,7 @@ LONDON_UNDERGROUND_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZLU"
# The Docklands Light Railway uses the analogous "ZZDL" system code (e.g.
# "9400ZZDLBEC" for Beckton). Like ZZLU it is unique to one network, so a
# TMU/MET stop carrying a ZZDL code is reclassified from the tram/metro family
# to its own "DLR station" category restoring DLR to the train/tube station
# to its own "DLR station" category, restoring DLR to the train/tube station
# list and giving it the DLR roundel rather than a generic tram icon.
LONDON_DLR_ATCO_PATTERN = r"(?i)^\d{3}[0G]ZZDL"
@ -337,7 +337,7 @@ class StationAccumulator:
# A single node is never both (ZZLU vs ZZDL), but a co-located
# interchange (Bank, Stratford, Canning Town, West Ham) merges its LU
# and DLR halves into one group carrying both flags; Tube is checked
# first so these resolve to "Tube station" — their primary identity —
# first so these resolve to "Tube station" (their primary identity),
# leaving "DLR station" for the DLR-only stops the fix targets.
if self.category == TRAM_METRO_CATEGORY and self.is_lu:
return TUBE_STATION_CATEGORY

View file

@ -85,24 +85,24 @@ RESOLUTION = NATIVE_RESOLUTION
# Defra encodes TRUE "no data" with this sentinel (NOT 0.0). A 0.0 cell that is
# otherwise inside the raster means "modelled below the lowest reporting band",
# i.e. genuinely quiet — see noise_overlay_tiles.py:167.
# i.e. genuinely quiet (see noise_overlay_tiles.py:167).
NOISE_NODATA_SENTINEL = np.float32(-96.0)
# Lowest modelled Defra Lden reporting band (dB). Verified against the actual
# rasters: the minimum positive in-coverage value is 40.0 dB with NO values in
# (0, 40) — below the band, cells are encoded as 0.0 (genuinely quiet). We floor
# (0, 40). Below the band, cells are encoded as 0.0 (genuinely quiet). We floor
# in-coverage cells to 40.0 so a below-band 0.0 surfaces as "we know it's quiet"
# (~40 dB) instead of collapsing to null ("we don't know"), WITHOUT inflating the
# ~35% of genuine 40-44.99 dB readings that a 45.0 floor would wrongly bump to 45.
# NB: 45.0 is the overlay's lowest *paint* stop (noise_overlay_tiles.
# NOISE_COLOR_STOPS[0]) a rendering threshold, not the data's reporting floor.
# NOISE_COLOR_STOPS[0]), a rendering threshold, not the data's reporting floor.
NOISE_QUIET_FLOOR_DB = np.float32(40.0)
# Sample noise at the postcode representative point itself (no neighbourhood
# window). A 50m MAX-of-window grabbed the single loudest 10m cell within ~1.2 ha
# of every postcode; because Defra road contours hug every modelled road and
# representative points sit on/near streets, that inflated postcode noise by
# roughly +9 dB (log scale) making ~94% of England read >=55 dB Lden and
# roughly +9 dB (log scale), making ~94% of England read >=55 dB Lden and
# collapsing the metric's discrimination at the quiet end. Radius 0 ->
# filter_size 1 -> the maximum_filter is skipped and each postcode reads the
# 10m cell it actually sits in.
@ -110,8 +110,8 @@ POSTCODE_NOISE_RADIUS_M = 0
# Adjacent download tiles overlap by the sampling radius so every postcode's
# sampling footprint is fully contained in at least one tile. With point
# sampling (radius 0) this is 0 a representative point falls inside exactly
# one tile but the relationship is kept so any future non-zero radius keeps
# sampling (radius 0) this is 0 (a representative point falls inside exactly
# one tile), but the relationship is kept so any future non-zero radius keeps
# its window seam-safe.
TILE_OVERLAP_M = POSTCODE_NOISE_RADIUS_M
@ -273,7 +273,7 @@ def _download_tile(
NoGeoTiffError,
httpx.HTTPStatusError,
# TransportError is the superset of TimeoutException, ConnectError,
# ReadError and ProtocolError including RemoteProtocolError, raised
# ReadError and ProtocolError, including RemoteProtocolError, raised
# when the WCS server closes the connection mid-stream ("incomplete
# chunked read"). All are transient; retry/split rather than letting
# one flaky tile crash the whole raster download.
@ -446,7 +446,7 @@ def sample_noise_at_postcodes(
# Defra rasters encode TRUE nodata as the -96.0 sentinel (and
# occasionally non-finite / dataset.nodata); genuinely quiet ground
# below the model's lowest reporting band is encoded as 0.0. Only
# the former is "we don't know" — the latter is a real "we know it's
# the former is "we don't know". The latter is a real "we know it's
# quiet" reading and must not collapse to null. So treat ONLY true
# nodata as -inf (it never wins a max and never counts as coverage),
# and clamp every in-coverage cell up to NOISE_QUIET_FLOOR_DB so a
@ -555,7 +555,7 @@ def main() -> None:
if not tile_paths:
print(
f"[{label}] WARNING: No tiles downloaded column will be all null"
f"[{label}] WARNING: No tiles downloaded; column will be all null"
)
series = pl.Series(col_name, [None] * len(lat), dtype=pl.Float32)
else:

View file

@ -7,7 +7,7 @@ site id and the site's polygon centroid. Sites without access points fall
back to polygon centroids.
Using access points rather than polygon centroids gives much more accurate
distance calculations a property next to Hyde Park won't show 400m just
distance calculations: a property next to Hyde Park won't show 400m just
because the centroid is in the middle of the park. The site id / centroid
columns let downstream consumers (poi_proximity) collapse the frame back to
one row per SITE for counting, so a park with 30 gates counts as one park.
@ -190,7 +190,7 @@ def download_greenspace(output: Path) -> None:
print(f"Reading {site_shps[0].name} for function types...")
site_funcs = _read_site_functions(site_shps[0])
# Step 2: Read access points (primary park entrances)
# Step 2: Read access points (primary: park entrances)
print(f"Reading {access_shps[0].name}...")
ap_lats, ap_lngs, ap_cats, ap_site_ids = _read_access_points(
access_shps[0], site_funcs

View file

@ -930,7 +930,7 @@ def main() -> None:
df.write_parquet(args.output)
print(f"Saved to {args.output}")
else:
print("No places found skipping output")
print("No places found, skipping output")
if __name__ == "__main__":

View file

@ -35,7 +35,7 @@ def _data_rows(df: pl.DataFrame) -> pl.DataFrame:
The preamble length varies (title, optional "This worksheet contains..."
note, then the header row starting with "Time period"), so locate the
header by content instead of counting rows a fixed slice leaves the
header by content instead of counting rows: a fixed slice leaves the
header in the data whenever ONS adds or removes a note line.
"""
header_marker = (
@ -78,7 +78,7 @@ def _latest_rents_long(df: pl.DataFrame) -> pl.DataFrame:
print(f"LAs in latest month: {df.height}")
# Melt to long format: one row per area x bedroom count.
# PIPR has no Studio category one-bed rent used as proxy for bedrooms=0.
# PIPR has no Studio category: one-bed rent used as proxy for bedrooms=0.
frames = []
for col, bedrooms in [
("rent_1bed", 0), # Studio (proxy)

View file

@ -2,8 +2,8 @@
Downloads the household-tenure breakdown (TS054, classification C2021_TENURE_9)
from the NOMIS API at LSOA 2021 granularity and folds the 8 detailed leaf
categories into our 3 output buckets Owner occupied / Social rent /
Private rent emitting one row per LSOA with the percentage of households in
categories into our 3 output buckets (Owner occupied / Social rent /
Private rent), emitting one row per LSOA with the percentage of households in
each. The three buckets sum to 100%, so downstream they render as a
composition/ratio (like the ethnicity and qualifications stacked bars) AND each
percentage is independently filterable.
@ -19,7 +19,7 @@ NOTE this table counts HOUSEHOLDS (not usual residents). The join key downstream
(merge.py) is `lsoa21`, the same key used for ethnicity, qualifications,
median age, and IoD.
Source: NOMIS (ONS Census 2021 TS054 dataset, NM_2072_1)
Source: NOMIS (ONS Census 2021, TS054 dataset, NM_2072_1)
License: Open Government Licence v3.0
"""

View file

@ -72,7 +72,7 @@ def test_ethnicity_routes_chinese_to_east_and_other_asian_to_se():
def test_ethnicity_percentages_independent_per_lsoa():
"""Two LSOAs get independent profiles the LSOA granularity is the point."""
"""Two LSOAs get independent profiles: the LSOA granularity is the point."""
df = pl.concat(
[
pl.DataFrame(

View file

@ -281,7 +281,7 @@ def test_sample_noise_preserves_genuine_reading_above_quiet_floor(
# The lowest Defra reporting band is 40.0 dB; genuine readings populate
# [40, ~80]. A genuine in-coverage reading at or just above the floor must be
# PRESERVED, not clamped UP to the floor — only true-quiet 0.0 is floored. A
# PRESERVED, not clamped UP to the floor. Only true-quiet 0.0 is floored. A
# quiet floor set too high (e.g. 45) would inflate the ~35% of real 40-44.99
# dB readings; this pins that they survive unchanged.
floor = float(noise.NOISE_QUIET_FLOOR_DB)

View file

@ -50,7 +50,7 @@ ENGLAND_PBF_URL = (
"https://download.geofabrik.de/europe/united-kingdom/england-latest.osm.pbf"
)
# Bus Open Data Service pre-converted GTFS covering all England bus/tram/ferry
# Bus Open Data Service: pre-converted GTFS covering all England bus/tram/ferry
BODS_GTFS_URL = "https://data.bus-data.dft.gov.uk/timetable/download/gtfs-file/all/"
# National Rail Open Data API
@ -597,7 +597,7 @@ def validate_gtfs_feed(
fail("has neither calendar.txt nor calendar_dates.txt")
if not _calendar_active_in_window(z, names, window_start, window_end):
fail(
f"no service active between {window_start} and {window_end} "
f"no service active between {window_start} and {window_end}: "
"the feed's calendars are stale/expired and it would contribute "
"zero service to routing"
)
@ -697,7 +697,7 @@ def _iter_stop_time_trips(lines, trip_id_idx: int):
dtd2mysql currently writes rows grouped by trip and ordered by
stop_sequence, but neither is guaranteed by GTFS. Grouping is verified (a
trip_id reappearing later raises instead of silently scrambling trips);
within-trip order is NOT assumed callers sort each group by its original
within-trip order is NOT assumed: callers sort each group by its original
stop_sequence.
"""
current_trip: str | None = None
@ -1175,7 +1175,7 @@ def main() -> None:
cif = download_national_rail_cif(raw_dir)
if cif is None:
raise RuntimeError(
"National Rail timetable was not downloaded set "
"National Rail timetable was not downloaded: set "
"NATIONAL_RAIL_EMAIL / NATIONAL_RAIL_PASSWORD (register free at "
"https://opendata.nationalrail.co.uk/). National Rail heavy rail is "
"required; without it the transit network models every train journey "