This commit is contained in:
Andras Schmelczer 2026-06-28 11:59:44 +01:00
parent f1601257c7
commit c2070693fb
68 changed files with 2305 additions and 212 deletions

View file

@ -7,7 +7,7 @@ set -euo pipefail
# Uses full England OSM + 2 GTFS feeds (BODS buses, National Rail).
# R5's TransportNetwork.fromDirectory() picks up all .osm.pbf and .zip files.
#
# Uses each place as origin with all postcodes as destinations R5 does one
# Uses each place as origin with all postcodes as destinations. R5 does one
# routing computation per place, then reads off travel times to all postcodes.
# For car/bicycle/walking this is symmetric (place->postcode = postcode->place).
#
@ -27,7 +27,7 @@ THREADS=12
# The execution cgroup caps process memory at 48 GB (see /sys/fs/cgroup/memory.max);
# the nominal "64 GB" host total is not all addressable to one process. 28g heap +
# ~15g native overhead (DuckDB JNI, R5 mapdb, Kryo deserialize, RAPTOR scratch)
# leaves ~5g cgroup headroom empirically the safe ceiling before SIGKILL.
# leaves ~5g cgroup headroom: empirically the safe ceiling before SIGKILL.
# Under 32g also keeps CompressedOops on, halving R5's reference-heavy footprint.
HEAP=28g
NETWORK_DIR=property-data/r5-network
@ -137,7 +137,7 @@ fi
if [ ! -f "$NETWORK_DIR/network.dat" ]; then
BUILD_DIR="$NETWORK_DIR/build"
echo "--- No cached network copying transit data to build dir ---"
echo "--- No cached network, copying transit data to build dir ---"
mkdir -p "$BUILD_DIR"
if [ ! -f "$OSM_PBF" ]; then
echo "Error: OSM PBF not found at $OSM_PBF"

View file

@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
* its own DuckDB connection. Routing tasks enqueue {@link WriteJob}s onto a
* bounded {@link ArrayBlockingQueue}; if writes lag, the queue applies backpressure
* to routing.
* - Within a mode, the latch counts down only after the write completes so progress
* - Within a mode, the latch counts down only after the write completes, so progress
* and the inter-mode barrier reflect fully-persisted work.
*/
public class App {
@ -253,8 +253,8 @@ public class App {
// Recycle the DuckDB connection every CONN_RECYCLE_EVERY writes. Long-lived
// in-memory DuckDB connections accumulate buffer pages / catalog state that
// doesn't get fully released by DROP TABLE alone; close + reopen forces it.
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50
// tighter recycling keeps the per-connection working set bounded.
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50.
// Tighter recycling keeps the per-connection working set bounded.
final int CONN_RECYCLE_EVERY = 10;
DuckDBConnection conn;
try {
@ -576,7 +576,7 @@ public class App {
/**
* Filter place indices to those near at least one England postcode.
* Uses a 0.1° grid (~11km cells) built from postcode locations a place is kept
* Uses a 0.1° grid (~11km cells) built from postcode locations: a place is kept
* if its grid cell or any adjacent cell contains an England postcode.
*/
private static Set<Integer> filterEnglandPlaces(

View file

@ -122,7 +122,7 @@ public class Parquet {
stmt.execute("COPY t TO '" + escapePath(tmp.toAbsolutePath().toString()) + "' (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1)");
// Drop the populated table NOW so DuckDB releases its in-memory storage
// for the next write. Without this, the previous origin's rows linger
// until the next call's DROP IF EXISTS accumulating across writers.
// until the next call's DROP IF EXISTS, accumulating across writers.
stmt.execute("DROP TABLE t");
}
Files.move(tmp, outPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);

View file

@ -114,13 +114,13 @@ public class Router {
/**
* Load or build the transport network with Kryo caching.
* The returned network is read-only after buildDistanceTables safe for concurrent use.
* The returned network is read-only after buildDistanceTables, safe for concurrent use.
*
* The evictable LinkageCache is left small (32 entries) because non-transit modes
* create one huge per-origin LinkedPointSet each (~1M dests for car @ 150km radius).
* Caching 1024 such entries OOMs the heap. Transit tile linkages instead go into
* the unevictable {@code linkageMap} via {@link #preloadTransitTileLinkages} after
* tiles are built that map has no count limit and is checked first on lookup.
* tiles are built. That map has no count limit and is checked first on lookup.
*/
static TransportNetwork loadNetwork(String dataDir, String cacheDir) throws Exception {
// Must be set BEFORE the TransportNetwork is deserialized, since its LinkageCache
@ -240,7 +240,7 @@ public class Router {
int n = lats.length;
int[] sorted = sortIndicesByLat(lats);
// Global lon span sets gridWidth all tiles share the same horizontal extent
// Global lon span sets gridWidth: all tiles share the same horizontal extent
// bound, so each tile is a horizontal band of postcodes.
double minLon = Double.MAX_VALUE, maxLon = -Double.MAX_VALUE;
for (double lon : lons) {
@ -390,7 +390,7 @@ public class Router {
/**
* Filter destination indices to those within a bounding box of maxRadiusKm from origin.
* Uses degree-based approximation slightly overestimates at corners, which is fine.
* Uses degree-based approximation. Slightly overestimates at corners, which is fine.
* Backed by STRtree: O(log n + k) per query instead of O(n) scan.
*/
@SuppressWarnings("unchecked")
@ -421,7 +421,7 @@ public class Router {
double[] lats, double[] lons, int maxDestsPerChunk) {
int n = lats.length;
// Sort indices by latitude for geographic chunking primitive long sort to
// Sort indices by latitude for geographic chunking: primitive long sort to
// avoid Integer[] autoboxing per origin (millions of Integer allocs at scale).
// Pack: high 32 bits = lat as sortable int, low 32 bits = original index.
int[] sorted = sortIndicesByLat(lats);