wip
This commit is contained in:
parent
f1601257c7
commit
c2070693fb
68 changed files with 2305 additions and 212 deletions
|
|
@ -7,7 +7,7 @@ set -euo pipefail
|
|||
# Uses full England OSM + 2 GTFS feeds (BODS buses, National Rail).
|
||||
# R5's TransportNetwork.fromDirectory() picks up all .osm.pbf and .zip files.
|
||||
#
|
||||
# Uses each place as origin with all postcodes as destinations — R5 does one
|
||||
# Uses each place as origin with all postcodes as destinations. R5 does one
|
||||
# routing computation per place, then reads off travel times to all postcodes.
|
||||
# For car/bicycle/walking this is symmetric (place->postcode = postcode->place).
|
||||
#
|
||||
|
|
@ -27,7 +27,7 @@ THREADS=12
|
|||
# The execution cgroup caps process memory at 48 GB (see /sys/fs/cgroup/memory.max);
|
||||
# the nominal "64 GB" host total is not all addressable to one process. 28g heap +
|
||||
# ~15g native overhead (DuckDB JNI, R5 mapdb, Kryo deserialize, RAPTOR scratch)
|
||||
# leaves ~5g cgroup headroom — empirically the safe ceiling before SIGKILL.
|
||||
# leaves ~5g cgroup headroom: empirically the safe ceiling before SIGKILL.
|
||||
# Under 32g also keeps CompressedOops on, halving R5's reference-heavy footprint.
|
||||
HEAP=28g
|
||||
NETWORK_DIR=property-data/r5-network
|
||||
|
|
@ -137,7 +137,7 @@ fi
|
|||
|
||||
if [ ! -f "$NETWORK_DIR/network.dat" ]; then
|
||||
BUILD_DIR="$NETWORK_DIR/build"
|
||||
echo "--- No cached network — copying transit data to build dir ---"
|
||||
echo "--- No cached network, copying transit data to build dir ---"
|
||||
mkdir -p "$BUILD_DIR"
|
||||
if [ ! -f "$OSM_PBF" ]; then
|
||||
echo "Error: OSM PBF not found at $OSM_PBF"
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
* its own DuckDB connection. Routing tasks enqueue {@link WriteJob}s onto a
|
||||
* bounded {@link ArrayBlockingQueue}; if writes lag, the queue applies backpressure
|
||||
* to routing.
|
||||
* - Within a mode, the latch counts down only after the write completes — so progress
|
||||
* - Within a mode, the latch counts down only after the write completes, so progress
|
||||
* and the inter-mode barrier reflect fully-persisted work.
|
||||
*/
|
||||
public class App {
|
||||
|
|
@ -253,8 +253,8 @@ public class App {
|
|||
// Recycle the DuckDB connection every CONN_RECYCLE_EVERY writes. Long-lived
|
||||
// in-memory DuckDB connections accumulate buffer pages / catalog state that
|
||||
// doesn't get fully released by DROP TABLE alone; close + reopen forces it.
|
||||
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50 —
|
||||
// tighter recycling keeps the per-connection working set bounded.
|
||||
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50.
|
||||
// Tighter recycling keeps the per-connection working set bounded.
|
||||
final int CONN_RECYCLE_EVERY = 10;
|
||||
DuckDBConnection conn;
|
||||
try {
|
||||
|
|
@ -576,7 +576,7 @@ public class App {
|
|||
|
||||
/**
|
||||
* Filter place indices to those near at least one England postcode.
|
||||
* Uses a 0.1° grid (~11km cells) built from postcode locations — a place is kept
|
||||
* Uses a 0.1° grid (~11km cells) built from postcode locations: a place is kept
|
||||
* if its grid cell or any adjacent cell contains an England postcode.
|
||||
*/
|
||||
private static Set<Integer> filterEnglandPlaces(
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ public class Parquet {
|
|||
stmt.execute("COPY t TO '" + escapePath(tmp.toAbsolutePath().toString()) + "' (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1)");
|
||||
// Drop the populated table NOW so DuckDB releases its in-memory storage
|
||||
// for the next write. Without this, the previous origin's rows linger
|
||||
// until the next call's DROP IF EXISTS — accumulating across writers.
|
||||
// until the next call's DROP IF EXISTS, accumulating across writers.
|
||||
stmt.execute("DROP TABLE t");
|
||||
}
|
||||
Files.move(tmp, outPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
|
|
|
|||
|
|
@ -114,13 +114,13 @@ public class Router {
|
|||
|
||||
/**
|
||||
* Load or build the transport network with Kryo caching.
|
||||
* The returned network is read-only after buildDistanceTables — safe for concurrent use.
|
||||
* The returned network is read-only after buildDistanceTables, safe for concurrent use.
|
||||
*
|
||||
* The evictable LinkageCache is left small (32 entries) because non-transit modes
|
||||
* create one huge per-origin LinkedPointSet each (~1M dests for car @ 150km radius).
|
||||
* Caching 1024 such entries OOMs the heap. Transit tile linkages instead go into
|
||||
* the unevictable {@code linkageMap} via {@link #preloadTransitTileLinkages} after
|
||||
* tiles are built — that map has no count limit and is checked first on lookup.
|
||||
* tiles are built. That map has no count limit and is checked first on lookup.
|
||||
*/
|
||||
static TransportNetwork loadNetwork(String dataDir, String cacheDir) throws Exception {
|
||||
// Must be set BEFORE the TransportNetwork is deserialized, since its LinkageCache
|
||||
|
|
@ -240,7 +240,7 @@ public class Router {
|
|||
int n = lats.length;
|
||||
int[] sorted = sortIndicesByLat(lats);
|
||||
|
||||
// Global lon span sets gridWidth — all tiles share the same horizontal extent
|
||||
// Global lon span sets gridWidth: all tiles share the same horizontal extent
|
||||
// bound, so each tile is a horizontal band of postcodes.
|
||||
double minLon = Double.MAX_VALUE, maxLon = -Double.MAX_VALUE;
|
||||
for (double lon : lons) {
|
||||
|
|
@ -390,7 +390,7 @@ public class Router {
|
|||
|
||||
/**
|
||||
* Filter destination indices to those within a bounding box of maxRadiusKm from origin.
|
||||
* Uses degree-based approximation — slightly overestimates at corners, which is fine.
|
||||
* Uses degree-based approximation. Slightly overestimates at corners, which is fine.
|
||||
* Backed by STRtree: O(log n + k) per query instead of O(n) scan.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
|
|
@ -421,7 +421,7 @@ public class Router {
|
|||
double[] lats, double[] lons, int maxDestsPerChunk) {
|
||||
int n = lats.length;
|
||||
|
||||
// Sort indices by latitude for geographic chunking — primitive long sort to
|
||||
// Sort indices by latitude for geographic chunking: primitive long sort to
|
||||
// avoid Integer[] autoboxing per origin (millions of Integer allocs at scale).
|
||||
// Pack: high 32 bits = lat as sortable int, low 32 bits = original index.
|
||||
int[] sorted = sortIndicesByLat(lats);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue