wip
This commit is contained in:
parent
f1601257c7
commit
c2070693fb
68 changed files with 2305 additions and 212 deletions
|
|
@ -47,7 +47,7 @@ import java.util.concurrent.atomic.AtomicInteger;
|
|||
* its own DuckDB connection. Routing tasks enqueue {@link WriteJob}s onto a
|
||||
* bounded {@link ArrayBlockingQueue}; if writes lag, the queue applies backpressure
|
||||
* to routing.
|
||||
* - Within a mode, the latch counts down only after the write completes — so progress
|
||||
* - Within a mode, the latch counts down only after the write completes, so progress
|
||||
* and the inter-mode barrier reflect fully-persisted work.
|
||||
*/
|
||||
public class App {
|
||||
|
|
@ -253,8 +253,8 @@ public class App {
|
|||
// Recycle the DuckDB connection every CONN_RECYCLE_EVERY writes. Long-lived
|
||||
// in-memory DuckDB connections accumulate buffer pages / catalog state that
|
||||
// doesn't get fully released by DROP TABLE alone; close + reopen forces it.
|
||||
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50 —
|
||||
// tighter recycling keeps the per-connection working set bounded.
|
||||
// Set to 10 (vs 50) after leak testing showed +60MB/origin growth at 50.
|
||||
// Tighter recycling keeps the per-connection working set bounded.
|
||||
final int CONN_RECYCLE_EVERY = 10;
|
||||
DuckDBConnection conn;
|
||||
try {
|
||||
|
|
@ -576,7 +576,7 @@ public class App {
|
|||
|
||||
/**
|
||||
* Filter place indices to those near at least one England postcode.
|
||||
* Uses a 0.1° grid (~11km cells) built from postcode locations — a place is kept
|
||||
* Uses a 0.1° grid (~11km cells) built from postcode locations: a place is kept
|
||||
* if its grid cell or any adjacent cell contains an England postcode.
|
||||
*/
|
||||
private static Set<Integer> filterEnglandPlaces(
|
||||
|
|
|
|||
|
|
@ -122,7 +122,7 @@ public class Parquet {
|
|||
stmt.execute("COPY t TO '" + escapePath(tmp.toAbsolutePath().toString()) + "' (FORMAT PARQUET, COMPRESSION ZSTD, COMPRESSION_LEVEL 1)");
|
||||
// Drop the populated table NOW so DuckDB releases its in-memory storage
|
||||
// for the next write. Without this, the previous origin's rows linger
|
||||
// until the next call's DROP IF EXISTS — accumulating across writers.
|
||||
// until the next call's DROP IF EXISTS, accumulating across writers.
|
||||
stmt.execute("DROP TABLE t");
|
||||
}
|
||||
Files.move(tmp, outPath, StandardCopyOption.REPLACE_EXISTING, StandardCopyOption.ATOMIC_MOVE);
|
||||
|
|
|
|||
|
|
@ -114,13 +114,13 @@ public class Router {
|
|||
|
||||
/**
|
||||
* Load or build the transport network with Kryo caching.
|
||||
* The returned network is read-only after buildDistanceTables — safe for concurrent use.
|
||||
* The returned network is read-only after buildDistanceTables, safe for concurrent use.
|
||||
*
|
||||
* The evictable LinkageCache is left small (32 entries) because non-transit modes
|
||||
* create one huge per-origin LinkedPointSet each (~1M dests for car @ 150km radius).
|
||||
* Caching 1024 such entries OOMs the heap. Transit tile linkages instead go into
|
||||
* the unevictable {@code linkageMap} via {@link #preloadTransitTileLinkages} after
|
||||
* tiles are built — that map has no count limit and is checked first on lookup.
|
||||
* tiles are built. That map has no count limit and is checked first on lookup.
|
||||
*/
|
||||
static TransportNetwork loadNetwork(String dataDir, String cacheDir) throws Exception {
|
||||
// Must be set BEFORE the TransportNetwork is deserialized, since its LinkageCache
|
||||
|
|
@ -240,7 +240,7 @@ public class Router {
|
|||
int n = lats.length;
|
||||
int[] sorted = sortIndicesByLat(lats);
|
||||
|
||||
// Global lon span sets gridWidth — all tiles share the same horizontal extent
|
||||
// Global lon span sets gridWidth: all tiles share the same horizontal extent
|
||||
// bound, so each tile is a horizontal band of postcodes.
|
||||
double minLon = Double.MAX_VALUE, maxLon = -Double.MAX_VALUE;
|
||||
for (double lon : lons) {
|
||||
|
|
@ -390,7 +390,7 @@ public class Router {
|
|||
|
||||
/**
|
||||
* Filter destination indices to those within a bounding box of maxRadiusKm from origin.
|
||||
* Uses degree-based approximation — slightly overestimates at corners, which is fine.
|
||||
* Uses degree-based approximation. Slightly overestimates at corners, which is fine.
|
||||
* Backed by STRtree: O(log n + k) per query instead of O(n) scan.
|
||||
*/
|
||||
@SuppressWarnings("unchecked")
|
||||
|
|
@ -421,7 +421,7 @@ public class Router {
|
|||
double[] lats, double[] lons, int maxDestsPerChunk) {
|
||||
int n = lats.length;
|
||||
|
||||
// Sort indices by latitude for geographic chunking — primitive long sort to
|
||||
// Sort indices by latitude for geographic chunking: primitive long sort to
|
||||
// avoid Integer[] autoboxing per origin (millions of Integer allocs at scale).
|
||||
// Pack: high 32 bits = lat as sortable int, low 32 bits = original index.
|
||||
int[] sorted = sortIndicesByLat(lats);
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue