andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

better transit times

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-22 11:13:39 +00:00
parent 974f005549
commit 205302dbb8
22 changed files with 247 additions and 69 deletions
Download patch file Download diff file Expand all files Collapse all files

31
r5-java/src/main/java/propertymap/App.java
View file

@ -9,6 +9,7 @@ import java.nio.file.Paths;
import java.time.LocalDate;
import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.ScheduledExecutorService;
@ -24,7 +25,8 @@ import java.util.concurrent.atomic.AtomicInteger;
* postcodes are written (unreachable = absent from file).
*
* Output per mode: one parquet file per origin in {output-dir}/{mode}/{name}.parquet
* with columns (pcds VARCHAR, travel_minutes SMALLINT).
* with columns (pcds VARCHAR, travel_minutes SMALLINT). Transit mode additionally
* includes a best_minutes SMALLINT column (5th percentile = best-case departure timing).
*/
public class App {
@ -117,15 +119,14 @@ public class App {
c, total, rate, etaH, failed.get());
}, 2, 2, TimeUnit.SECONDS);
// Submit all work, wait for completion via CountDownLatch-like pattern
java.util.concurrent.CountDownLatch latch = new java.util.concurrent.CountDownLatch(remaining.size());
CountDownLatch latch = new CountDownLatch(remaining.size());
for (int idx : remaining) {
pool.submit(() -> {
try {
processOrigin(network, postcodes, postcodeLats, postcodeLons,
originLats[idx], originLons[idx],
modeDir, mode, date, originNames[idx], threadConn.get());
modeDir, mode, date, idx, originNames[idx], threadConn.get());
completed.incrementAndGet();
} catch (Exception e) {
failed.incrementAndGet();
@ -138,6 +139,7 @@ public class App {
latch.await();
reporter.shutdown();
reporter.awaitTermination(5, TimeUnit.SECONDS);
double elapsedH = (System.currentTimeMillis() - startMs) / 3_600_000.0;
int n = completed.get();
@ -150,10 +152,10 @@ public class App {
TransportNetwork network,
String[] postcodes, double[] postcodeLats, double[] postcodeLons,
double originLat, double originLon,
Path modeDir, String mode, LocalDate date, String name,
Path modeDir, String mode, LocalDate date, int index, String name,
DuckDBConnection conn) throws Exception {
Path outPath = modeDir.resolve(sanitizeFilename(name) + ".parquet");
Path outPath = modeDir.resolve(originFilename(index, name));
Exception lastError = null;
for (int attempt = 0; attempt <= MAX_RETRIES; attempt++) {
@ -168,16 +170,22 @@ public class App {
String[] codes = new String[reachable];
short[] times = new short[reachable];
short[] bestTimes = result.bestTimes() != null ? new short[reachable] : null;
int j = 0;
for (int i = 0; i < result.times().length; i++) {
if (result.times()[i] >= 0) {
codes[j] = postcodes[result.originalIndices()[i]];
times[j] = result.times()[i];
if (bestTimes != null) bestTimes[j] = result.bestTimes()[i];
j++;
}
}
Parquet.writeTravelTimes(conn, outPath, codes, times);
if (bestTimes != null) {
Parquet.writeTransitTravelTimes(conn, outPath, codes, times, bestTimes);
} else {
Parquet.writeTravelTimes(conn, outPath, codes, times);
}
return;
} catch (Exception e) {
lastError = e;
@ -194,7 +202,7 @@ public class App {
private static List<Integer> findRemaining(Path modeDir, String[] names) throws Exception {
List<Integer> remaining = new ArrayList<>();
for (int i = 0; i < names.length; i++) {
Path f = modeDir.resolve(sanitizeFilename(names[i]) + ".parquet");
Path f = modeDir.resolve(originFilename(i, names[i]));
if (!Files.exists(f) || Files.size(f) == 0) {
remaining.add(i);
}
@ -202,11 +210,12 @@ public class App {
return remaining;
}
/** Sanitize a place name into a safe filename (lowercase, spaces to hyphens, strip non-alphanumeric). */
private static String sanitizeFilename(String name) {
return name.toLowerCase()
/** Build a filename from index + place name (index prefix prevents collisions after sanitization). */
private static String originFilename(int index, String name) {
String safe = name.toLowerCase()
.replaceAll("[^a-z0-9 -]", "")
.replaceAll("\\s+", "-");
return String.format("%04d-%s.parquet", index, safe);
}
private static String requiredArg(String[] args, String name) {