More FE changes
This commit is contained in:
parent
f114ada255
commit
a48eb945e0
48 changed files with 4127 additions and 1751 deletions
|
|
@ -17,14 +17,12 @@ export class ScreenshotCache {
|
|||
|
||||
/**
|
||||
* Build a cache key by quantizing view params and hashing.
|
||||
* - lat/lon quantized to 2 decimal places
|
||||
* - zoom quantized to integer
|
||||
* - filters, configurable filters, and POI categories sorted alphabetically
|
||||
* lat/lon are rounded to 2 decimals and zoom to an integer so nearby views
|
||||
* share a cache entry; all other params are sorted for order-independence.
|
||||
*/
|
||||
buildKey(params: URLSearchParams): string {
|
||||
const normalized: Record<string, string> = {};
|
||||
|
||||
// Quantize lat/lon/zoom
|
||||
const lat = params.get('lat');
|
||||
const lon = params.get('lon');
|
||||
const zoom = params.get('zoom');
|
||||
|
|
@ -34,44 +32,10 @@ export class ScreenshotCache {
|
|||
normalized.zoom = Math.round(parseFloat(zoom)).toString();
|
||||
}
|
||||
|
||||
// Sort filters
|
||||
const filters = params.getAll('filter').sort();
|
||||
if (filters.length > 0) {
|
||||
normalized.filters = filters.join(',');
|
||||
}
|
||||
|
||||
const schools = params.getAll('school').sort();
|
||||
if (schools.length > 0) {
|
||||
normalized.school = schools.join(',');
|
||||
}
|
||||
|
||||
const crimes = params.getAll('crime').sort();
|
||||
if (crimes.length > 0) {
|
||||
normalized.crime = crimes.join(',');
|
||||
}
|
||||
|
||||
// Sort POI categories
|
||||
const pois = params.getAll('poi').sort();
|
||||
if (pois.length > 0) {
|
||||
normalized.poi = pois.join(',');
|
||||
}
|
||||
|
||||
// Sort travel time entries
|
||||
const tt = params.getAll('tt').sort();
|
||||
if (tt.length > 0) {
|
||||
normalized.tt = tt.join(',');
|
||||
}
|
||||
|
||||
if (params.get('tab')) {
|
||||
normalized.tab = params.get('tab')!;
|
||||
}
|
||||
|
||||
if (params.get('og')) {
|
||||
normalized.og = params.get('og')!;
|
||||
}
|
||||
|
||||
if (params.get('path')) {
|
||||
normalized.path = params.get('path')!;
|
||||
const quantized = new Set(['lat', 'lon', 'zoom']);
|
||||
const keys = [...new Set(params.keys())].filter((k) => !quantized.has(k)).sort();
|
||||
for (const key of keys) {
|
||||
normalized[key] = params.getAll(key).sort().join(',');
|
||||
}
|
||||
|
||||
const input = JSON.stringify(normalized);
|
||||
|
|
|
|||
|
|
@ -1,28 +1,37 @@
|
|||
import express, { type Request, type Response } from 'express';
|
||||
import { ScreenshotCache } from './cache.js';
|
||||
import { takeScreenshot, checkWebGL, closeBrowser, initialize } from './screenshot.js';
|
||||
import { buildScreenshotRequest, ValidationError } from './validation.js';
|
||||
import express, { type Request, type Response } from "express";
|
||||
import { ScreenshotCache } from "./cache.js";
|
||||
import {
|
||||
takeScreenshot,
|
||||
checkWebGL,
|
||||
closeBrowser,
|
||||
initialize,
|
||||
} from "./screenshot.js";
|
||||
import { buildScreenshotRequest, ValidationError } from "./validation.js";
|
||||
|
||||
const PORT = parseInt(process.env.PORT || '8002', 10);
|
||||
const PORT = parseRequiredPositiveIntEnv("PORT");
|
||||
const APP_URL = process.env.APP_URL;
|
||||
const CACHE_DIR = process.env.CACHE_DIR;
|
||||
const SCREENSHOT_CONCURRENCY = parsePositiveIntEnv('SCREENSHOT_CONCURRENCY', 3);
|
||||
const RATE_LIMIT_WINDOW_MS = parsePositiveIntEnv('SCREENSHOT_RATE_WINDOW_MS', 60_000);
|
||||
const RATE_LIMIT_MAX = parsePositiveIntEnv('SCREENSHOT_RATE_LIMIT', 30);
|
||||
const SCREENSHOT_CONCURRENCY = parseRequiredPositiveIntEnv(
|
||||
"SCREENSHOT_CONCURRENCY",
|
||||
);
|
||||
const RATE_LIMIT_WINDOW_MS = parseRequiredPositiveIntEnv(
|
||||
"SCREENSHOT_RATE_WINDOW_MS",
|
||||
);
|
||||
const RATE_LIMIT_MAX = parseRequiredPositiveIntEnv("SCREENSHOT_RATE_LIMIT");
|
||||
|
||||
if (!APP_URL) {
|
||||
console.error('Error: APP_URL environment variable is required');
|
||||
console.error("Error: APP_URL environment variable is required");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
if (!CACHE_DIR) {
|
||||
console.error('Error: CACHE_DIR environment variable is required');
|
||||
console.error("Error: CACHE_DIR environment variable is required");
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const cache = new ScreenshotCache(CACHE_DIR);
|
||||
const app = express();
|
||||
app.set('trust proxy', true);
|
||||
app.set("trust proxy", true);
|
||||
|
||||
let activeScreenshots = 0;
|
||||
let lastRateLimitPrune = 0;
|
||||
|
|
@ -34,9 +43,16 @@ type PendingScreenshotSlot = {
|
|||
};
|
||||
const screenshotSlotQueue: PendingScreenshotSlot[] = [];
|
||||
|
||||
function parsePositiveIntEnv(name: string, fallback: number): number {
|
||||
const value = Number.parseInt(process.env[name] || '', 10);
|
||||
return Number.isFinite(value) && value > 0 ? value : fallback;
|
||||
function parseRequiredPositiveIntEnv(name: string): number {
|
||||
const raw = process.env[name];
|
||||
if (!raw) {
|
||||
throw new Error(`${name} environment variable is required`);
|
||||
}
|
||||
const value = Number.parseInt(raw, 10);
|
||||
if (!Number.isFinite(value) || value <= 0) {
|
||||
throw new Error(`${name} must be a positive integer`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
function grantScreenshotSlot(): ReleaseScreenshotSlot {
|
||||
|
|
@ -51,7 +67,10 @@ function grantScreenshotSlot(): ReleaseScreenshotSlot {
|
|||
}
|
||||
|
||||
function drainScreenshotSlotQueue(): void {
|
||||
while (activeScreenshots < SCREENSHOT_CONCURRENCY && screenshotSlotQueue.length > 0) {
|
||||
while (
|
||||
activeScreenshots < SCREENSHOT_CONCURRENCY &&
|
||||
screenshotSlotQueue.length > 0
|
||||
) {
|
||||
const pending = screenshotSlotQueue.shift();
|
||||
if (!pending) return;
|
||||
pending.cleanup();
|
||||
|
|
@ -59,7 +78,9 @@ function drainScreenshotSlotQueue(): void {
|
|||
}
|
||||
}
|
||||
|
||||
function acquireScreenshotSlot(res: Response): Promise<ReleaseScreenshotSlot | null> {
|
||||
function acquireScreenshotSlot(
|
||||
res: Response,
|
||||
): Promise<ReleaseScreenshotSlot | null> {
|
||||
if (activeScreenshots < SCREENSHOT_CONCURRENCY) {
|
||||
return Promise.resolve(grantScreenshotSlot());
|
||||
}
|
||||
|
|
@ -76,17 +97,23 @@ function acquireScreenshotSlot(res: Response): Promise<ReleaseScreenshotSlot | n
|
|||
|
||||
pending = {
|
||||
resolve,
|
||||
cleanup: () => res.off('close', onClose),
|
||||
cleanup: () => res.off("close", onClose),
|
||||
};
|
||||
res.on('close', onClose);
|
||||
res.on("close", onClose);
|
||||
screenshotSlotQueue.push(pending);
|
||||
console.log(`Queued screenshot request; queue length: ${screenshotSlotQueue.length}`);
|
||||
console.log(
|
||||
`Queued screenshot request; queue length: ${screenshotSlotQueue.length}`,
|
||||
);
|
||||
});
|
||||
}
|
||||
|
||||
function rateLimitKey(req: Request): string {
|
||||
const forwardedFor = req.get('x-forwarded-for')?.split(',')[0]?.trim();
|
||||
return forwardedFor || req.ip || req.socket.remoteAddress || 'unknown';
|
||||
const forwardedFor = req.get("x-forwarded-for")?.split(",")[0]?.trim();
|
||||
const key = forwardedFor || req.ip || req.socket.remoteAddress;
|
||||
if (!key) {
|
||||
throw new Error("Unable to determine request IP for rate limiting");
|
||||
}
|
||||
return key;
|
||||
}
|
||||
|
||||
function allowScreenshotRequest(req: Request): boolean {
|
||||
|
|
@ -113,11 +140,11 @@ function allowScreenshotRequest(req: Request): boolean {
|
|||
return true;
|
||||
}
|
||||
|
||||
app.get('/health', (_req, res) => {
|
||||
res.status(200).send('ok');
|
||||
app.get("/health", (_req, res) => {
|
||||
res.status(200).send("ok");
|
||||
});
|
||||
|
||||
app.get('/debug', async (_req, res) => {
|
||||
app.get("/debug", async (_req, res) => {
|
||||
try {
|
||||
const info = await checkWebGL();
|
||||
res.json(info);
|
||||
|
|
@ -126,32 +153,34 @@ app.get('/debug', async (_req, res) => {
|
|||
}
|
||||
});
|
||||
|
||||
app.get('/screenshot', async (req, res) => {
|
||||
app.get("/screenshot", async (req, res) => {
|
||||
let releaseSlot: (() => void) | null = null;
|
||||
try {
|
||||
const { pagePath, qs } = buildScreenshotRequest(req.query as Record<string, unknown>);
|
||||
if (pagePath !== '/') qs.set('path', pagePath);
|
||||
const { pagePath, qs } = buildScreenshotRequest(
|
||||
req.query as Record<string, unknown>,
|
||||
);
|
||||
if (pagePath !== "/") qs.set("path", pagePath);
|
||||
|
||||
// Include auth status in cache key so authenticated screenshots
|
||||
// (with hexagons outside free zone) are cached separately
|
||||
const authHeader = req.headers.authorization;
|
||||
if (authHeader) qs.set('_auth', '1');
|
||||
if (authHeader) qs.set("_auth", "1");
|
||||
const cacheKey = cache.buildKey(qs);
|
||||
qs.delete('_auth');
|
||||
qs.delete('path');
|
||||
qs.delete("_auth");
|
||||
qs.delete("path");
|
||||
|
||||
// Check cache first
|
||||
const cached = cache.get(cacheKey);
|
||||
if (cached) {
|
||||
res.setHeader('Content-Type', 'image/jpeg');
|
||||
res.setHeader('Cache-Control', 'public, max-age=86400');
|
||||
res.setHeader('X-Cache', 'HIT');
|
||||
res.setHeader("Content-Type", "image/jpeg");
|
||||
res.setHeader("Cache-Control", "public, max-age=86400");
|
||||
res.setHeader("X-Cache", "HIT");
|
||||
cached.pipe(res);
|
||||
return;
|
||||
}
|
||||
|
||||
if (!allowScreenshotRequest(req)) {
|
||||
res.status(429).json({ error: 'Screenshot rate limit exceeded' });
|
||||
res.status(429).json({ error: "Screenshot rate limit exceeded" });
|
||||
return;
|
||||
}
|
||||
|
||||
|
|
@ -161,26 +190,28 @@ app.get('/screenshot', async (req, res) => {
|
|||
}
|
||||
|
||||
// Build the URL for the frontend in screenshot mode
|
||||
qs.set('screenshot', '1');
|
||||
qs.set("screenshot", "1");
|
||||
const url = `${APP_URL}${pagePath}?${qs}`;
|
||||
|
||||
console.log(`Taking screenshot: ${url}${authHeader ? ' (authenticated)' : ''}`);
|
||||
console.log(
|
||||
`Taking screenshot: ${url}${authHeader ? " (authenticated)" : ""}`,
|
||||
);
|
||||
const jpeg = await takeScreenshot(url, authHeader);
|
||||
|
||||
// Cache it
|
||||
cache.set(cacheKey, jpeg);
|
||||
|
||||
res.setHeader('Content-Type', 'image/jpeg');
|
||||
res.setHeader('Cache-Control', 'public, max-age=86400');
|
||||
res.setHeader('X-Cache', 'MISS');
|
||||
res.setHeader("Content-Type", "image/jpeg");
|
||||
res.setHeader("Cache-Control", "public, max-age=86400");
|
||||
res.setHeader("X-Cache", "MISS");
|
||||
res.send(jpeg);
|
||||
} catch (err) {
|
||||
if (err instanceof ValidationError) {
|
||||
res.status(err.status).json({ error: err.message });
|
||||
return;
|
||||
}
|
||||
console.error('Screenshot failed:', err);
|
||||
res.status(500).json({ error: 'Screenshot failed' });
|
||||
console.error("Screenshot failed:", err);
|
||||
res.status(500).json({ error: "Screenshot failed" });
|
||||
} finally {
|
||||
releaseSlot?.();
|
||||
}
|
||||
|
|
@ -191,18 +222,20 @@ const server = app.listen(PORT, () => {
|
|||
console.log(` APP_URL: ${APP_URL}`);
|
||||
console.log(` CACHE_DIR: ${CACHE_DIR}`);
|
||||
console.log(` SCREENSHOT_CONCURRENCY: ${SCREENSHOT_CONCURRENCY}`);
|
||||
console.log(` SCREENSHOT_RATE_LIMIT: ${RATE_LIMIT_MAX}/${RATE_LIMIT_WINDOW_MS}ms`);
|
||||
console.log(
|
||||
` SCREENSHOT_RATE_LIMIT: ${RATE_LIMIT_MAX}/${RATE_LIMIT_WINDOW_MS}ms`,
|
||||
);
|
||||
|
||||
// Pre-warm browser and populate network cache in background.
|
||||
// The health endpoint is available immediately; screenshot requests
|
||||
// during warm-up will still work (just slower on the first call).
|
||||
initialize(APP_URL).catch((err) => {
|
||||
console.error('Initialization failed:', err);
|
||||
console.error("Initialization failed:", err);
|
||||
});
|
||||
});
|
||||
|
||||
// Graceful shutdown
|
||||
for (const signal of ['SIGTERM', 'SIGINT']) {
|
||||
for (const signal of ["SIGTERM", "SIGINT"]) {
|
||||
process.on(signal, async () => {
|
||||
console.log(`Received ${signal}, shutting down...`);
|
||||
server.close();
|
||||
|
|
|
|||
|
|
@ -14,6 +14,7 @@ test('buildScreenshotRequest accepts supported screenshot parameters', () => {
|
|||
filter: ['Last known price:100000:500000', 'Total floor area (sqm):50:150'],
|
||||
school: 'primary:good:2:1:10',
|
||||
crime: ['Burglary (avg/yr):0:5', 'Vehicle crime (avg/yr):0:10'],
|
||||
ethnicity: ['% White:10:80', '% South Asian:5:35'],
|
||||
poi: 'supermarket',
|
||||
tt: 'transit:kings-cross:Kings Cross:b:0:30',
|
||||
});
|
||||
|
|
@ -32,6 +33,7 @@ test('buildScreenshotRequest accepts supported screenshot parameters', () => {
|
|||
'Burglary (avg/yr):0:5',
|
||||
'Vehicle crime (avg/yr):0:10',
|
||||
]);
|
||||
assert.deepEqual(result.qs.getAll('ethnicity'), ['% White:10:80', '% South Asian:5:35']);
|
||||
});
|
||||
|
||||
test('buildScreenshotRequest rejects invalid numeric values', () => {
|
||||
|
|
|
|||
|
|
@ -12,7 +12,7 @@ const MAX_VALUE_LENGTH = 500;
|
|||
const NUMERIC_RE = /^-?(?:\d+|\d*\.\d+)$/;
|
||||
const PATH_RE = /^\/(?:invite\/[A-Za-z0-9]{1,20})?$/;
|
||||
const SAFE_VALUE_RE = /^[^\u0000-\u001f\u007f]+$/;
|
||||
const REPEATED_KEYS = ['filter', 'school', 'crime', 'poi', 'tt'] as const;
|
||||
const REPEATED_KEYS = ['filter', 'school', 'crime', 'ethnicity', 'poi', 'tt'] as const;
|
||||
|
||||
type Query = Record<string, unknown>;
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue