andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Various changes

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-03 19:26:34 +00:00
parent a42591c701
commit c388059f68
19 changed files with 1373 additions and 87 deletions
Download patch file Download diff file Expand all files Collapse all files

12
Taskfile.yml
View file

@ -41,6 +41,18 @@ tasks:
cmds: cmds:
- cargo run --release -- --data {{.WIDE_OUTPUT}} --pois {{.POIS_FILTERED_OUTPUT}} - cargo run --release -- --data {{.WIDE_OUTPUT}} --pois {{.POIS_FILTERED_OUTPUT}}
dev:og:
desc: Run OG screenshot sidecar on port 8002
dir: og-screenshot
env:
CACHE_DIR: /tmp/og-cache
NARROWIT_URL: http://localhost:3030
cmds:
- npm install
- npx playwright install --with-deps chromium
- npm run build
- npm start
dev:frontend: dev:frontend:
desc: Run frontend dev server on port 3030 (proxies /api to :8001) desc: Run frontend dev server on port 3030 (proxies /api to :8001)
dir: frontend dir: frontend

1
frontend/src/App.tsx
View file

@ -607,6 +607,7 @@ export default function App() {
onHexagonHover={() => {}} onHexagonHover={() => {}}
initialViewState={initialViewState} initialViewState={initialViewState}
theme={theme} theme={theme}
screenshotMode
/> />
</div> </div>
); );

10
frontend/src/components/PropertiesPane.tsx
View file

@ -239,6 +239,7 @@ function PropertyCard({ property }: { property: Property }) {
'number_habitable_rooms' 'number_habitable_rooms'
); );
const age = getNum(property, 'Approximate construction age', 'construction_age_band'); const age = getNum(property, 'Approximate construction age', 'construction_age_band');
const councilTax = getNum(property, 'Council tax (£/yr)');
const councilTaxD = getNum(property, 'Council tax Band D (£/yr)'); const councilTaxD = getNum(property, 'Council tax Band D (£/yr)');
return ( return (
@ -306,12 +307,17 @@ function PropertyCard({ property }: { property: Property }) {
{property.potential_energy_rating} {property.potential_energy_rating}
</div> </div>
)} )}
{councilTaxD !== undefined && ( {councilTax !== undefined ? (
<div>
<span className="text-warm-500 dark:text-warm-400">Council tax:</span> £
{fmt(councilTax)}/yr
</div>
) : councilTaxD !== undefined ? (
<div> <div>
<span className="text-warm-500 dark:text-warm-400">Council tax (D):</span> £ <span className="text-warm-500 dark:text-warm-400">Council tax (D):</span> £
{fmt(councilTaxD)}/yr {fmt(councilTaxD)}/yr
</div> </div>
)} ) : null}
</div> </div>
</div> </div>
); );

10
frontend/src/index.html
View file

@ -3,7 +3,15 @@
<head> <head>
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Narrowit</title> <title>Narrowit — Every neighbourhood in England & Wales</title>
<meta name="description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />
<meta property="og:title" content="Narrowit — Every neighbourhood in England & Wales" />
<meta property="og:description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />
<meta property="og:type" content="website" />
<meta property="og:image" content="/api/og-image" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:title" content="Narrowit — Every neighbourhood in England & Wales" />
<meta name="twitter:description" content="Explore property prices, energy ratings, crime stats, school ratings, and more across England & Wales on one interactive map." />
<script> <script>
(function() { (function() {
if (localStorage.getItem('theme') === 'dark') { if (localStorage.getItem('theme') === 'dark') {

854
server-rs/Cargo.lock generated
View file

File diff suppressed because it is too large Load diff

5
server-rs/Cargo.toml
View file

@ -5,7 +5,7 @@ edition = "2021"
[dependencies] [dependencies]
anyhow = "1" anyhow = "1"
clap = { version = "4", features = ["derive"] } clap = { version = "4", features = ["derive", "env"] }
axum = "0.8" axum = "0.8"
tower-http = { version = "0.6", features = ["cors", "fs", "compression-gzip", "compression-zstd", "trace"] } tower-http = { version = "0.6", features = ["cors", "fs", "compression-gzip", "compression-zstd", "trace"] }
tokio = { version = "1", features = ["full"] } tokio = { version = "1", features = ["full"] }
@ -18,6 +18,9 @@ lasso = "0.7"
rustc-hash = "2" rustc-hash = "2"
tracing = "0.1" tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] } tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
reqwest = { version = "0.12", features = ["rustls-tls"] }
regex = "1"
urlencoding = "2"
[lints.clippy] [lints.clippy]
min_ident_chars = "warn" min_ident_chars = "warn"

122
server-rs/src/features.rs
View file

@ -598,9 +598,114 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
}, },
], ],
}, },
FeatureGroup {
name: "Council Tax",
features: &[
FeatureConfig {
name: "Council tax (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Exact annual council tax based on the property's actual band",
detail: "Annual council tax for this property based on its actual VOA council tax band and the local authority's 2025-26 rates. Only available where the property's band was successfully matched from the VOA valuation list. For a dwelling occupied by two adults, including adult social care and parish precepts.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band A (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band A property (2-adult dwelling)",
detail: "Council tax for a Band A dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band A covers properties valued up to £40,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 6/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band B (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band B property (2-adult dwelling)",
detail: "Council tax for a Band B dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band B covers properties valued £40,001-£52,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 7/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band C (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band C property (2-adult dwelling)",
detail: "Council tax for a Band C dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band C covers properties valued £52,001-£68,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 8/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band D (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band D property (2-adult dwelling)",
detail: "Council tax for a Band D dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band D covers properties valued £68,001-£88,000 at 1 April 1991 and is the standard reference band used for national comparisons. Includes adult social care and parish precepts.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band E (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band E property (2-adult dwelling)",
detail: "Council tax for a Band E dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band E covers properties valued £88,001-£120,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 11/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band F (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band F property (2-adult dwelling)",
detail: "Council tax for a Band F dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band F covers properties valued £120,001-£160,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 13/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band G (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band G property (2-adult dwelling)",
detail: "Council tax for a Band G dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band G covers properties valued £160,001-£320,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 15/9.",
source: "council-tax",
},
FeatureConfig {
name: "Council tax Band H (£/yr)",
bounds: Bounds::Percentile {
low: 2.0,
high: 98.0,
},
step: 10.0,
description: "Annual council tax for a Band H property (2-adult dwelling)",
detail: "Council tax for a Band H dwelling occupied by two adults in the local authority area, for financial year 2025-26. Band H covers properties valued above £320,000 at 1 April 1991. Includes adult social care and parish precepts. The ratio to Band D is 18/9 (double Band D).",
source: "council-tax",
},
],
},
]; ];
pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[EnumFeatureGroup { pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[
EnumFeatureGroup {
name: "Property", name: "Property",
features: &[ features: &[
EnumFeatureConfig { EnumFeatureConfig {
@ -639,7 +744,20 @@ pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[EnumFeatureGroup {
source: "epc", source: "epc",
}, },
], ],
}]; },
EnumFeatureGroup {
name: "Council Tax",
features: &[
EnumFeatureConfig {
name: "Council tax band",
order: Some(&["A", "B", "C", "D", "E", "F", "G", "H"]),
description: "VOA council tax valuation band (A-H)",
detail: "The council tax band assigned by the Valuation Office Agency based on the estimated open market value of the property at 1 April 1991. Band A (up to £40k) to Band H (over £320k). Matched from the VOA valuation list by address within postcode.",
source: "council-tax",
},
],
},
];
/// Flat ordered list of all numeric feature names (follows group order). /// Flat ordered list of all numeric feature names (follows group order).
pub fn all_numeric_feature_names() -> Vec<&'static str> { pub fn all_numeric_feature_names() -> Vec<&'static str> {

11
server-rs/src/filter.rs
View file

@ -36,13 +36,20 @@ pub fn parse_filters(
let name = parts[0].trim(); let name = parts[0].trim();
let rest = parts[1].trim(); let rest = parts[1].trim();
if let Some(enum_idx) = enum_features.iter().position(|enum_feat| enum_feat.name == name) { if let Some(enum_idx) = enum_features
.iter()
.position(|enum_feat| enum_feat.name == name)
{
let enum_feat = &enum_features[enum_idx]; let enum_feat = &enum_features[enum_idx];
let allowed: Vec<u8> = rest let allowed: Vec<u8> = rest
.split('|') .split('|')
.filter_map(|value| { .filter_map(|value| {
let value = value.trim(); let value = value.trim();
enum_feat.values.iter().position(|existing| existing == value).map(|position| position as u8) enum_feat
.values
.iter()
.position(|existing| existing == value)
.map(|position| position as u8)
}) })
.collect(); .collect();
enums.push(ParsedEnumFilter { enum_idx, allowed }); enums.push(ParsedEnumFilter { enum_idx, allowed });

118
server-rs/src/main.rs
View file

@ -3,6 +3,7 @@ mod data;
mod features; mod features;
mod filter; mod filter;
mod grid_index; mod grid_index;
mod og_middleware;
mod routes; mod routes;
mod state; mod state;
#[cfg(test)] #[cfg(test)]
@ -12,6 +13,7 @@ use std::path::PathBuf;
use std::sync::Arc; use std::sync::Arc;
use anyhow::{bail, Context}; use anyhow::{bail, Context};
use axum::middleware;
use axum::routing::get; use axum::routing::get;
use axum::Router; use axum::Router;
use clap::Parser; use clap::Parser;
@ -38,6 +40,18 @@ struct Cli {
/// Path to the frontend dist directory /// Path to the frontend dist directory
#[arg(long)] #[arg(long)]
dist: Option<PathBuf>, dist: Option<PathBuf>,
/// URL of the OG screenshot sidecar (e.g. http://og-screenshot:8002)
#[arg(long, env = "OG_SIDECAR_URL")]
og_sidecar_url: Option<String>,
/// Public-facing URL for absolute og:image URLs
#[arg(
long,
env = "PUBLIC_URL",
default_value = "https://narrowit.schmelczer.dev"
)]
public_url: String,
} }
#[tokio::main] #[tokio::main]
@ -69,7 +83,11 @@ async fn main() -> anyhow::Result<()> {
); );
info!("Building spatial grid index (0.01° cells)"); info!("Building spatial grid index (0.01° cells)");
let grid = grid_index::GridIndex::build(&property_data.lat, &property_data.lon, consts::GRID_CELL_SIZE); let grid = grid_index::GridIndex::build(
&property_data.lat,
&property_data.lon,
consts::GRID_CELL_SIZE,
);
info!( info!(
"Precomputing H3 cells at resolution {}", "Precomputing H3 cells at resolution {}",
@ -88,7 +106,8 @@ async fn main() -> anyhow::Result<()> {
info!(pois = poi_data.lat.len(), "POI data loaded"); info!(pois = poi_data.lat.len(), "POI data loaded");
info!("Building POI spatial grid index"); info!("Building POI spatial grid index");
let poi_grid = grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE); let poi_grid =
grid_index::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);
let min_keys: Vec<String> = property_data let min_keys: Vec<String> = property_data
.feature_names .feature_names
@ -119,10 +138,7 @@ async fn main() -> anyhow::Result<()> {
for row in 0..num_pois { for row in 0..num_pois {
let category = poi_data.category.get(row).to_string(); let category = poi_data.category.get(row).to_string();
let group = poi_data.group.get(row).to_string(); let group = poi_data.group.get(row).to_string();
group_cats group_cats.entry(group).or_default().insert(category);
.entry(group)
.or_default()
.insert(category);
} }
// Validate that data groups match the hardcoded order exactly // Validate that data groups match the hardcoded order exactly
let expected: std::collections::HashSet<&str> = let expected: std::collections::HashSet<&str> =
@ -137,11 +153,17 @@ async fn main() -> anyhow::Result<()> {
missing_from_data, missing_from_order missing_from_data, missing_from_order
); );
} }
consts::POI_GROUP_ORDER.iter().map(|group_name| group_name.to_string()).collect::<Vec<_>>() consts::POI_GROUP_ORDER
.iter()
.map(|group_name| group_name.to_string())
.collect::<Vec<_>>()
.into_iter() .into_iter()
.map(|name| { .map(|name| {
let mut categories: Vec<String> = let mut categories: Vec<String> = group_cats
group_cats.remove(&name).context("POI group validated but missing from map")?.into_iter().collect(); .remove(&name)
.context("POI group validated but missing from map")?
.into_iter()
.collect();
categories.sort(); categories.sort();
Ok(state::POICategoryGroup { name, categories }) Ok(state::POICategoryGroup { name, categories })
}) })
@ -156,6 +178,45 @@ async fn main() -> anyhow::Result<()> {
.map(|(index, enum_feature)| (enum_feature.name.clone(), index)) .map(|(index, enum_feature)| (enum_feature.name.clone(), index))
.collect(); .collect();
// Read index.html at startup for crawler OG injection
let frontend_dist = cli.dist.unwrap_or_else(|| {
if let Ok(executable) = std::env::current_exe() {
let executable_dir = executable
.parent()
.unwrap_or_else(|| std::path::Path::new("."));
let dist_next_to_binary = executable_dir.join("dist");
if dist_next_to_binary.exists() {
return dist_next_to_binary;
}
}
PathBuf::from("frontend/dist")
});
let index_html = if frontend_dist.exists() {
let index_path = frontend_dist.join("index.html");
match std::fs::read_to_string(&index_path) {
Ok(html) => {
info!("Loaded index.html for OG injection");
Some(html)
}
Err(err) => {
tracing::warn!("Could not read index.html: {}", err);
None
}
}
} else {
None
};
let http_client = reqwest::Client::new();
if cli.og_sidecar_url.is_some() {
info!(
"OG sidecar configured: {}",
cli.og_sidecar_url.as_deref().unwrap()
);
}
let state = Arc::new(AppState { let state = Arc::new(AppState {
data: property_data, data: property_data,
grid, grid,
@ -168,6 +229,10 @@ async fn main() -> anyhow::Result<()> {
enum_max_keys, enum_max_keys,
poi_category_groups, poi_category_groups,
enum_name_to_idx, enum_name_to_idx,
og_sidecar_url: cli.og_sidecar_url,
public_url: cli.public_url,
index_html,
http_client,
}); });
let cors = CorsLayer::new() let cors = CorsLayer::new()
@ -181,6 +246,8 @@ async fn main() -> anyhow::Result<()> {
let state_poi_categories = state.clone(); let state_poi_categories = state.clone();
let state_hexagon_properties = state.clone(); let state_hexagon_properties = state.clone();
let state_hexagon_stats = state.clone(); let state_hexagon_stats = state.clone();
let state_og_image = state.clone();
let state_crawler = state.clone();
let api = Router::new() let api = Router::new()
.route( .route(
@ -208,26 +275,31 @@ async fn main() -> anyhow::Result<()> {
.route( .route(
"/api/hexagon-stats", "/api/hexagon-stats",
get(move |query| routes::get_hexagon_stats(state_hexagon_stats.clone(), query)), get(move |query| routes::get_hexagon_stats(state_hexagon_stats.clone(), query)),
)
.route(
"/api/og-image",
get(move |query| routes::get_og_image(state_og_image.clone(), query)),
); );
let frontend_dist = cli.dist.unwrap_or_else(|| {
// Check next to the binary first, then fall back to working directory
if let Ok(executable) = std::env::current_exe() {
let executable_dir = executable.parent().unwrap_or_else(|| std::path::Path::new("."));
let dist_next_to_binary = executable_dir.join("dist");
if dist_next_to_binary.exists() {
return dist_next_to_binary;
}
}
PathBuf::from("frontend/dist")
});
let app = if frontend_dist.exists() { let app = if frontend_dist.exists() {
api.fallback_service(ServeDir::new(frontend_dist)) api.fallback_service(ServeDir::new(&frontend_dist))
} else { } else {
api api
}; };
let app = app let app = app
.layer(middleware::from_fn(
move |req: axum::extract::Request, next: middleware::Next| {
let st = state_crawler.clone();
async move {
// Inject state into request extensions for the middleware
let (mut parts, body) = req.into_parts();
parts.extensions.insert(st);
let req = axum::extract::Request::from_parts(parts, body);
og_middleware::og_middleware(req, next).await
}
},
))
.layer(cors) .layer(cors)
.layer(CompressionLayer::new().zstd(true).gzip(true)) .layer(CompressionLayer::new().zstd(true).gzip(true))
.layer(TraceLayer::new_for_http()); .layer(TraceLayer::new_for_http());
@ -237,8 +309,6 @@ async fn main() -> anyhow::Result<()> {
.await .await
.with_context(|| format!("Failed to bind to {addr}"))?; .with_context(|| format!("Failed to bind to {addr}"))?;
info!("Server listening on {}", addr); info!("Server listening on {}", addr);
axum::serve(listener, app) axum::serve(listener, app).await.context("Server error")?;
.await
.context("Server error")?;
Ok(()) Ok(())
} }

65
server-rs/src/og_middleware.rs Normal file
View file

@ -0,0 +1,65 @@
use std::sync::Arc;
use axum::body::Body;
use axum::extract::Request;
use axum::http::header;
use axum::middleware::Next;
use axum::response::Response;
use regex::Regex;
use crate::state::AppState;
pub async fn og_middleware(request: Request, next: Next) -> Response {
// Capture the query string before passing the request through
let query_string = request.uri().query().unwrap_or("").to_string();
// Get state from extensions
let state = request.extensions().get::<Arc<AppState>>().cloned();
let response = next.run(request).await;
let content_type = response
.headers()
.get(header::CONTENT_TYPE)
.and_then(|val| val.to_str().ok())
.unwrap_or("");
if !content_type.contains("text/html") {
return response;
}
let state = match state {
Some(st) => st,
None => return response,
};
let index_html = match &state.index_html {
Some(html) => html,
None => return response,
};
// Build OG-injected HTML
let og_image_url = if query_string.is_empty() {
format!("{}/api/og-image", state.public_url)
} else {
format!("{}/api/og-image?{}", state.public_url, query_string)
};
let og_tags = format!(
r#"<title>Narrowit</title>
<meta property="og:title" content="Narrowit — UK Property Map" />
<meta property="og:description" content="Interactive property data visualization for England &amp; Wales" />
<meta property="og:image" content="{og_image_url}" />
<meta property="og:image:width" content="1200" />
<meta property="og:image:height" content="630" />
<meta name="twitter:card" content="summary_large_image" />
<meta name="twitter:image" content="{og_image_url}" />"#
);
// Replace the <title> tag with title + OG meta tags
let re = Regex::new(r"<title>Narrowit</title>").unwrap();
let html = re.replace(index_html, og_tags.as_str()).to_string();
let (parts, _body) = response.into_parts();
Response::from_parts(parts, Body::from(html))
}

7
server-rs/src/routes/features.rs
View file

@ -66,8 +66,11 @@ pub async fn get_features(state: Arc<AppState>) -> Json<FeaturesResponse> {
for feature_group in FEATURE_GROUPS { for feature_group in FEATURE_GROUPS {
if feature_group.name == group_name { if feature_group.name == group_name {
for feature_config in feature_group.features { for feature_config in feature_group.features {
if let Some(feat_idx) = if let Some(feat_idx) = state
state.data.feature_names.iter().position(|feat_name| feat_name == feature_config.name) .data
.feature_names
.iter()
.position(|feat_name| feat_name == feature_config.name)
{ {
let stats = &state.data.feature_stats[feat_idx]; let stats = &state.data.feature_stats[feat_idx];
features.push(FeatureInfo::Numeric { features.push(FeatureInfo::Numeric {

27
server-rs/src/routes/hexagon_stats.rs
View file

@ -31,7 +31,10 @@ pub async fn get_hexagon_stats(
) -> Result<impl IntoResponse, (StatusCode, String)> { ) -> Result<impl IntoResponse, (StatusCode, String)> {
let cell = h3o::CellIndex::from_str(&params.h3).map_err(|error| { let cell = h3o::CellIndex::from_str(&params.h3).map_err(|error| {
warn!(h3 = %params.h3, error = %error, "Invalid H3 cell index"); warn!(h3 = %params.h3, error = %error, "Invalid H3 cell index");
(StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {}", error)) (
StatusCode::BAD_REQUEST,
format!("Invalid H3 cell: {}", error),
)
})?; })?;
let cell_u64: u64 = cell.into(); let cell_u64: u64 = cell.into();
@ -60,13 +63,14 @@ pub async fn get_hexagon_stats(
// Parse optional `fields` param into sets of feature names. // Parse optional `fields` param into sets of feature names.
// None = include all, Some = only include listed features. // None = include all, Some = only include listed features.
let field_set: Option<std::collections::HashSet<String>> = params.fields.as_ref().map(|fields_str| { let field_set: Option<std::collections::HashSet<String>> =
fields_str params.fields.as_ref().map(|fields_str| {
.split(',') fields_str
.map(|field| field.trim().to_string()) .split(',')
.filter(|field| !field.is_empty()) .map(|field| field.trim().to_string())
.collect() .filter(|field| !field.is_empty())
}); .collect()
});
let result = tokio::task::spawn_blocking(move || { let result = tokio::task::spawn_blocking(move || {
let start_time = std::time::Instant::now(); let start_time = std::time::Instant::now();
@ -158,9 +162,10 @@ pub async fn get_hexagon_stats(
// Bin into histogram using global edges (cast to f64 for bin index math) // Bin into histogram using global edges (cast to f64 for bin index math)
if bin_width > 0.0 { if bin_width > 0.0 {
let bin_index = let bin_index = ((value as f64 - histogram_min as f64) / bin_width as f64)
((value as f64 - histogram_min as f64) / bin_width as f64).floor() as isize; .floor() as isize;
let clamped_index = bin_index.max(0).min((HISTOGRAM_BINS - 1) as isize) as usize; let clamped_index =
bin_index.max(0).min((HISTOGRAM_BINS - 1) as isize) as usize;
bins[clamped_index] += 1; bins[clamped_index] += 1;
} }
} }

94
server-rs/src/routes/hexagons.rs
View file

@ -115,7 +115,13 @@ impl CellAgg {
/// Add a row, only aggregating the features at the given indices. /// Add a row, only aggregating the features at the given indices.
#[inline] #[inline]
fn add_row_selective(&mut self, feature_data: &[f32], row: usize, num_features: usize, indices: &[usize]) { fn add_row_selective(
&mut self,
feature_data: &[f32],
row: usize,
num_features: usize,
indices: &[usize],
) {
self.count += 1; self.count += 1;
let base = row * num_features; let base = row * num_features;
for &feat_index in indices { for &feat_index in indices {
@ -133,7 +139,13 @@ impl CellAgg {
/// Track min/max ordinal index for selected enum features only. /// Track min/max ordinal index for selected enum features only.
#[inline] #[inline]
fn add_enums_selective(&mut self, enum_data: &[u8], row: usize, num_enums: usize, indices: &[usize]) { fn add_enums_selective(
&mut self,
enum_data: &[u8],
row: usize,
num_enums: usize,
indices: &[usize],
) {
let base = row * num_enums; let base = row * num_enums;
for &enum_index in indices { for &enum_index in indices {
let value = enum_data[base + enum_index]; let value = enum_data[base + enum_index];
@ -175,7 +187,9 @@ pub(crate) fn write_json_escaped(buf: &mut String, text: &str) {
'\n' => buf.push_str("\\n"), '\n' => buf.push_str("\\n"),
'\r' => buf.push_str("\\r"), '\r' => buf.push_str("\\r"),
'\t' => buf.push_str("\\t"), '\t' => buf.push_str("\\t"),
ctrl if ctrl < '\x20' => { let _ = write!(buf, "\\u{:04x}", ctrl as u32); } ctrl if ctrl < '\x20' => {
let _ = write!(buf, "\\u{:04x}", ctrl as u32);
}
other => buf.push(other), other => buf.push(other),
} }
} }
@ -214,21 +228,31 @@ fn write_hexagons_json(
if let Some(indices) = numeric_indices { if let Some(indices) = numeric_indices {
for &feat_index in indices { for &feat_index in indices {
if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite() { if aggregation.mins[feat_index].is_finite()
&& aggregation.maxs[feat_index].is_finite()
{
let _ = write!( let _ = write!(
buf, buf,
",\"{}\":{},\"{}\":{}", ",\"{}\":{},\"{}\":{}",
min_keys[feat_index], aggregation.mins[feat_index], max_keys[feat_index], aggregation.maxs[feat_index] min_keys[feat_index],
aggregation.mins[feat_index],
max_keys[feat_index],
aggregation.maxs[feat_index]
); );
} }
} }
} else { } else {
for feat_index in 0..num_features { for feat_index in 0..num_features {
if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite() { if aggregation.mins[feat_index].is_finite()
&& aggregation.maxs[feat_index].is_finite()
{
let _ = write!( let _ = write!(
buf, buf,
",\"{}\":{},\"{}\":{}", ",\"{}\":{},\"{}\":{}",
min_keys[feat_index], aggregation.mins[feat_index], max_keys[feat_index], aggregation.maxs[feat_index] min_keys[feat_index],
aggregation.mins[feat_index],
max_keys[feat_index],
aggregation.maxs[feat_index]
); );
} }
} }
@ -240,8 +264,10 @@ fn write_hexagons_json(
let _ = write!( let _ = write!(
buf, buf,
",\"{}\":{},\"{}\":{}", ",\"{}\":{},\"{}\":{}",
enum_min_keys[enum_index], aggregation.enum_mins[enum_index], enum_min_keys[enum_index],
enum_max_keys[enum_index], aggregation.enum_maxs[enum_index] aggregation.enum_mins[enum_index],
enum_max_keys[enum_index],
aggregation.enum_maxs[enum_index]
); );
} }
} }
@ -251,8 +277,10 @@ fn write_hexagons_json(
let _ = write!( let _ = write!(
buf, buf,
",\"{}\":{},\"{}\":{}", ",\"{}\":{},\"{}\":{}",
enum_min_keys[enum_index], aggregation.enum_mins[enum_index], enum_min_keys[enum_index],
enum_max_keys[enum_index], aggregation.enum_maxs[enum_index] aggregation.enum_mins[enum_index],
enum_max_keys[enum_index],
aggregation.enum_maxs[enum_index]
); );
} }
} }
@ -325,24 +353,30 @@ pub async fn get_hexagons(
// Parse optional `fields` param into numeric and enum index sets. // Parse optional `fields` param into numeric and enum index sets.
// If `fields` is absent (None), all features are included. // If `fields` is absent (None), all features are included.
// If `fields` is present (even empty string), only listed features are included. // If `fields` is present (even empty string), only listed features are included.
let field_indices: Option<(Vec<usize>, Vec<usize>)> = params.fields.as_ref().map(|fields_str| { let field_indices: Option<(Vec<usize>, Vec<usize>)> =
let mut numeric_indices = Vec::new(); params.fields.as_ref().map(|fields_str| {
let mut enum_indices = Vec::new(); let mut numeric_indices = Vec::new();
if !fields_str.is_empty() { let mut enum_indices = Vec::new();
for name in fields_str.split(',') { if !fields_str.is_empty() {
let name = name.trim(); for name in fields_str.split(',') {
if name.is_empty() { let name = name.trim();
continue; if name.is_empty() {
} continue;
if let Some(idx) = state.data.feature_names.iter().position(|feat| feat == name) { }
numeric_indices.push(idx); if let Some(idx) = state
} else if let Some(&idx) = state.enum_name_to_idx.get(name) { .data
enum_indices.push(idx); .feature_names
.iter()
.position(|feat| feat == name)
{
numeric_indices.push(idx);
} else if let Some(&idx) = state.enum_name_to_idx.get(name) {
enum_indices.push(idx);
}
} }
} }
} (numeric_indices, enum_indices)
(numeric_indices, enum_indices) });
});
let json_body = tokio::task::spawn_blocking(move || -> Result<String, String> { let json_body = tokio::task::spawn_blocking(move || -> Result<String, String> {
let t0 = std::time::Instant::now(); let t0 = std::time::Instant::now();
@ -380,7 +414,11 @@ pub async fn get_hexagons(
// Choose aggregation strategy based on whether fields are specified // Choose aggregation strategy based on whether fields are specified
let has_selective = field_indices.is_some(); let has_selective = field_indices.is_some();
let (sel_numeric, sel_enum) = field_indices.as_ref().map_or((&[][..], &[][..]), |(ni, ei)| (ni.as_slice(), ei.as_slice())); let (sel_numeric, sel_enum) = field_indices
.as_ref()
.map_or((&[][..], &[][..]), |(ni, ei)| {
(ni.as_slice(), ei.as_slice())
});
let aggregate_row = |groups: &mut FxHashMap<u64, CellAgg>, cell_id: u64, row: usize| { let aggregate_row = |groups: &mut FxHashMap<u64, CellAgg>, cell_id: u64, row: usize| {
let aggregation = groups let aggregation = groups

4
server-rs/src/routes/mod.rs
View file

@ -1,6 +1,7 @@
mod features; mod features;
pub(crate) mod hexagons;
mod hexagon_stats; mod hexagon_stats;
pub(crate) mod hexagons;
mod og_image;
pub(crate) mod parse; pub(crate) mod parse;
mod pois; mod pois;
pub(crate) mod properties; pub(crate) mod properties;
@ -8,5 +9,6 @@ pub(crate) mod properties;
pub use features::get_features; pub use features::get_features;
pub use hexagon_stats::get_hexagon_stats; pub use hexagon_stats::get_hexagon_stats;
pub use hexagons::get_hexagons; pub use hexagons::get_hexagons;
pub use og_image::get_og_image;
pub use pois::{get_poi_categories, get_pois}; pub use pois::{get_poi_categories, get_pois};
pub use properties::get_hexagon_properties; pub use properties::get_hexagon_properties;

80
server-rs/src/routes/og_image.rs Normal file
View file

@ -0,0 +1,80 @@
use std::sync::Arc;
use axum::extract::Query;
use axum::http::{header, StatusCode};
use axum::response::IntoResponse;
use crate::state::AppState;
#[derive(serde::Deserialize)]
pub struct OgImageQuery {
#[serde(rename = "v")]
view: Option<String>,
#[serde(rename = "f")]
filters: Option<String>,
poi: Option<String>,
tab: Option<String>,
}
pub async fn get_og_image(
state: Arc<AppState>,
Query(query): Query<OgImageQuery>,
) -> impl IntoResponse {
let sidecar_url = match &state.og_sidecar_url {
Some(url) => url,
None => {
return (StatusCode::SERVICE_UNAVAILABLE, "OG sidecar not configured").into_response();
}
};
let mut params = Vec::new();
if let Some(ref val) = query.view {
params.push(format!("v={}", urlencoding::encode(val)));
}
if let Some(ref val) = query.filters {
params.push(format!("f={}", urlencoding::encode(val)));
}
if let Some(ref val) = query.poi {
params.push(format!("poi={}", urlencoding::encode(val)));
}
if let Some(ref val) = query.tab {
params.push(format!("tab={}", urlencoding::encode(val)));
}
let qs = if params.is_empty() {
String::new()
} else {
format!("?{}", params.join("&"))
};
let url = format!("{}/screenshot{}", sidecar_url, qs);
tracing::info!("Proxying OG screenshot request to: {}", url);
match state.http_client.get(&url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
Ok(bytes) => (
StatusCode::OK,
[
(header::CONTENT_TYPE, "image/png"),
(header::CACHE_CONTROL, "public, max-age=86400"),
],
bytes,
)
.into_response(),
Err(err) => {
tracing::warn!("Failed to read sidecar response: {}", err);
(StatusCode::BAD_GATEWAY, "Failed to read screenshot").into_response()
}
},
Ok(resp) => {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
tracing::warn!("Sidecar returned status {}: {}", status, body);
(StatusCode::BAD_GATEWAY, "Screenshot sidecar error").into_response()
}
Err(err) => {
tracing::warn!("Failed to reach sidecar: {}", err);
(StatusCode::BAD_GATEWAY, "Screenshot sidecar unavailable").into_response()
}
}
}

6
server-rs/src/routes/pois.rs
View file

@ -35,7 +35,11 @@ pub async fn get_pois(
.categories .categories
.as_deref() .as_deref()
.filter(|text| !text.is_empty()) .filter(|text| !text.is_empty())
.map(|text| text.split(',').map(|part| part.trim().to_string()).collect()); .map(|text| {
text.split(',')
.map(|part| part.trim().to_string())
.collect()
});
let num_categories = category_filter.as_ref().map(|cats| cats.len()).unwrap_or(0); let num_categories = category_filter.as_ref().map(|cats| cats.len()).unwrap_or(0);

15
server-rs/src/routes/properties.rs
View file

@ -8,7 +8,10 @@ use rustc_hash::FxHashMap;
use serde::{Deserialize, Serialize}; use serde::{Deserialize, Serialize};
use tracing::{info, warn}; use tracing::{info, warn};
use crate::consts::{DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN, MAX_PROPERTIES_LIMIT}; use crate::consts::{
DEFAULT_PROPERTIES_LIMIT, ENUM_NULL, H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN,
MAX_PROPERTIES_LIMIT,
};
use crate::data::EnumFeatureData; use crate::data::EnumFeatureData;
use crate::filter::{parse_filters, row_passes_filters}; use crate::filter::{parse_filters, row_passes_filters};
use crate::state::AppState; use crate::state::AppState;
@ -91,7 +94,10 @@ pub async fn get_hexagon_properties(
) -> Result<Json<HexagonPropertiesResponse>, (StatusCode, String)> { ) -> Result<Json<HexagonPropertiesResponse>, (StatusCode, String)> {
let cell = h3o::CellIndex::from_str(&params.h3).map_err(|error| { let cell = h3o::CellIndex::from_str(&params.h3).map_err(|error| {
warn!(h3 = %params.h3, error = %error, "Invalid H3 cell index"); warn!(h3 = %params.h3, error = %error, "Invalid H3 cell index");
(StatusCode::BAD_REQUEST, format!("Invalid H3 cell: {}", error)) (
StatusCode::BAD_REQUEST,
format!("Invalid H3 cell: {}", error),
)
})?; })?;
let cell_u64: u64 = cell.into(); let cell_u64: u64 = cell.into();
@ -165,7 +171,10 @@ pub async fn get_hexagon_properties(
}); });
let total = matching_rows.len(); let total = matching_rows.len();
let limit = params.limit.unwrap_or(DEFAULT_PROPERTIES_LIMIT).min(MAX_PROPERTIES_LIMIT); let limit = params
.limit
.unwrap_or(DEFAULT_PROPERTIES_LIMIT)
.min(MAX_PROPERTIES_LIMIT);
let offset = params.offset.unwrap_or(0); let offset = params.offset.unwrap_or(0);
let truncated = total > offset + limit; let truncated = total > offset + limit;

8
server-rs/src/state.rs
View file

@ -30,4 +30,12 @@ pub struct AppState {
pub poi_category_groups: Vec<POICategoryGroup>, pub poi_category_groups: Vec<POICategoryGroup>,
/// Precomputed map from enum feature name to index in data.enum_features /// Precomputed map from enum feature name to index in data.enum_features
pub enum_name_to_idx: FxHashMap<String, usize>, pub enum_name_to_idx: FxHashMap<String, usize>,
/// URL of the OG screenshot sidecar service (e.g. http://og-screenshot:8002)
pub og_sidecar_url: Option<String>,
/// Public-facing URL for absolute og:image URLs (e.g. https://narrowit.schmelczer.dev)
pub public_url: String,
/// Contents of index.html read at startup, used for crawler OG injection
pub index_html: Option<String>,
/// Shared HTTP client for proxying to the screenshot sidecar
pub http_client: reqwest::Client,
} }

11
uv.lock generated
View file

@ -1122,6 +1122,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/6b/2b/55d980cfa2c93bd40ff4c290bf824d792bd41d2fe3487b07707559071760/numpy-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7b6b5e28bbd47b7532698e5db2fe1db693d84b58c254e4389d99a27bb9b8f6b", size = 18330053, upload-time = "2026-01-10T06:44:34.617Z" }, { url = "https://files.pythonhosted.org/packages/6b/2b/55d980cfa2c93bd40ff4c290bf824d792bd41d2fe3487b07707559071760/numpy-2.4.1-cp314-cp314t-musllinux_1_2_x86_64.whl", hash = "sha256:e7b6b5e28bbd47b7532698e5db2fe1db693d84b58c254e4389d99a27bb9b8f6b", size = 18330053, upload-time = "2026-01-10T06:44:34.617Z" },
] ]
[[package]]
name = "odfpy"
version = "1.4.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "defusedxml", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/97/73/8ade73f6749177003f7ce3304f524774adda96e6aaab30ea79fd8fda7934/odfpy-1.4.1.tar.gz", hash = "sha256:db766a6e59c5103212f3cc92ec8dd50a0f3a02790233ed0b52148b70d3c438ec", size = 717045, upload-time = "2020-01-18T16:55:48.852Z" }
[[package]] [[package]]
name = "osmium" name = "osmium"
version = "4.2.0" version = "4.2.0"
@ -1346,6 +1355,7 @@ dependencies = [
{ name = "jupyter", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "jupyter", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "matplotlib", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "matplotlib", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "numpy", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "numpy", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "odfpy", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "osmium", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "osmium", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "pandas", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "pandas", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
{ name = "plotly", marker = "python_full_version < '3.14' and sys_platform == 'linux'" }, { name = "plotly", marker = "python_full_version < '3.14' and sys_platform == 'linux'" },
@ -1376,6 +1386,7 @@ requires-dist = [
{ name = "jupyter", specifier = ">=1.0.0" }, { name = "jupyter", specifier = ">=1.0.0" },
{ name = "matplotlib", specifier = ">=3.10.8" }, { name = "matplotlib", specifier = ">=3.10.8" },
{ name = "numpy", specifier = ">=1.26.0" }, { name = "numpy", specifier = ">=1.26.0" },
{ name = "odfpy", specifier = ">=1.4.1" },
{ name = "osmium", specifier = ">=4.0.0" }, { name = "osmium", specifier = ">=4.0.0" },
{ name = "pandas", specifier = ">=2.0.0" }, { name = "pandas", specifier = ">=2.0.0" },
{ name = "plotly", specifier = ">=6.5.2" }, { name = "plotly", specifier = ">=6.5.2" },