andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Last night

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-08 10:21:37 +00:00
parent 2906b01734
commit 42ee2d4c51
47 changed files with 848 additions and 478 deletions
Download patch file Download diff file Expand all files Collapse all files

29
server-rs/Cargo.lock generated
View file

@ -2367,7 +2367,6 @@ dependencies = [
"anyhow",
"axum",
"clap",
"futures",
"h3o",
"lasso",
"metrics",
@ -2383,7 +2382,6 @@ dependencies = [
"serde",
"serde_json",
"tokio",
"tokio-stream",
"tower-http",
"tracing",
"tracing-subscriber",
@ -2660,7 +2658,6 @@ dependencies = [
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
@ -2685,14 +2682,12 @@ dependencies = [
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
]
@ -3328,17 +3323,6 @@ dependencies = [
"tokio",
]
[[package]]
name = "tokio-stream"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "32da49809aab5c3bc678af03902d4ccddea2a87d028d86392a4b1560c6906c70"
dependencies = [
"futures-core",
"pin-project-lite",
"tokio",
]
[[package]]
name = "tokio-util"
version = "0.7.18"
@ -3673,19 +3657,6 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.85"

4
server-rs/Cargo.toml
View file

@ -21,9 +21,7 @@ tracing = "0.1"
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
metrics = "0.24"
metrics-exporter-prometheus = "0.16"
reqwest = { version = "0.12", features = ["rustls-tls", "json", "stream"] }
futures = "0.3"
tokio-stream = "0.1"
reqwest = { version = "0.12", features = ["rustls-tls", "json"] }
regex = "1"
urlencoding = "2"
rust_xlsxwriter = "0.79"

29
server-rs/src/auth.rs
View file

@ -18,10 +18,6 @@ pub struct PocketBaseUser {
pub id: String,
pub email: String,
#[serde(default)]
pub name: String,
#[serde(default)]
pub avatar: String,
#[serde(default)]
pub verified: bool,
}
@ -94,21 +90,10 @@ async fn validate_token(
}
pub async fn auth_middleware(req: Request, next: Next) -> Response {
let pocketbase_url = req
let state = req
.extensions()
.get::<Arc<crate::state::AppState>>()
.and_then(|st| st.pocketbase_url.as_deref())
.map(String::from);
let token_cache = req
.extensions()
.get::<Arc<crate::state::AppState>>()
.map(|st| st.token_cache.clone());
let http_client = req
.extensions()
.get::<Arc<crate::state::AppState>>()
.map(|st| st.http_client.clone());
.cloned();
let token = req
.headers()
@ -117,14 +102,14 @@ pub async fn auth_middleware(req: Request, next: Next) -> Response {
.and_then(|hv| hv.strip_prefix("Bearer "))
.map(String::from);
let user = match (&pocketbase_url, &token, &token_cache, &http_client) {
(Some(pb_url), Some(tk), Some(cache), Some(client)) => {
if let Some(cached) = cache.get(tk) {
let user = match (&state, &token) {
(Some(st), Some(tk)) => {
if let Some(cached) = st.token_cache.get(tk) {
Some(cached)
} else {
match validate_token(client, pb_url, tk).await {
match validate_token(&st.http_client, &st.pocketbase_url, tk).await {
Some(user) => {
cache.insert(tk.clone(), user.clone());
st.token_cache.insert(tk.clone(), user.clone());
Some(user)
}
None => {

4
server-rs/src/consts.rs
View file

@ -10,3 +10,7 @@ pub const GRID_CELL_SIZE: f32 = 0.01;
pub const MAX_POIS_PER_REQUEST: usize = 2500;
pub const DEFAULT_PROPERTIES_LIMIT: usize = 100;
pub const MAX_PROPERTIES_LIMIT: usize = 500;
pub const AREA_SUMMARY_SYSTEM_PROMPT: &str = "You are an experienced estate agent with an expertise in area analysis. Help the user find his/her dream area or perfect postcode to settle in. The user is looking to buy a property based on the filters they provide. Given area statistics, write at most a single concise sentences summarising the key characteristics of the area. Be factual and highlight notable values. Do not use bullet points or headers — just flowing prose. Do not use markdown formatting. Highlight unusual facts that stand out from the average, but do not exaggerate. If there are no notable characteristics, say so. Always write at most a single sentence! Reason about the relation of different statistics to each other.";
pub const AREA_SUMMARY_MAX_TOKENS: usize = 300;
pub const AREA_SUMMARY_TEMPERATURE: f32 = 0.3;

55
server-rs/src/main.rs
View file

@ -14,12 +14,13 @@ use std::sync::Arc;
use anyhow::{bail, Context};
use axum::middleware;
use axum::routing::{any, get};
use axum::routing::{any, get, post};
use axum::Router;
use clap::Parser;
use tower_http::compression::CompressionLayer;
use tower_http::cors::{Any, CorsLayer};
use tower_http::services::ServeDir;
use tower_http::services::{ServeDir, ServeFile};
use tower_http::trace::TraceLayer;
use tracing::{info, warn};
use tracing_subscriber::EnvFilter;
@ -49,9 +50,9 @@ struct Cli {
#[arg(long)]
dist: Option<PathBuf>,
/// URL of the OG screenshot sidecar (e.g. http://og-screenshot:8002)
#[arg(long, env = "OG_SIDECAR_URL")]
og_sidecar_url: Option<String>,
/// URL of the screenshot service (e.g. http://screenshot:8002)
#[arg(long, env = "SCREENSHOT_URL")]
screenshot_url: String,
/// Public-facing URL for absolute og:image URLs
#[arg(
@ -63,7 +64,15 @@ struct Cli {
/// PocketBase server URL for authentication (e.g. http://localhost:8090)
#[arg(long, env = "POCKETBASE_URL")]
pocketbase_url: Option<String>,
pocketbase_url: String,
/// Ollama server URL for AI area summaries (e.g. http://ollama:11434)
#[arg(long, env = "OLLAMA_URL")]
ollama_url: String,
/// Ollama model name for area summaries
#[arg(long, env = "OLLAMA_MODEL", default_value = "gemma3:12b")]
ollama_model: String,
}
#[tokio::main]
@ -168,6 +177,11 @@ async fn main() -> anyhow::Result<()> {
.iter()
.map(|name| format!("max_{}", name))
.collect();
let avg_keys: Vec<String> = property_data
.feature_names
.iter()
.map(|name| format!("avg_{}", name))
.collect();
let poi_category_groups = poi_data.category_groups()?;
@ -203,12 +217,7 @@ async fn main() -> anyhow::Result<()> {
let http_client = reqwest::Client::new();
if cli.og_sidecar_url.is_some() {
info!(
"OG sidecar configured: {}",
cli.og_sidecar_url.as_deref().unwrap()
);
}
info!("Screenshot service configured: {}", cli.screenshot_url);
let features_response = routes::build_features_response(&property_data);
info!(
@ -223,9 +232,8 @@ async fn main() -> anyhow::Result<()> {
postcode_data.postcodes.len(),
);
if let Some(ref pb_url) = cli.pocketbase_url {
info!("PocketBase configured: {}", pb_url);
}
info!("PocketBase configured: {}", cli.pocketbase_url);
info!("Ollama configured: {} (model: {})", cli.ollama_url, cli.ollama_model);
let token_cache = Arc::new(auth::TokenCache::new());
@ -239,13 +247,16 @@ async fn main() -> anyhow::Result<()> {
feature_name_to_index,
min_keys,
max_keys,
avg_keys,
poi_category_groups,
features_response,
og_sidecar_url: cli.og_sidecar_url,
screenshot_url: cli.screenshot_url,
public_url: cli.public_url,
index_html,
http_client,
pocketbase_url: cli.pocketbase_url,
ollama_url: cli.ollama_url,
ollama_model: cli.ollama_model,
token_cache,
});
@ -266,6 +277,7 @@ async fn main() -> anyhow::Result<()> {
let state_export = state.clone();
let state_crawler = state.clone();
let state_pb = state.clone();
let state_area_summary = state.clone();
let api = Router::new()
.route(
@ -310,7 +322,11 @@ async fn main() -> anyhow::Result<()> {
"/api/export",
get(move |query| routes::get_export(state_export.clone(), query)),
)
.route("/api/me", get(routes::get_me));
.route("/api/me", get(routes::get_me))
.route(
"/api/area-summary",
post(move |body| routes::post_area_summary(state_area_summary.clone(), body)),
);
// Add tile routes
let reader_tile = tile_reader.clone();
@ -336,7 +352,10 @@ async fn main() -> anyhow::Result<()> {
);
let app = if frontend_dist.exists() {
api.fallback_service(ServeDir::new(&frontend_dist))
api.fallback_service(
ServeDir::new(&frontend_dist)
.not_found_service(ServeFile::new(frontend_dist.join("index.html"))),
)
} else {
api
};

6
server-rs/src/og_middleware.rs
View file

@ -39,11 +39,11 @@ pub async fn og_middleware(request: Request, next: Next) -> Response {
None => return response,
};
// Build OG-injected HTML
// Build OG-injected HTML (og=1 triggers heading overlay on screenshot)
let og_image_url = if query_string.is_empty() {
format!("{}/api/og-image", state.public_url)
format!("{}/api/og-image?og=1", state.public_url)
} else {
format!("{}/api/og-image?{}", state.public_url, query_string)
format!("{}/api/og-image?og=1&{}", state.public_url, query_string)
};
let og_tags = format!(

12
server-rs/src/routes/area_summary.rs
View file

@ -45,12 +45,12 @@ fn build_prompt(req: &AreaSummaryRequest) -> String {
let area_type = if req.is_postcode { "postcode" } else { "area" };
parts.push(format!(
"Summarise this {} of England ({}) which contain {} properties matching the filters.",
area_type, req.location, req.count
"Summarise this {} of England which contains {} properties matching my requirements.\n",
area_type, req.count
));
if !req.filters.is_empty() {
parts.push(format!("Active filters: {}.", req.filters.join(", ")));
parts.push(format!("Active filters: {}.\n", req.filters.join(", ")));
}
if !req.numeric_stats.is_empty() {
@ -59,7 +59,11 @@ fn build_prompt(req: &AreaSummaryRequest) -> String {
.iter()
.map(|stat| format!("{}: {:.1}", stat.name, stat.mean))
.collect();
parts.push(format!("Average values: {}.", stats.join(", ")));
parts.push(format!(
"Average values of the {}: {}.",
if req.is_postcode { "postcode" } else { "area" },
stats.join(", ")
));
}
for es in &req.enum_stats {

20
server-rs/src/routes/export.rs
View file

@ -93,13 +93,13 @@ fn extract_filter_feature_names(filters_str: Option<&str>) -> Vec<String> {
names
}
/// Fetch the OG screenshot image from the sidecar service.
async fn fetch_og_image(
/// Fetch a screenshot image from the screenshot service for Excel export.
async fn fetch_screenshot(
state: &AppState,
view_param: &str,
filters_str: Option<&str>,
) -> Option<Vec<u8>> {
let sidecar_url = state.og_sidecar_url.as_deref()?;
let screenshot_base = &state.screenshot_url;
let mut params = vec![format!("v={}", urlencoding::encode(view_param))];
if let Some(fs) = filters_str {
@ -107,25 +107,25 @@ async fn fetch_og_image(
params.push(format!("f={}", urlencoding::encode(fs)));
}
}
let url = format!("{}/screenshot?{}", sidecar_url, params.join("&"));
let url = format!("{}/screenshot?{}", screenshot_base, params.join("&"));
match state.http_client.get(&url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
Ok(bytes) => {
info!(bytes = bytes.len(), "Fetched OG image for export");
info!(bytes = bytes.len(), "Fetched screenshot for export");
Some(bytes.to_vec())
}
Err(err) => {
warn!("Failed to read OG sidecar response for export: {err}");
warn!("Failed to read screenshot response for export: {err}");
None
}
},
Ok(resp) => {
warn!(status = %resp.status(), "OG sidecar returned error for export");
warn!(status = %resp.status(), "Screenshot service returned error for export");
None
}
Err(err) => {
warn!("Failed to reach OG sidecar for export: {err}");
warn!("Failed to reach screenshot service for export: {err}");
None
}
}
@ -163,8 +163,8 @@ pub async fn get_export(
};
let view_param = format!("{:.4},{:.4},{:.1}", center_lat, center_lon, zoom);
// Fetch OG image from sidecar (async, before spawn_blocking)
let og_image_bytes = fetch_og_image(&state, &view_param, filters_str.as_deref()).await;
// Fetch screenshot (async, before spawn_blocking)
let og_image_bytes = fetch_screenshot(&state, &view_param, filters_str.as_deref()).await;
// Build feature name → description map from the precomputed features response
let feature_descriptions: FxHashMap<String, String> = state

27
server-rs/src/routes/hexagons.rs
View file

@ -38,6 +38,8 @@ struct CellAgg {
count: u32,
mins: Box<[f32]>,
maxs: Box<[f32]>,
sums: Box<[f64]>,
feat_counts: Box<[u32]>,
}
impl CellAgg {
@ -46,6 +48,8 @@ impl CellAgg {
count: 0,
mins: vec![f32::INFINITY; num_features].into_boxed_slice(),
maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(),
sums: vec![0.0f64; num_features].into_boxed_slice(),
feat_counts: vec![0u32; num_features].into_boxed_slice(),
}
}
@ -65,6 +69,8 @@ impl CellAgg {
if value > self.maxs[feat_index] {
self.maxs[feat_index] = value;
}
self.sums[feat_index] += value as f64;
self.feat_counts[feat_index] += 1;
}
}
}
@ -89,6 +95,8 @@ impl CellAgg {
if value > self.maxs[feat_index] {
self.maxs[feat_index] = value;
}
self.sums[feat_index] += value as f64;
self.feat_counts[feat_index] += 1;
}
}
}
@ -99,6 +107,7 @@ fn build_feature_maps(
groups: &FxHashMap<u64, CellAgg>,
min_keys: &[String],
max_keys: &[String],
avg_keys: &[String],
num_features: usize,
indices: Option<&[usize]>,
query_bounds: (f64, f64, f64, f64), // (south, west, north, east)
@ -122,6 +131,14 @@ fn build_feature_maps(
let mut map = Map::new();
map.insert("h3".into(), Value::String(cell.to_string()));
map.insert("count".into(), Value::Number(aggregation.count.into()));
let center: h3o::LatLng = cell.into();
if let (Some(lat), Some(lon)) = (
serde_json::Number::from_f64(center.lat()),
serde_json::Number::from_f64(center.lng()),
) {
map.insert("lat".into(), Value::Number(lat));
map.insert("lon".into(), Value::Number(lon));
}
let iter: Box<dyn Iterator<Item = usize>> = if let Some(idx) = indices {
Box::new(idx.iter().copied())
@ -130,14 +147,16 @@ fn build_feature_maps(
};
for feat_index in iter {
if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite()
{
if let (Some(min_num), Some(max_num)) = (
if aggregation.feat_counts[feat_index] > 0 {
let avg = aggregation.sums[feat_index] / aggregation.feat_counts[feat_index] as f64;
if let (Some(min_num), Some(max_num), Some(avg_num)) = (
serde_json::Number::from_f64(aggregation.mins[feat_index] as f64),
serde_json::Number::from_f64(aggregation.maxs[feat_index] as f64),
serde_json::Number::from_f64(avg),
) {
map.insert(min_keys[feat_index].clone(), Value::Number(min_num));
map.insert(max_keys[feat_index].clone(), Value::Number(max_num));
map.insert(avg_keys[feat_index].clone(), Value::Number(avg_num));
}
}
}
@ -208,6 +227,7 @@ pub async fn get_hexagons(
let feature_data = &state.data.feature_data;
let min_keys = &state.min_keys;
let max_keys = &state.max_keys;
let avg_keys = &state.avg_keys;
let h3_res = h3o::Resolution::try_from(resolution)
.map_err(|error| format!("Invalid H3 resolution {}: {}", resolution, error))?;
@ -277,6 +297,7 @@ pub async fn get_hexagons(
&groups,
min_keys,
max_keys,
avg_keys,
num_features,
field_indices.as_deref(),
(south, west, north, east),

27
server-rs/src/routes/og_image.rs
View file

@ -15,20 +15,20 @@ pub struct OgImageQuery {
filters: Option<String>,
poi: Option<String>,
tab: Option<String>,
/// When "1", renders the OG heading overlay on the screenshot
og: Option<String>,
}
pub async fn get_og_image(
state: Arc<AppState>,
Query(query): Query<OgImageQuery>,
) -> impl IntoResponse {
let sidecar_url = match &state.og_sidecar_url {
Some(url) => url,
None => {
return (StatusCode::SERVICE_UNAVAILABLE, "OG sidecar not configured").into_response();
}
};
let screenshot_base = &state.screenshot_url;
let mut params = Vec::new();
if query.og.as_deref() == Some("1") {
params.push("og=1".to_string());
}
if let Some(ref val) = query.view {
params.push(format!("v={}", urlencoding::encode(val)));
}
@ -47,9 +47,8 @@ pub async fn get_og_image(
} else {
format!("?{}", params.join("&"))
};
let url = format!("{}/screenshot{}", sidecar_url, qs);
info!("Proxying OG screenshot request to: {}", url);
let url = format!("{}/screenshot{}", screenshot_base, qs);
info!("Proxying screenshot request to: {}", url);
match state.http_client.get(&url).send().await {
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
@ -63,19 +62,19 @@ pub async fn get_og_image(
)
.into_response(),
Err(err) => {
warn!("Failed to read sidecar response: {}", err);
warn!("Failed to read screenshot response: {}", err);
(StatusCode::BAD_GATEWAY, "Failed to read screenshot").into_response()
}
},
Ok(resp) => {
let status = resp.status();
let body = resp.text().await.unwrap_or_default();
warn!("Sidecar returned status {}: {}", status, body);
(StatusCode::BAD_GATEWAY, "Screenshot sidecar error").into_response()
warn!("Screenshot service returned status {}: {}", status, body);
(StatusCode::BAD_GATEWAY, "Screenshot service error").into_response()
}
Err(err) => {
warn!("Failed to reach sidecar: {}", err);
(StatusCode::BAD_GATEWAY, "Screenshot sidecar unavailable").into_response()
warn!("Failed to reach screenshot service: {}", err);
(StatusCode::BAD_GATEWAY, "Screenshot service unavailable").into_response()
}
}
}

18
server-rs/src/routes/postcodes.rs
View file

@ -31,6 +31,8 @@ struct PostcodeAgg {
count: u32,
mins: Box<[f32]>,
maxs: Box<[f32]>,
sums: Box<[f64]>,
feat_counts: Box<[u32]>,
}
impl PostcodeAgg {
@ -39,6 +41,8 @@ impl PostcodeAgg {
count: 0,
mins: vec![f32::INFINITY; num_features].into_boxed_slice(),
maxs: vec![f32::NEG_INFINITY; num_features].into_boxed_slice(),
sums: vec![0.0f64; num_features].into_boxed_slice(),
feat_counts: vec![0u32; num_features].into_boxed_slice(),
}
}
@ -55,6 +59,8 @@ impl PostcodeAgg {
if value > self.maxs[feat_index] {
self.maxs[feat_index] = value;
}
self.sums[feat_index] += value as f64;
self.feat_counts[feat_index] += 1;
}
}
}
@ -78,6 +84,8 @@ impl PostcodeAgg {
if value > self.maxs[feat_index] {
self.maxs[feat_index] = value;
}
self.sums[feat_index] += value as f64;
self.feat_counts[feat_index] += 1;
}
}
}
@ -127,6 +135,7 @@ pub async fn get_postcodes(
let feature_data = &state.data.feature_data;
let min_keys = &state.min_keys;
let max_keys = &state.max_keys;
let avg_keys = &state.avg_keys;
let has_selective = field_indices.is_some();
let sel_indices = field_indices.as_deref().unwrap_or(&[]);
@ -272,15 +281,16 @@ pub async fn get_postcodes(
};
for feat_index in iter {
if aggregation.mins[feat_index].is_finite()
&& aggregation.maxs[feat_index].is_finite()
{
if let (Some(min_num), Some(max_num)) = (
if aggregation.feat_counts[feat_index] > 0 {
let avg = aggregation.sums[feat_index] / aggregation.feat_counts[feat_index] as f64;
if let (Some(min_num), Some(max_num), Some(avg_num)) = (
serde_json::Number::from_f64(aggregation.mins[feat_index] as f64),
serde_json::Number::from_f64(aggregation.maxs[feat_index] as f64),
serde_json::Number::from_f64(avg),
) {
props.insert(min_keys[feat_index].clone(), Value::Number(min_num));
props.insert(max_keys[feat_index].clone(), Value::Number(max_num));
props.insert(avg_keys[feat_index].clone(), Value::Number(avg_num));
}
}
}

14
server-rs/src/state.rs
View file

@ -23,20 +23,26 @@ pub struct AppState {
pub min_keys: Vec<String>,
/// Precomputed JSON key names: "max_{feature_name}" for each feature
pub max_keys: Vec<String>,
/// Precomputed JSON key names: "avg_{feature_name}" for each feature
pub avg_keys: Vec<String>,
/// Precomputed POI category groups (sorted)
pub poi_category_groups: Vec<POICategoryGroup>,
/// Precomputed features response for /api/features endpoint
pub features_response: FeaturesResponse,
/// URL of the OG screenshot sidecar service (e.g. http://og-screenshot:8002)
pub og_sidecar_url: Option<String>,
/// URL of the screenshot service (e.g. http://screenshot:8002)
pub screenshot_url: String,
/// Public-facing URL for absolute og:image URLs (e.g. https://narrowit.schmelczer.dev)
pub public_url: String,
/// Contents of index.html read at startup, used for crawler OG injection
pub index_html: Option<String>,
/// Shared HTTP client for proxying to the screenshot sidecar and PocketBase
/// Shared HTTP client for proxying to the screenshot service and PocketBase
pub http_client: reqwest::Client,
/// PocketBase server URL for authentication (e.g. http://localhost:8090)
pub pocketbase_url: Option<String>,
pub pocketbase_url: String,
/// Ollama server URL for AI area summaries (e.g. http://ollama:11434)
pub ollama_url: String,
/// Ollama model name for area summaries (e.g. gemma3:12b)
pub ollama_model: String,
/// Token validation cache (60s TTL)
pub token_cache: Arc<TokenCache>,
}