Add pocketbase and other changes
This commit is contained in:
parent
a9717d570d
commit
229150b641
14 changed files with 1178 additions and 91 deletions
60
server-rs/Cargo.lock
generated
60
server-rs/Cargo.lock
generated
|
|
@ -126,6 +126,15 @@ dependencies = [
|
|||
"object",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1"
|
||||
dependencies = [
|
||||
"derive_arbitrary",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "argminmax"
|
||||
version = "0.6.3"
|
||||
|
|
@ -642,6 +651,17 @@ version = "0.1.3"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "85d3cef41d236720ed453e102153a53e4cc3d2fde848c0078a50cf249e8e3e5b"
|
||||
|
||||
[[package]]
|
||||
name = "derive_arbitrary"
|
||||
version = "1.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a"
|
||||
dependencies = [
|
||||
"proc-macro2",
|
||||
"quote",
|
||||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "displaydoc"
|
||||
version = "0.2.5"
|
||||
|
|
@ -2351,11 +2371,13 @@ dependencies = [
|
|||
"lasso",
|
||||
"metrics",
|
||||
"metrics-exporter-prometheus",
|
||||
"parking_lot",
|
||||
"pmtiles",
|
||||
"polars",
|
||||
"rayon",
|
||||
"regex",
|
||||
"reqwest",
|
||||
"rust_xlsxwriter",
|
||||
"rustc-hash",
|
||||
"serde",
|
||||
"serde_json",
|
||||
|
|
@ -2684,6 +2706,15 @@ dependencies = [
|
|||
"windows-sys 0.52.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rust_xlsxwriter"
|
||||
version = "0.79.4"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "c743cb9f2a4524676020e26ee5f298445a82d882b09956811b1e78ca7e42b440"
|
||||
dependencies = [
|
||||
"zip",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "rustc-hash"
|
||||
version = "2.1.1"
|
||||
|
|
@ -4086,6 +4117,23 @@ dependencies = [
|
|||
"syn",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zip"
|
||||
version = "2.4.2"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "fabe6324e908f85a1c52063ce7aa26b68dcb7eb6dbc83a2d148403c9bc3eba50"
|
||||
dependencies = [
|
||||
"arbitrary",
|
||||
"crc32fast",
|
||||
"crossbeam-utils",
|
||||
"displaydoc",
|
||||
"flate2",
|
||||
"indexmap",
|
||||
"memchr",
|
||||
"thiserror 2.0.18",
|
||||
"zopfli",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zlib-rs"
|
||||
version = "0.6.0"
|
||||
|
|
@ -4098,6 +4146,18 @@ version = "1.0.19"
|
|||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "3ff05f8caa9038894637571ae6b9e29466c1f4f829d26c9b28f869a29cbe3445"
|
||||
|
||||
[[package]]
|
||||
name = "zopfli"
|
||||
version = "0.8.3"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "f05cd8797d63865425ff89b5c4a48804f35ba0ce8d125800027ad6017d2b5249"
|
||||
dependencies = [
|
||||
"bumpalo",
|
||||
"crc32fast",
|
||||
"log",
|
||||
"simd-adler32",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "zstd"
|
||||
version = "0.13.3"
|
||||
|
|
|
|||
|
|
@ -13,6 +13,7 @@ polars = { version = "0.46", features = ["parquet", "lazy", "dtype-struct", "dty
|
|||
h3o = "0.7"
|
||||
serde = { version = "1", features = ["derive"] }
|
||||
serde_json = "1"
|
||||
parking_lot = "0.12"
|
||||
rayon = "1"
|
||||
lasso = "0.7"
|
||||
rustc-hash = "2"
|
||||
|
|
@ -20,9 +21,10 @@ tracing = "0.1"
|
|||
tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
|
||||
metrics = "0.24"
|
||||
metrics-exporter-prometheus = "0.16"
|
||||
reqwest = { version = "0.12", features = ["rustls-tls"] }
|
||||
reqwest = { version = "0.12", features = ["rustls-tls", "json"] }
|
||||
regex = "1"
|
||||
urlencoding = "2"
|
||||
rust_xlsxwriter = "0.79"
|
||||
pmtiles = { version = "0.12", features = ["mmap-async-tokio"] }
|
||||
|
||||
[lints.clippy]
|
||||
|
|
|
|||
145
server-rs/src/auth.rs
Normal file
145
server-rs/src/auth.rs
Normal file
|
|
@ -0,0 +1,145 @@
|
|||
use std::sync::Arc;
|
||||
use std::time::Instant;
|
||||
|
||||
use axum::extract::Request;
|
||||
use axum::middleware::Next;
|
||||
use axum::response::Response;
|
||||
use parking_lot::RwLock;
|
||||
use reqwest::Client;
|
||||
use rustc_hash::FxHashMap;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use tracing::warn;
|
||||
|
||||
const TOKEN_TTL_SECS: u64 = 60;
|
||||
const MAX_CACHE_ENTRIES: usize = 1000;
|
||||
|
||||
#[derive(Debug, Clone, Serialize, Deserialize)]
|
||||
pub struct PocketBaseUser {
|
||||
pub id: String,
|
||||
pub email: String,
|
||||
#[serde(default)]
|
||||
pub name: String,
|
||||
#[serde(default)]
|
||||
pub avatar: String,
|
||||
#[serde(default)]
|
||||
pub verified: bool,
|
||||
}
|
||||
|
||||
#[derive(Clone)]
|
||||
pub struct OptionalUser(pub Option<PocketBaseUser>);
|
||||
|
||||
pub struct TokenCache {
|
||||
entries: RwLock<FxHashMap<String, (PocketBaseUser, Instant)>>,
|
||||
}
|
||||
|
||||
impl TokenCache {
|
||||
pub fn new() -> Self {
|
||||
Self {
|
||||
entries: RwLock::new(FxHashMap::default()),
|
||||
}
|
||||
}
|
||||
|
||||
fn get(&self, token: &str) -> Option<PocketBaseUser> {
|
||||
let map = self.entries.read();
|
||||
if let Some((user, created)) = map.get(token) {
|
||||
if created.elapsed().as_secs() < TOKEN_TTL_SECS {
|
||||
return Some(user.clone());
|
||||
}
|
||||
}
|
||||
None
|
||||
}
|
||||
|
||||
fn insert(&self, token: String, user: PocketBaseUser) {
|
||||
let mut map = self.entries.write();
|
||||
if map.len() >= MAX_CACHE_ENTRIES {
|
||||
// Evict expired entries first
|
||||
let now = Instant::now();
|
||||
map.retain(|_, (_, created)| now.duration_since(*created).as_secs() < TOKEN_TTL_SECS);
|
||||
// If still too many, clear all
|
||||
if map.len() >= MAX_CACHE_ENTRIES {
|
||||
map.clear();
|
||||
}
|
||||
}
|
||||
map.insert(token, (user, Instant::now()));
|
||||
}
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
struct AuthRefreshResponse {
|
||||
record: PocketBaseUser,
|
||||
}
|
||||
|
||||
async fn validate_token(
|
||||
client: &Client,
|
||||
pocketbase_url: &str,
|
||||
token: &str,
|
||||
) -> Option<PocketBaseUser> {
|
||||
let url = format!(
|
||||
"{}/api/collections/users/auth-refresh",
|
||||
pocketbase_url.trim_end_matches('/')
|
||||
);
|
||||
let res = client
|
||||
.post(&url)
|
||||
.header("Authorization", format!("Bearer {token}"))
|
||||
.send()
|
||||
.await
|
||||
.ok()?;
|
||||
|
||||
if !res.status().is_success() {
|
||||
return None;
|
||||
}
|
||||
|
||||
let body: AuthRefreshResponse = res.json().await.ok()?;
|
||||
Some(body.record)
|
||||
}
|
||||
|
||||
pub async fn auth_middleware(req: Request, next: Next) -> Response {
|
||||
let pocketbase_url = req
|
||||
.extensions()
|
||||
.get::<Arc<crate::state::AppState>>()
|
||||
.and_then(|st| st.pocketbase_url.as_deref())
|
||||
.map(String::from);
|
||||
|
||||
let token_cache = req
|
||||
.extensions()
|
||||
.get::<Arc<crate::state::AppState>>()
|
||||
.map(|st| st.token_cache.clone());
|
||||
|
||||
let http_client = req
|
||||
.extensions()
|
||||
.get::<Arc<crate::state::AppState>>()
|
||||
.map(|st| st.http_client.clone());
|
||||
|
||||
let token = req
|
||||
.headers()
|
||||
.get("authorization")
|
||||
.and_then(|hv| hv.to_str().ok())
|
||||
.and_then(|hv| hv.strip_prefix("Bearer "))
|
||||
.map(String::from);
|
||||
|
||||
let user = match (&pocketbase_url, &token, &token_cache, &http_client) {
|
||||
(Some(pb_url), Some(tk), Some(cache), Some(client)) => {
|
||||
if let Some(cached) = cache.get(tk) {
|
||||
Some(cached)
|
||||
} else {
|
||||
match validate_token(client, pb_url, tk).await {
|
||||
Some(user) => {
|
||||
cache.insert(tk.clone(), user.clone());
|
||||
Some(user)
|
||||
}
|
||||
None => {
|
||||
warn!("Invalid auth token");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
_ => None,
|
||||
};
|
||||
|
||||
let (mut parts, body) = req.into_parts();
|
||||
parts.extensions.insert(OptionalUser(user));
|
||||
let req = Request::from_parts(parts, body);
|
||||
|
||||
next.run(req).await
|
||||
}
|
||||
|
|
@ -22,6 +22,10 @@ pub struct FeatureConfig {
|
|||
pub source: &'static str,
|
||||
}
|
||||
|
||||
/// Features whose histogram bins should be exactly 1 unit wide (one per integer).
|
||||
/// p1/p99 are snapped to integer boundaries before binning.
|
||||
pub const INTEGER_BIN_FEATURES: &[&str] = &["Number of bedrooms & living rooms"];
|
||||
|
||||
pub struct FeatureGroup {
|
||||
pub name: &'static str,
|
||||
pub features: &'static [FeatureConfig],
|
||||
|
|
@ -107,6 +111,17 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
|||
detail: "Total number of habitable rooms (bedrooms plus living rooms) as recorded in the Energy Performance Certificate. Kitchens and bathrooms are typically excluded unless they are large enough to count as habitable rooms.",
|
||||
source: "epc",
|
||||
},
|
||||
FeatureConfig {
|
||||
name: "Transaction year",
|
||||
bounds: Bounds::Fixed {
|
||||
min: 1995.0,
|
||||
max: 2026.0,
|
||||
},
|
||||
step: 1.0,
|
||||
description: "Year of the most recent sale from the Land Registry",
|
||||
detail: "The year (and fractional month) of the most recent recorded sale for this property from HM Land Registry Price Paid data. Used to show price history trends in the area chart.",
|
||||
source: "price-paid",
|
||||
},
|
||||
FeatureConfig {
|
||||
name: "Construction age",
|
||||
bounds: Bounds::Fixed {
|
||||
|
|
@ -593,23 +608,77 @@ pub static FEATURE_GROUPS: &[FeatureGroup] = &[
|
|||
|
||||
pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[
|
||||
EnumFeatureGroup {
|
||||
name: "Property",
|
||||
features: &[
|
||||
EnumFeatureConfig {
|
||||
name: "Leashold/Freehold",
|
||||
order: Some(&["Freehold", "Leasehold"]),
|
||||
description: "Whether the property is leasehold or freehold",
|
||||
detail: "From HM Land Registry Price Paid data. Freehold means you own the building and the land it stands on. Leasehold means you own the building but not the land — you have a lease from the freeholder for a set number of years.",
|
||||
source: "price-paid",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Property type",
|
||||
order: Some(&["Detached", "Semi-Detached", "Terraced", "Flat"]),
|
||||
description: "Type of property: detached, semi-detached, terraced, or flat",
|
||||
detail: "From HM Land Registry Price Paid data. The broad property type classification: Detached, Semi-Detached, Terraced, or Flat/Maisonette.",
|
||||
source: "price-paid",
|
||||
},
|
||||
],
|
||||
name: "Property",
|
||||
features: &[
|
||||
EnumFeatureConfig {
|
||||
name: "Leashold/Freehold",
|
||||
order: Some(&["Freehold", "Leasehold"]),
|
||||
description: "Whether the property is leasehold or freehold",
|
||||
detail: "From HM Land Registry Price Paid data. Freehold means you own the building and the land it stands on. Leasehold means you own the building but not the land — you have a lease from the freeholder for a set number of years.",
|
||||
source: "price-paid",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Property type",
|
||||
order: Some(&["Detached", "Semi-Detached", "Terraced", "Flat"]),
|
||||
description: "Type of property: detached, semi-detached, terraced, or flat",
|
||||
detail: "From HM Land Registry Price Paid data. The broad property type classification: Detached, Semi-Detached, Terraced, or Flat/Maisonette.",
|
||||
source: "price-paid",
|
||||
},
|
||||
],
|
||||
},
|
||||
EnumFeatureGroup {
|
||||
name: "Environment",
|
||||
features: &[
|
||||
EnumFeatureConfig {
|
||||
name: "Environmental risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Highest ground stability risk across all six hazard types",
|
||||
detail: "Overall ground stability risk for the area, taken as the maximum across all six GeoSure hazard categories (collapsible deposits, compressible ground, landslides, running sand, shrink-swell, and soluble rocks). From Ordnance Survey GeoSure data on a 5km hex grid.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Collapsible deposits risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of ground collapse from natural underground cavities",
|
||||
detail: "From OS GeoSure. Indicates the likelihood of ground collapse due to natural cavities formed by dissolution of soluble rocks or the collapse of old mines and natural pipes. Rated on a 5km hex grid across Great Britain.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Compressible ground risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of ground compression causing subsidence",
|
||||
detail: "From OS GeoSure. Indicates the potential for ground to compress under loading, which can cause gradual settlement or subsidence of buildings and infrastructure. Typically associated with soft clay, silt, or peat deposits.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Landslide risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of landslide or slope instability",
|
||||
detail: "From OS GeoSure. Indicates the susceptibility of the ground to landslides and slope instability. Based on slope angle, geology, and historical landslide records.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Running sand risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of sand becoming fluid when saturated",
|
||||
detail: "From OS GeoSure. Indicates the potential for fine-grained sand to behave like a fluid when saturated with water, which can affect excavations and foundations.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Shrink-swell risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of clay shrinking and swelling with moisture changes",
|
||||
detail: "From OS GeoSure. Indicates the potential for clay-rich soils to shrink when dry and swell when wet, causing ground movement that can damage buildings and infrastructure. One of the most common causes of subsidence in the UK.",
|
||||
source: "geosure",
|
||||
},
|
||||
EnumFeatureConfig {
|
||||
name: "Soluble rocks risk",
|
||||
order: Some(&["Low", "Moderate", "Significant"]),
|
||||
description: "Risk of sinkholes from dissolution of soluble rocks",
|
||||
detail: "From OS GeoSure. Indicates the potential for soluble rocks (limestone, chalk, gypsum) to dissolve, creating underground voids that can lead to sinkholes and ground subsidence.",
|
||||
source: "geosure",
|
||||
},
|
||||
],
|
||||
},
|
||||
];
|
||||
|
||||
|
|
@ -638,6 +707,11 @@ pub fn order_for(name: &str) -> Option<&'static [&'static str]> {
|
|||
.and_then(|feature| feature.order)
|
||||
}
|
||||
|
||||
/// Whether this feature should use integer-width histogram bins.
|
||||
pub fn has_integer_bins(name: &str) -> bool {
|
||||
INTEGER_BIN_FEATURES.contains(&name)
|
||||
}
|
||||
|
||||
/// Look up the Bounds config for a numeric feature by name.
|
||||
pub fn bounds_for(name: &str) -> Option<&'static Bounds> {
|
||||
FEATURE_GROUPS
|
||||
|
|
@ -651,12 +725,14 @@ pub fn bounds_for(name: &str) -> Option<&'static Bounds> {
|
|||
/// The server will panic at startup if the data contains groups not in this list or vice versa.
|
||||
pub const POI_GROUP_ORDER: &[&str] = &[
|
||||
"Public Transport",
|
||||
"Amenity",
|
||||
"Building",
|
||||
"Craft",
|
||||
"Healthcare",
|
||||
"Leisure",
|
||||
"Office",
|
||||
"Shop",
|
||||
"Tourism",
|
||||
"Education",
|
||||
"Health",
|
||||
"Emergency Services",
|
||||
"Other",
|
||||
"Groceries",
|
||||
"Local Businesses",
|
||||
"Culture",
|
||||
"Services",
|
||||
"Shops",
|
||||
];
|
||||
|
|
|
|||
|
|
@ -1,3 +1,4 @@
|
|||
mod auth;
|
||||
mod consts;
|
||||
mod data;
|
||||
mod features;
|
||||
|
|
@ -13,7 +14,7 @@ use std::sync::Arc;
|
|||
|
||||
use anyhow::{bail, Context};
|
||||
use axum::middleware;
|
||||
use axum::routing::get;
|
||||
use axum::routing::{any, get};
|
||||
use axum::Router;
|
||||
use clap::Parser;
|
||||
use tower_http::compression::CompressionLayer;
|
||||
|
|
@ -59,6 +60,10 @@ struct Cli {
|
|||
default_value = "https://narrowit.schmelczer.dev"
|
||||
)]
|
||||
public_url: String,
|
||||
|
||||
/// PocketBase server URL for authentication (e.g. http://localhost:8090)
|
||||
#[arg(long, env = "POCKETBASE_URL")]
|
||||
pocketbase_url: Option<String>,
|
||||
}
|
||||
|
||||
#[tokio::main]
|
||||
|
|
@ -117,8 +122,7 @@ async fn main() -> anyhow::Result<()> {
|
|||
info!(pois = poi_data.lat.len(), "POI data loaded");
|
||||
|
||||
info!("Building POI spatial grid index");
|
||||
let poi_grid =
|
||||
utils::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);
|
||||
let poi_grid = utils::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);
|
||||
|
||||
// Load postcode boundaries
|
||||
let postcodes_path = &cli.postcodes;
|
||||
|
|
@ -219,6 +223,12 @@ async fn main() -> anyhow::Result<()> {
|
|||
postcode_data.postcodes.len(),
|
||||
);
|
||||
|
||||
if let Some(ref pb_url) = cli.pocketbase_url {
|
||||
info!("PocketBase configured: {}", pb_url);
|
||||
}
|
||||
|
||||
let token_cache = Arc::new(auth::TokenCache::new());
|
||||
|
||||
let state = Arc::new(AppState {
|
||||
data: property_data,
|
||||
grid,
|
||||
|
|
@ -235,6 +245,8 @@ async fn main() -> anyhow::Result<()> {
|
|||
public_url: cli.public_url,
|
||||
index_html,
|
||||
http_client,
|
||||
pocketbase_url: cli.pocketbase_url,
|
||||
token_cache,
|
||||
});
|
||||
|
||||
let cors = CorsLayer::new()
|
||||
|
|
@ -251,7 +263,9 @@ async fn main() -> anyhow::Result<()> {
|
|||
let state_hexagon_properties = state.clone();
|
||||
let state_hexagon_stats = state.clone();
|
||||
let state_og_image = state.clone();
|
||||
let state_export = state.clone();
|
||||
let state_crawler = state.clone();
|
||||
let state_pb = state.clone();
|
||||
|
||||
let api = Router::new()
|
||||
.route(
|
||||
|
|
@ -291,7 +305,12 @@ async fn main() -> anyhow::Result<()> {
|
|||
.route(
|
||||
"/api/og-image",
|
||||
get(move |query| routes::get_og_image(state_og_image.clone(), query)),
|
||||
);
|
||||
)
|
||||
.route(
|
||||
"/api/export",
|
||||
get(move |query| routes::get_export(state_export.clone(), query)),
|
||||
)
|
||||
.route("/api/me", get(routes::get_me));
|
||||
|
||||
// Add tile routes
|
||||
let reader_tile = tile_reader.clone();
|
||||
|
|
@ -307,7 +326,14 @@ async fn main() -> anyhow::Result<()> {
|
|||
routes::get_style(axum::extract::State(reader_style.clone()), headers, query)
|
||||
}),
|
||||
)
|
||||
.route("/metrics", get(move || metrics::metrics_handler(metrics_handle.clone())));
|
||||
.route(
|
||||
"/metrics",
|
||||
get(move || metrics::metrics_handler(metrics_handle.clone())),
|
||||
)
|
||||
.route(
|
||||
"/pb/{*rest}",
|
||||
any(move |req| routes::proxy_to_pocketbase(state_pb.clone(), req)),
|
||||
);
|
||||
|
||||
let app = if frontend_dist.exists() {
|
||||
api.fallback_service(ServeDir::new(&frontend_dist))
|
||||
|
|
@ -317,11 +343,12 @@ async fn main() -> anyhow::Result<()> {
|
|||
|
||||
let app = app
|
||||
.layer(middleware::from_fn(metrics::track_metrics))
|
||||
.layer(middleware::from_fn(auth::auth_middleware))
|
||||
.layer(middleware::from_fn(
|
||||
move |req: axum::extract::Request, next: middleware::Next| {
|
||||
let st = state_crawler.clone();
|
||||
async move {
|
||||
// Inject state into request extensions for the middleware
|
||||
// Inject state into request extensions for auth + OG middleware
|
||||
let (mut parts, body) = req.into_parts();
|
||||
parts.extensions.insert(st);
|
||||
let req = axum::extract::Request::from_parts(parts, body);
|
||||
|
|
|
|||
|
|
@ -105,9 +105,13 @@ mod tests {
|
|||
use super::*;
|
||||
|
||||
fn feature_name_to_index() -> FxHashMap<String, usize> {
|
||||
[("price".into(), 0), ("area".into(), 1), ("rating".into(), 2)]
|
||||
.into_iter()
|
||||
.collect()
|
||||
[
|
||||
("price".into(), 0),
|
||||
("area".into(), 1),
|
||||
("rating".into(), 2),
|
||||
]
|
||||
.into_iter()
|
||||
.collect()
|
||||
}
|
||||
|
||||
fn enum_values() -> FxHashMap<usize, Vec<String>> {
|
||||
|
|
@ -147,8 +151,11 @@ mod tests {
|
|||
|
||||
#[test]
|
||||
fn parse_filters_numeric() {
|
||||
let (numeric, enums) =
|
||||
parse_filters(Some("price:100:500"), &feature_name_to_index(), &enum_values());
|
||||
let (numeric, enums) = parse_filters(
|
||||
Some("price:100:500"),
|
||||
&feature_name_to_index(),
|
||||
&enum_values(),
|
||||
);
|
||||
assert_eq!(numeric.len(), 1);
|
||||
assert_eq!(numeric[0].feat_idx, 0);
|
||||
assert_eq!(numeric[0].min, 100.0);
|
||||
|
|
@ -176,7 +183,11 @@ mod tests {
|
|||
let (n, e) = parse_filters(Some(""), &feature_name_to_index(), &enum_values());
|
||||
assert!(n.is_empty() && e.is_empty());
|
||||
|
||||
let (n, e) = parse_filters(Some("unknown:1:2"), &feature_name_to_index(), &enum_values());
|
||||
let (n, e) = parse_filters(
|
||||
Some("unknown:1:2"),
|
||||
&feature_name_to_index(),
|
||||
&enum_values(),
|
||||
);
|
||||
assert!(n.is_empty() && e.is_empty());
|
||||
}
|
||||
|
||||
|
|
@ -350,9 +361,27 @@ mod tests {
|
|||
max: 250.0,
|
||||
}];
|
||||
|
||||
assert!(!row_passes_filters(0, &filters, &[], &feature_data, num_features));
|
||||
assert!(row_passes_filters(1, &filters, &[], &feature_data, num_features));
|
||||
assert!(!row_passes_filters(2, &filters, &[], &feature_data, num_features));
|
||||
assert!(!row_passes_filters(
|
||||
0,
|
||||
&filters,
|
||||
&[],
|
||||
&feature_data,
|
||||
num_features
|
||||
));
|
||||
assert!(row_passes_filters(
|
||||
1,
|
||||
&filters,
|
||||
&[],
|
||||
&feature_data,
|
||||
num_features
|
||||
));
|
||||
assert!(!row_passes_filters(
|
||||
2,
|
||||
&filters,
|
||||
&[],
|
||||
&feature_data,
|
||||
num_features
|
||||
));
|
||||
}
|
||||
|
||||
#[test]
|
||||
|
|
|
|||
|
|
@ -1,16 +1,22 @@
|
|||
mod export;
|
||||
mod features;
|
||||
mod hexagon_stats;
|
||||
pub(crate) mod hexagons;
|
||||
mod me;
|
||||
mod og_image;
|
||||
mod pb_proxy;
|
||||
mod pois;
|
||||
mod postcodes;
|
||||
pub(crate) mod properties;
|
||||
mod tiles;
|
||||
|
||||
pub use features::{build_features_response, get_features, FeaturesResponse};
|
||||
pub use export::get_export;
|
||||
pub use features::{build_features_response, get_features, FeatureInfo, FeaturesResponse};
|
||||
pub use hexagon_stats::get_hexagon_stats;
|
||||
pub use hexagons::get_hexagons;
|
||||
pub use me::get_me;
|
||||
pub use og_image::get_og_image;
|
||||
pub use pb_proxy::proxy_to_pocketbase;
|
||||
pub use pois::{get_poi_categories, get_pois};
|
||||
pub use postcodes::{get_postcode_lookup, get_postcodes};
|
||||
pub use properties::get_hexagon_properties;
|
||||
|
|
|
|||
507
server-rs/src/routes/export.rs
Normal file
507
server-rs/src/routes/export.rs
Normal file
|
|
@ -0,0 +1,507 @@
|
|||
use std::collections::hash_map::DefaultHasher;
|
||||
use std::hash::{Hash, Hasher};
|
||||
use std::sync::Arc;
|
||||
|
||||
use axum::extract::Query;
|
||||
use axum::http::{header, StatusCode};
|
||||
use axum::response::IntoResponse;
|
||||
use rust_xlsxwriter::{Format, FormatAlign, FormatBorder, Image, Url, Workbook};
|
||||
use rustc_hash::{FxHashMap, FxHashSet};
|
||||
use serde::Deserialize;
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::parsing::{parse_bounds, parse_filters, row_passes_filters};
|
||||
use crate::routes::FeatureInfo;
|
||||
use crate::state::AppState;
|
||||
|
||||
const MAX_EXPORT_POSTCODES: usize = 250;
|
||||
/// Height (in pixels) reserved for the OG image row
|
||||
const IMAGE_ROW_HEIGHT: f64 = 225.0;
|
||||
|
||||
#[derive(Deserialize)]
|
||||
pub struct ExportParams {
|
||||
bounds: Option<String>,
|
||||
filters: Option<String>,
|
||||
fields: Option<String>,
|
||||
}
|
||||
|
||||
/// Per-postcode accumulator for export aggregation (mean for numeric, mode for enum).
|
||||
struct PostcodeExportAgg {
|
||||
count: u32,
|
||||
sums: Vec<f64>,
|
||||
finite_counts: Vec<u32>,
|
||||
/// feat_idx -> (value_bits -> count) for enum mode calculation
|
||||
enum_freqs: FxHashMap<usize, FxHashMap<u32, u32>>,
|
||||
}
|
||||
|
||||
impl PostcodeExportAgg {
|
||||
fn new(num_features: usize) -> Self {
|
||||
Self {
|
||||
count: 0,
|
||||
sums: vec![0.0; num_features],
|
||||
finite_counts: vec![0; num_features],
|
||||
enum_freqs: FxHashMap::default(),
|
||||
}
|
||||
}
|
||||
|
||||
#[inline]
|
||||
fn add_row(
|
||||
&mut self,
|
||||
feature_data: &[f32],
|
||||
row: usize,
|
||||
num_features: usize,
|
||||
enum_indices: &FxHashMap<usize, ()>,
|
||||
) {
|
||||
self.count += 1;
|
||||
let base = row * num_features;
|
||||
let row_slice = &feature_data[base..base + num_features];
|
||||
for (feat_idx, &value) in row_slice.iter().enumerate() {
|
||||
if !value.is_finite() {
|
||||
continue;
|
||||
}
|
||||
if enum_indices.contains_key(&feat_idx) {
|
||||
*self
|
||||
.enum_freqs
|
||||
.entry(feat_idx)
|
||||
.or_default()
|
||||
.entry(value.to_bits())
|
||||
.or_insert(0) += 1;
|
||||
} else {
|
||||
self.sums[feat_idx] += value as f64;
|
||||
self.finite_counts[feat_idx] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/// Extract feature names referenced in the filters param (preserving order).
|
||||
fn extract_filter_feature_names(filters_str: Option<&str>) -> Vec<String> {
|
||||
let input = match filters_str.filter(|text| !text.is_empty()) {
|
||||
Some(text) => text,
|
||||
None => return Vec::new(),
|
||||
};
|
||||
let mut names = Vec::new();
|
||||
for entry in input.split(',') {
|
||||
let parts: Vec<&str> = entry.splitn(2, ':').collect();
|
||||
if parts.len() == 2 {
|
||||
let name = parts[0].trim().to_string();
|
||||
if !names.contains(&name) {
|
||||
names.push(name);
|
||||
}
|
||||
}
|
||||
}
|
||||
names
|
||||
}
|
||||
|
||||
/// Fetch the OG screenshot image from the sidecar service.
|
||||
async fn fetch_og_image(
|
||||
state: &AppState,
|
||||
view_param: &str,
|
||||
filters_str: Option<&str>,
|
||||
) -> Option<Vec<u8>> {
|
||||
let sidecar_url = state.og_sidecar_url.as_deref()?;
|
||||
|
||||
let mut params = vec![format!("v={}", urlencoding::encode(view_param))];
|
||||
if let Some(fs) = filters_str {
|
||||
if !fs.is_empty() {
|
||||
params.push(format!("f={}", urlencoding::encode(fs)));
|
||||
}
|
||||
}
|
||||
let url = format!("{}/screenshot?{}", sidecar_url, params.join("&"));
|
||||
|
||||
match state.http_client.get(&url).send().await {
|
||||
Ok(resp) if resp.status().is_success() => match resp.bytes().await {
|
||||
Ok(bytes) => {
|
||||
info!(bytes = bytes.len(), "Fetched OG image for export");
|
||||
Some(bytes.to_vec())
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Failed to read OG sidecar response for export: {err}");
|
||||
None
|
||||
}
|
||||
},
|
||||
Ok(resp) => {
|
||||
warn!(status = %resp.status(), "OG sidecar returned error for export");
|
||||
None
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Failed to reach OG sidecar for export: {err}");
|
||||
None
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
pub async fn get_export(
|
||||
state: Arc<AppState>,
|
||||
Query(params): Query<ExportParams>,
|
||||
) -> Result<impl IntoResponse, (StatusCode, String)> {
|
||||
let bounds_str = params.bounds.ok_or((
|
||||
StatusCode::BAD_REQUEST,
|
||||
"bounds parameter is required".into(),
|
||||
))?;
|
||||
|
||||
let (south, west, north, east) = parse_bounds(&bounds_str)?;
|
||||
|
||||
let filters_str = params.filters.clone();
|
||||
let fields_str = params.fields.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
params.filters.as_deref(),
|
||||
&state.feature_name_to_index,
|
||||
&state.data.enum_values,
|
||||
);
|
||||
|
||||
let public_url = state.public_url.clone();
|
||||
|
||||
// Compute view param for OG image and dashboard URL
|
||||
let center_lat = (south + north) / 2.0;
|
||||
let center_lon = (west + east) / 2.0;
|
||||
let lat_span = north - south;
|
||||
let zoom = if lat_span > 0.0 {
|
||||
(360.0 / lat_span).log2().clamp(1.0, 18.0)
|
||||
} else {
|
||||
12.0
|
||||
};
|
||||
let view_param = format!("{:.4},{:.4},{:.1}", center_lat, center_lon, zoom);
|
||||
|
||||
// Fetch OG image from sidecar (async, before spawn_blocking)
|
||||
let og_image_bytes = fetch_og_image(&state, &view_param, filters_str.as_deref()).await;
|
||||
|
||||
// Build feature name → description map from the precomputed features response
|
||||
let feature_descriptions: FxHashMap<String, String> = state
|
||||
.features_response
|
||||
.groups
|
||||
.iter()
|
||||
.flat_map(|group| &group.features)
|
||||
.map(|feat| match feat {
|
||||
FeatureInfo::Numeric {
|
||||
name, description, ..
|
||||
} => (name.clone(), description.to_string()),
|
||||
FeatureInfo::Enum {
|
||||
name, description, ..
|
||||
} => (name.clone(), description.to_string()),
|
||||
})
|
||||
.collect();
|
||||
|
||||
let bytes = tokio::task::spawn_blocking(move || -> Result<Vec<u8>, String> {
|
||||
let t0 = std::time::Instant::now();
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
let feature_names = &state.data.feature_names;
|
||||
let enum_values = &state.data.enum_values;
|
||||
let postcode_data = &state.postcode_data;
|
||||
|
||||
// Build set of enum feature indices for quick lookup
|
||||
let enum_indices: FxHashMap<usize, ()> = enum_values.keys().map(|&idx| (idx, ())).collect();
|
||||
|
||||
// Group rows by postcode
|
||||
let mut postcode_rows: FxHashMap<usize, Vec<usize>> = FxHashMap::default();
|
||||
state
|
||||
.grid
|
||||
.for_each_in_bounds(south, west, north, east, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if !row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
&parsed_enum_filters,
|
||||
feature_data,
|
||||
num_features,
|
||||
) {
|
||||
return;
|
||||
}
|
||||
let postcode = state.data.postcode(row);
|
||||
if let Some(&pc_idx) = postcode_data.postcode_to_idx.get(postcode) {
|
||||
postcode_rows.entry(pc_idx).or_default().push(row);
|
||||
}
|
||||
});
|
||||
|
||||
// Aggregate per postcode
|
||||
let mut postcode_aggs: Vec<(usize, PostcodeExportAgg)> =
|
||||
Vec::with_capacity(postcode_rows.len());
|
||||
for (pc_idx, rows) in postcode_rows {
|
||||
let mut agg = PostcodeExportAgg::new(num_features);
|
||||
for &row in &rows {
|
||||
agg.add_row(feature_data, row, num_features, &enum_indices);
|
||||
}
|
||||
if agg.count > 0 {
|
||||
postcode_aggs.push((pc_idx, agg));
|
||||
}
|
||||
}
|
||||
|
||||
// Sort by property count descending
|
||||
postcode_aggs.sort_unstable_by(|lhs, rhs| rhs.1.count.cmp(&lhs.1.count));
|
||||
|
||||
// Sample if too many postcodes
|
||||
let was_sampled = postcode_aggs.len() > MAX_EXPORT_POSTCODES;
|
||||
if was_sampled {
|
||||
let mut hasher = DefaultHasher::new();
|
||||
bounds_str.hash(&mut hasher);
|
||||
let seed = hasher.finish();
|
||||
|
||||
let len = postcode_aggs.len();
|
||||
for pick in 0..MAX_EXPORT_POSTCODES {
|
||||
let swap_idx = pick
|
||||
+ ((seed.wrapping_mul(pick as u64 + 1).wrapping_add(pick as u64)) as usize
|
||||
% (len - pick));
|
||||
postcode_aggs.swap(pick, swap_idx);
|
||||
}
|
||||
postcode_aggs.truncate(MAX_EXPORT_POSTCODES);
|
||||
postcode_aggs.sort_unstable_by(|lhs, rhs| rhs.1.count.cmp(&lhs.1.count));
|
||||
}
|
||||
|
||||
// Determine column order: filter features first, then remaining
|
||||
let filter_feature_names = extract_filter_feature_names(filters_str.as_deref());
|
||||
|
||||
let field_indices: Option<Vec<usize>> = fields_str.as_ref().map(|fs| {
|
||||
if fs.is_empty() {
|
||||
return Vec::new();
|
||||
}
|
||||
fs.split(',')
|
||||
.filter_map(|name| {
|
||||
let name = name.trim();
|
||||
if name.is_empty() {
|
||||
return None;
|
||||
}
|
||||
state.feature_name_to_index.get(name).copied()
|
||||
})
|
||||
.collect()
|
||||
});
|
||||
|
||||
let all_feature_indices: Vec<usize> = if let Some(ref indices) = field_indices {
|
||||
indices.clone()
|
||||
} else {
|
||||
let mut ordered = Vec::with_capacity(num_features);
|
||||
let mut used = FxHashSet::default();
|
||||
|
||||
for name in &filter_feature_names {
|
||||
if let Some(&idx) = state.feature_name_to_index.get(name.as_str()) {
|
||||
if used.insert(idx) {
|
||||
ordered.push(idx);
|
||||
}
|
||||
}
|
||||
}
|
||||
for idx in 0..num_features {
|
||||
if used.insert(idx) {
|
||||
ordered.push(idx);
|
||||
}
|
||||
}
|
||||
ordered
|
||||
};
|
||||
|
||||
// Build Excel workbook
|
||||
let mut workbook = Workbook::new();
|
||||
let sheet = workbook.add_worksheet();
|
||||
|
||||
// Formats
|
||||
let header_fmt = Format::new()
|
||||
.set_bold()
|
||||
.set_border_bottom(FormatBorder::Thin)
|
||||
.set_align(FormatAlign::Center);
|
||||
|
||||
let desc_fmt = Format::new()
|
||||
.set_italic()
|
||||
.set_font_color("#666666")
|
||||
.set_font_size(9)
|
||||
.set_align(FormatAlign::Center)
|
||||
.set_text_wrap();
|
||||
|
||||
let link_fmt = Format::new()
|
||||
.set_font_color("#0563C1")
|
||||
.set_underline(rust_xlsxwriter::FormatUnderline::Single)
|
||||
.set_font_size(11);
|
||||
|
||||
let note_fmt = Format::new()
|
||||
.set_italic()
|
||||
.set_font_color("#666666")
|
||||
.set_align(FormatAlign::Left);
|
||||
|
||||
// Row 0: "View on Narrowit" link
|
||||
let mut dashboard_url = format!("{}/", public_url);
|
||||
let mut query_parts: Vec<String> = Vec::new();
|
||||
query_parts.push(format!("v={}", view_param));
|
||||
if let Some(ref fs) = filters_str {
|
||||
if !fs.is_empty() {
|
||||
query_parts.push(format!("f={}", urlencoding::encode(fs)));
|
||||
}
|
||||
}
|
||||
if !query_parts.is_empty() {
|
||||
dashboard_url.push('?');
|
||||
dashboard_url.push_str(&query_parts.join("&"));
|
||||
}
|
||||
|
||||
sheet
|
||||
.write_url(0, 0, Url::new(&dashboard_url).set_text("View on Narrowit"))
|
||||
.map_err(|err| format!("Failed to write URL: {err}"))?;
|
||||
sheet
|
||||
.set_row_format(0, &link_fmt)
|
||||
.map_err(|err| format!("Failed to set row format: {err}"))?;
|
||||
|
||||
// Row 1: OG image (if available)
|
||||
let mut current_row = 1u32;
|
||||
if let Some(ref img_bytes) = og_image_bytes {
|
||||
match Image::new_from_buffer(img_bytes) {
|
||||
Ok(mut image) => {
|
||||
// Scale image to fit: ~400px wide, auto height preserving aspect ratio
|
||||
image = image.set_scale_to_size(400, 300, true);
|
||||
sheet
|
||||
.insert_image(current_row, 0, &image)
|
||||
.map_err(|err| format!("Failed to insert OG image: {err}"))?;
|
||||
// Set row height to accommodate the image
|
||||
sheet
|
||||
.set_row_height(current_row, IMAGE_ROW_HEIGHT)
|
||||
.map_err(|err| format!("Failed to set image row height: {err}"))?;
|
||||
current_row += 1;
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("Failed to parse OG image for export: {err}");
|
||||
// Skip image row, don't leave a gap
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Leave a blank row between image and header
|
||||
current_row += 1;
|
||||
|
||||
// Header row
|
||||
let header_row = current_row;
|
||||
sheet
|
||||
.write_string_with_format(header_row, 0, "Postcode", &header_fmt)
|
||||
.map_err(|err| format!("Failed to write header: {err}"))?;
|
||||
sheet
|
||||
.write_string_with_format(header_row, 1, "Properties", &header_fmt)
|
||||
.map_err(|err| format!("Failed to write header: {err}"))?;
|
||||
|
||||
for (col_offset, &feat_idx) in all_feature_indices.iter().enumerate() {
|
||||
let col = (col_offset + 2) as u16;
|
||||
sheet
|
||||
.write_string_with_format(header_row, col, &feature_names[feat_idx], &header_fmt)
|
||||
.map_err(|err| format!("Failed to write header: {err}"))?;
|
||||
}
|
||||
|
||||
// Description row (below header)
|
||||
let desc_row = header_row + 1;
|
||||
// Empty descriptions for Postcode and Properties columns
|
||||
sheet
|
||||
.write_string_with_format(desc_row, 0, "", &desc_fmt)
|
||||
.map_err(|err| format!("Failed to write desc: {err}"))?;
|
||||
sheet
|
||||
.write_string_with_format(desc_row, 1, "Count of properties", &desc_fmt)
|
||||
.map_err(|err| format!("Failed to write desc: {err}"))?;
|
||||
|
||||
for (col_offset, &feat_idx) in all_feature_indices.iter().enumerate() {
|
||||
let col = (col_offset + 2) as u16;
|
||||
let desc = feature_descriptions
|
||||
.get(&feature_names[feat_idx])
|
||||
.map(String::as_str)
|
||||
.unwrap_or("");
|
||||
sheet
|
||||
.write_string_with_format(desc_row, col, desc, &desc_fmt)
|
||||
.map_err(|err| format!("Failed to write desc: {err}"))?;
|
||||
}
|
||||
|
||||
// Write data rows (starting after description row)
|
||||
let data_start_row = desc_row + 1;
|
||||
for (row_offset, (pc_idx, agg)) in postcode_aggs.iter().enumerate() {
|
||||
let row = data_start_row + row_offset as u32;
|
||||
|
||||
sheet
|
||||
.write_string(row, 0, &postcode_data.postcodes[*pc_idx])
|
||||
.map_err(|err| format!("Failed to write postcode: {err}"))?;
|
||||
|
||||
sheet
|
||||
.write_number(row, 1, agg.count as f64)
|
||||
.map_err(|err| format!("Failed to write count: {err}"))?;
|
||||
|
||||
for (col_offset, &feat_idx) in all_feature_indices.iter().enumerate() {
|
||||
let col = (col_offset + 2) as u16;
|
||||
|
||||
if enum_indices.contains_key(&feat_idx) {
|
||||
if let Some(freqs) = agg.enum_freqs.get(&feat_idx) {
|
||||
if let Some((&mode_bits, _)) = freqs.iter().max_by_key(|(_, &count)| count)
|
||||
{
|
||||
let mode_f32 = f32::from_bits(mode_bits);
|
||||
let mode_idx = mode_f32 as usize;
|
||||
if let Some(values) = enum_values.get(&feat_idx) {
|
||||
if mode_idx < values.len() {
|
||||
sheet.write_string(row, col, &values[mode_idx]).map_err(
|
||||
|err| format!("Failed to write enum value: {err}"),
|
||||
)?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
} else {
|
||||
let fc = agg.finite_counts[feat_idx];
|
||||
if fc > 0 {
|
||||
let mean = agg.sums[feat_idx] / fc as f64;
|
||||
sheet
|
||||
.write_number(row, col, mean)
|
||||
.map_err(|err| format!("Failed to write numeric value: {err}"))?;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// If sampled, add a note at the bottom
|
||||
if was_sampled {
|
||||
let note_row = data_start_row + postcode_aggs.len() as u32 + 1;
|
||||
let total_cols = (all_feature_indices.len() + 2) as u16;
|
||||
sheet
|
||||
.merge_range(
|
||||
note_row,
|
||||
0,
|
||||
note_row,
|
||||
total_cols.saturating_sub(1),
|
||||
&format!(
|
||||
"Only the first {} postcodes shown (randomly sampled from results)",
|
||||
MAX_EXPORT_POSTCODES
|
||||
),
|
||||
¬e_fmt,
|
||||
)
|
||||
.map_err(|err| format!("Failed to write note: {err}"))?;
|
||||
}
|
||||
|
||||
// Column widths
|
||||
sheet.set_column_width(0, 12).ok();
|
||||
sheet.set_column_width(1, 12).ok();
|
||||
for col_offset in 0..all_feature_indices.len() {
|
||||
let col = (col_offset + 2) as u16;
|
||||
let feat_name = &feature_names[all_feature_indices[col_offset]];
|
||||
let width = (feat_name.len() as f64 * 1.1).clamp(10.0, 30.0);
|
||||
sheet.set_column_width(col, width).ok();
|
||||
}
|
||||
|
||||
let buf = workbook
|
||||
.save_to_buffer()
|
||||
.map_err(|err| format!("Failed to save workbook: {err}"))?;
|
||||
|
||||
let t_total = t0.elapsed();
|
||||
info!(
|
||||
postcodes = postcode_aggs.len(),
|
||||
sampled = was_sampled,
|
||||
features = all_feature_indices.len(),
|
||||
has_og_image = og_image_bytes.is_some(),
|
||||
bytes = buf.len(),
|
||||
total_ms = format_args!("{:.1}", t_total.as_secs_f64() * 1000.0),
|
||||
"GET /api/export"
|
||||
);
|
||||
|
||||
Ok(buf)
|
||||
})
|
||||
.await
|
||||
.map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err.to_string()))?
|
||||
.map_err(|err| (StatusCode::INTERNAL_SERVER_ERROR, err))?;
|
||||
|
||||
Ok((
|
||||
[
|
||||
(
|
||||
header::CONTENT_TYPE,
|
||||
"application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
|
||||
),
|
||||
(
|
||||
header::CONTENT_DISPOSITION,
|
||||
"attachment; filename=\"narrowit-export.xlsx\"",
|
||||
),
|
||||
],
|
||||
bytes,
|
||||
))
|
||||
}
|
||||
|
|
@ -39,11 +39,19 @@ pub struct EnumFeatureStats {
|
|||
counts: HashMap<String, u64>,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PricePoint {
|
||||
year: f32,
|
||||
price: f32,
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HexagonStatsResponse {
|
||||
count: usize,
|
||||
numeric_features: Vec<NumericFeatureStats>,
|
||||
enum_features: Vec<EnumFeatureStats>,
|
||||
#[serde(skip_serializing_if = "Vec::is_empty")]
|
||||
price_history: Vec<PricePoint>,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
|
|
@ -148,6 +156,43 @@ pub async fn get_hexagon_stats(
|
|||
|
||||
let total_count = matching_rows.len();
|
||||
|
||||
// Collect price history (year, price) pairs
|
||||
let price_history = {
|
||||
let year_idx = state.feature_name_to_index.get("Transaction year").copied();
|
||||
let price_idx = state.feature_name_to_index.get("Last known price").copied();
|
||||
match (year_idx, price_idx) {
|
||||
(Some(yi), Some(pi)) => {
|
||||
let mut points: Vec<PricePoint> = matching_rows
|
||||
.iter()
|
||||
.filter_map(|&row| {
|
||||
let year = feature_data[row * num_features + yi];
|
||||
let price = feature_data[row * num_features + pi];
|
||||
if year.is_finite() && price.is_finite() {
|
||||
Some(PricePoint { year, price })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// Cap at 5000 points by evenly sampling
|
||||
if points.len() > 5000 {
|
||||
let step = points.len() as f64 / 5000.0;
|
||||
points = (0..5000)
|
||||
.map(|i| {
|
||||
let idx = (i as f64 * step) as usize;
|
||||
PricePoint {
|
||||
year: points[idx].year,
|
||||
price: points[idx].price,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
points
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
};
|
||||
|
||||
let mut numeric_features = Vec::new();
|
||||
let mut enum_features_out = Vec::new();
|
||||
|
||||
|
|
@ -267,6 +312,7 @@ pub async fn get_hexagon_stats(
|
|||
count: total_count,
|
||||
numeric_features,
|
||||
enum_features: enum_features_out,
|
||||
price_history,
|
||||
})
|
||||
})
|
||||
.await
|
||||
|
|
|
|||
|
|
@ -9,7 +9,9 @@ use serde_json::{Map, Value};
|
|||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN};
|
||||
use crate::parsing::{bounds_intersect, h3_cell_bounds, parse_bounds, parse_filters, row_passes_filters};
|
||||
use crate::parsing::{
|
||||
bounds_intersect, h3_cell_bounds, parse_bounds, parse_filters, row_passes_filters,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
#[derive(Serialize)]
|
||||
|
|
@ -111,7 +113,9 @@ fn build_feature_maps(
|
|||
|
||||
// Filter out cells that don't intersect the query bounds
|
||||
let (c_south, c_west, c_north, c_east) = h3_cell_bounds(cell, 0.0);
|
||||
if !bounds_intersect(c_south, c_west, c_north, c_east, q_south, q_west, q_north, q_east) {
|
||||
if !bounds_intersect(
|
||||
c_south, c_west, c_north, c_east, q_south, q_west, q_north, q_east,
|
||||
) {
|
||||
continue;
|
||||
}
|
||||
|
||||
|
|
@ -126,8 +130,7 @@ fn build_feature_maps(
|
|||
};
|
||||
|
||||
for feat_index in iter {
|
||||
if aggregation.mins[feat_index].is_finite()
|
||||
&& aggregation.maxs[feat_index].is_finite()
|
||||
if aggregation.mins[feat_index].is_finite() && aggregation.maxs[feat_index].is_finite()
|
||||
{
|
||||
if let (Some(min_num), Some(max_num)) = (
|
||||
serde_json::Number::from_f64(aggregation.mins[feat_index] as f64),
|
||||
|
|
|
|||
14
server-rs/src/routes/me.rs
Normal file
14
server-rs/src/routes/me.rs
Normal file
|
|
@ -0,0 +1,14 @@
|
|||
use axum::http::StatusCode;
|
||||
use axum::response::IntoResponse;
|
||||
use axum::Extension;
|
||||
use serde_json::json;
|
||||
|
||||
use crate::auth::OptionalUser;
|
||||
|
||||
pub async fn get_me(Extension(user): Extension<OptionalUser>) -> impl IntoResponse {
|
||||
let body = match user.0 {
|
||||
Some(usr) => json!({ "user": usr }),
|
||||
None => json!({ "user": null }),
|
||||
};
|
||||
(StatusCode::OK, axum::Json(body))
|
||||
}
|
||||
90
server-rs/src/routes/pb_proxy.rs
Normal file
90
server-rs/src/routes/pb_proxy.rs
Normal file
|
|
@ -0,0 +1,90 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use axum::body::Body;
|
||||
use axum::extract::Request;
|
||||
use axum::http::{HeaderName, StatusCode};
|
||||
use axum::response::{IntoResponse, Response};
|
||||
use tracing::warn;
|
||||
|
||||
use crate::state::AppState;
|
||||
|
||||
pub async fn proxy_to_pocketbase(state: Arc<AppState>, req: Request) -> impl IntoResponse {
|
||||
let pb_url = match &state.pocketbase_url {
|
||||
Some(url) => url.trim_end_matches('/'),
|
||||
None => {
|
||||
return Response::builder()
|
||||
.status(StatusCode::SERVICE_UNAVAILABLE)
|
||||
.body(Body::from("PocketBase not configured"))
|
||||
.unwrap();
|
||||
}
|
||||
};
|
||||
|
||||
let path = req.uri().path();
|
||||
let target_path = path.strip_prefix("/pb").unwrap_or(path);
|
||||
let query = req
|
||||
.uri()
|
||||
.query()
|
||||
.map(|qs| format!("?{qs}"))
|
||||
.unwrap_or_default();
|
||||
let url = format!("{pb_url}{target_path}{query}");
|
||||
|
||||
let method = req.method().clone();
|
||||
let mut builder = state.http_client.request(method, &url);
|
||||
|
||||
// Forward headers except host
|
||||
for (name, value) in req.headers() {
|
||||
if name != "host" {
|
||||
builder = builder.header(name.clone(), value.clone());
|
||||
}
|
||||
}
|
||||
|
||||
// Forward body
|
||||
let body_bytes = match axum::body::to_bytes(req.into_body(), 10 * 1024 * 1024).await {
|
||||
Ok(bytes) => bytes,
|
||||
Err(err) => {
|
||||
warn!("Failed to read request body: {err}");
|
||||
return Response::builder()
|
||||
.status(StatusCode::BAD_REQUEST)
|
||||
.body(Body::from("Failed to read request body"))
|
||||
.unwrap();
|
||||
}
|
||||
};
|
||||
builder = builder.body(body_bytes);
|
||||
|
||||
match builder.send().await {
|
||||
Ok(upstream) => {
|
||||
let status = upstream.status();
|
||||
let mut response = Response::builder().status(status);
|
||||
|
||||
for (name, value) in upstream.headers() {
|
||||
// Skip hop-by-hop headers
|
||||
if name == "transfer-encoding" {
|
||||
continue;
|
||||
}
|
||||
response = response.header(
|
||||
HeaderName::from_bytes(name.as_ref())
|
||||
.unwrap_or(HeaderName::from_static("x-invalid")),
|
||||
value.clone(),
|
||||
);
|
||||
}
|
||||
|
||||
match upstream.bytes().await {
|
||||
Ok(bytes) => response.body(Body::from(bytes)).unwrap(),
|
||||
Err(err) => {
|
||||
warn!("Failed to read upstream response: {err}");
|
||||
Response::builder()
|
||||
.status(StatusCode::BAD_GATEWAY)
|
||||
.body(Body::from("Failed to read upstream response"))
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
}
|
||||
Err(err) => {
|
||||
warn!("PocketBase proxy error: {err}");
|
||||
Response::builder()
|
||||
.status(StatusCode::BAD_GATEWAY)
|
||||
.body(Body::from("PocketBase unavailable"))
|
||||
.unwrap()
|
||||
}
|
||||
}
|
||||
}
|
||||
|
|
@ -13,6 +13,7 @@ use crate::state::AppState;
|
|||
|
||||
#[derive(Serialize)]
|
||||
pub struct PostcodesResponse {
|
||||
r#type: &'static str,
|
||||
features: Vec<Map<String, Value>>,
|
||||
}
|
||||
|
||||
|
|
@ -181,37 +182,88 @@ pub async fn get_postcodes(
|
|||
continue;
|
||||
}
|
||||
|
||||
// Compute postcode polygon bounding box and check intersection with query bounds
|
||||
let vertices = &postcode_data.vertices[pc_idx];
|
||||
// Compute postcode polygon bounding box across ALL parts and check intersection
|
||||
let rings = &postcode_data.polygons[pc_idx];
|
||||
let (mut pc_south, mut pc_north) = (f64::INFINITY, f64::NEG_INFINITY);
|
||||
let (mut pc_west, mut pc_east) = (f64::INFINITY, f64::NEG_INFINITY);
|
||||
for &[lon, lat] in vertices {
|
||||
let lon_f = lon as f64;
|
||||
let lat_f = lat as f64;
|
||||
if lat_f < pc_south { pc_south = lat_f; }
|
||||
if lat_f > pc_north { pc_north = lat_f; }
|
||||
if lon_f < pc_west { pc_west = lon_f; }
|
||||
if lon_f > pc_east { pc_east = lon_f; }
|
||||
for ring in rings {
|
||||
for &[lon, lat] in ring {
|
||||
let lon_f = lon as f64;
|
||||
let lat_f = lat as f64;
|
||||
if lat_f < pc_south {
|
||||
pc_south = lat_f;
|
||||
}
|
||||
if lat_f > pc_north {
|
||||
pc_north = lat_f;
|
||||
}
|
||||
if lon_f < pc_west {
|
||||
pc_west = lon_f;
|
||||
}
|
||||
if lon_f > pc_east {
|
||||
pc_east = lon_f;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
if !bounds_intersect(pc_south, pc_west, pc_north, pc_east, south, west, north, east) {
|
||||
if !bounds_intersect(
|
||||
pc_south, pc_west, pc_north, pc_east, south, west, north, east,
|
||||
) {
|
||||
filtered_out += 1;
|
||||
continue;
|
||||
}
|
||||
|
||||
let mut map = Map::new();
|
||||
map.insert(
|
||||
// Build GeoJSON geometry: Polygon (1 ring) or MultiPolygon (2+ rings)
|
||||
let geometry = if rings.len() == 1 {
|
||||
let coords: Vec<Value> = rings[0]
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![Value::from(*lon as f64), Value::from(*lat as f64)])
|
||||
})
|
||||
.collect();
|
||||
let mut geo = Map::new();
|
||||
geo.insert("type".into(), Value::String("Polygon".into()));
|
||||
geo.insert(
|
||||
"coordinates".into(),
|
||||
Value::Array(vec![Value::Array(coords)]),
|
||||
);
|
||||
geo
|
||||
} else {
|
||||
let polys: Vec<Value> = rings
|
||||
.iter()
|
||||
.map(|ring| {
|
||||
let coords: Vec<Value> = ring
|
||||
.iter()
|
||||
.map(|[lon, lat]| {
|
||||
Value::Array(vec![
|
||||
Value::from(*lon as f64),
|
||||
Value::from(*lat as f64),
|
||||
])
|
||||
})
|
||||
.collect();
|
||||
Value::Array(vec![Value::Array(coords)])
|
||||
})
|
||||
.collect();
|
||||
let mut geo = Map::new();
|
||||
geo.insert("type".into(), Value::String("MultiPolygon".into()));
|
||||
geo.insert("coordinates".into(), Value::Array(polys));
|
||||
geo
|
||||
};
|
||||
|
||||
// Build properties
|
||||
let centroid = postcode_data.centroids[pc_idx];
|
||||
let mut props = Map::new();
|
||||
props.insert(
|
||||
"postcode".into(),
|
||||
Value::String(postcode_data.postcodes[pc_idx].clone()),
|
||||
);
|
||||
map.insert("count".into(), Value::Number(aggregation.count.into()));
|
||||
|
||||
// Add vertices as array of [lon, lat] pairs
|
||||
let vertices_array: Vec<Value> = vertices
|
||||
.iter()
|
||||
.map(|[lon, lat]| Value::Array(vec![Value::from(*lon as f64), Value::from(*lat as f64)]))
|
||||
.collect();
|
||||
map.insert("vertices".into(), Value::Array(vertices_array));
|
||||
props.insert("count".into(), Value::Number(aggregation.count.into()));
|
||||
props.insert(
|
||||
"centroid".into(),
|
||||
Value::Array(vec![
|
||||
Value::from(centroid.1 as f64), // lon
|
||||
Value::from(centroid.0 as f64), // lat
|
||||
]),
|
||||
);
|
||||
|
||||
let iter: Box<dyn Iterator<Item = usize>> = if let Some(idx) = field_indices.as_ref() {
|
||||
Box::new(idx.iter().copied())
|
||||
|
|
@ -227,13 +279,19 @@ pub async fn get_postcodes(
|
|||
serde_json::Number::from_f64(aggregation.mins[feat_index] as f64),
|
||||
serde_json::Number::from_f64(aggregation.maxs[feat_index] as f64),
|
||||
) {
|
||||
map.insert(min_keys[feat_index].clone(), Value::Number(min_num));
|
||||
map.insert(max_keys[feat_index].clone(), Value::Number(max_num));
|
||||
props.insert(min_keys[feat_index].clone(), Value::Number(min_num));
|
||||
props.insert(max_keys[feat_index].clone(), Value::Number(max_num));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
features.push(map);
|
||||
// Build GeoJSON Feature
|
||||
let mut feature = Map::new();
|
||||
feature.insert("type".into(), Value::String("Feature".into()));
|
||||
feature.insert("geometry".into(), Value::Object(geometry));
|
||||
feature.insert("properties".into(), Value::Object(props));
|
||||
|
||||
features.push(feature);
|
||||
}
|
||||
|
||||
let t_total = t0.elapsed();
|
||||
|
|
@ -248,7 +306,10 @@ pub async fn get_postcodes(
|
|||
"GET /api/postcodes"
|
||||
);
|
||||
|
||||
Ok(PostcodesResponse { features })
|
||||
Ok(PostcodesResponse {
|
||||
r#type: "FeatureCollection",
|
||||
features,
|
||||
})
|
||||
})
|
||||
.await
|
||||
.map_err(|error| (StatusCode::INTERNAL_SERVER_ERROR, error.to_string()))?
|
||||
|
|
@ -257,20 +318,11 @@ pub async fn get_postcodes(
|
|||
Ok(Json(response))
|
||||
}
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct PostcodeLookupResponse {
|
||||
pub postcode: String,
|
||||
pub latitude: f64,
|
||||
pub longitude: f64,
|
||||
/// Polygon vertices as [[lon, lat], ...] for rendering highlight
|
||||
pub vertices: Vec<[f64; 2]>,
|
||||
}
|
||||
|
||||
/// Look up a single postcode and return its centroid coordinates and polygon.
|
||||
/// Look up a single postcode and return its centroid coordinates and geometry.
|
||||
pub async fn get_postcode_lookup(
|
||||
state: Arc<AppState>,
|
||||
Path(postcode): Path<String>,
|
||||
) -> Result<Json<PostcodeLookupResponse>, StatusCode> {
|
||||
) -> Result<Json<Value>, StatusCode> {
|
||||
// Normalize the postcode: uppercase, remove extra spaces, ensure single space
|
||||
let normalized = postcode
|
||||
.to_uppercase()
|
||||
|
|
@ -282,17 +334,40 @@ pub async fn get_postcode_lookup(
|
|||
|
||||
if let Some(&idx) = postcode_data.postcode_to_idx.get(&normalized) {
|
||||
let (lat, lon) = postcode_data.centroids[idx];
|
||||
let vertices: Vec<[f64; 2]> = postcode_data.vertices[idx]
|
||||
.iter()
|
||||
.map(|[lo, la]| [*lo as f64, *la as f64])
|
||||
.collect();
|
||||
let rings = &postcode_data.polygons[idx];
|
||||
|
||||
// Build GeoJSON geometry
|
||||
let geometry = if rings.len() == 1 {
|
||||
let coords: Vec<Value> = rings[0]
|
||||
.iter()
|
||||
.map(|[lo, la]| {
|
||||
Value::Array(vec![Value::from(*lo as f64), Value::from(*la as f64)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "Polygon", "coordinates": [coords] })
|
||||
} else {
|
||||
let polys: Vec<Value> = rings
|
||||
.iter()
|
||||
.map(|ring| {
|
||||
let coords: Vec<Value> = ring
|
||||
.iter()
|
||||
.map(|[lo, la]| {
|
||||
Value::Array(vec![Value::from(*lo as f64), Value::from(*la as f64)])
|
||||
})
|
||||
.collect();
|
||||
Value::Array(vec![Value::Array(coords)])
|
||||
})
|
||||
.collect();
|
||||
serde_json::json!({ "type": "MultiPolygon", "coordinates": polys })
|
||||
};
|
||||
|
||||
info!(postcode = %normalized, "GET /api/postcode/{postcode}");
|
||||
Ok(Json(PostcodeLookupResponse {
|
||||
postcode: normalized,
|
||||
latitude: lat as f64,
|
||||
longitude: lon as f64,
|
||||
vertices,
|
||||
}))
|
||||
Ok(Json(serde_json::json!({
|
||||
"postcode": normalized,
|
||||
"latitude": lat as f64,
|
||||
"longitude": lon as f64,
|
||||
"geometry": geometry,
|
||||
})))
|
||||
} else {
|
||||
Err(StatusCode::NOT_FOUND)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,5 +1,8 @@
|
|||
use std::sync::Arc;
|
||||
|
||||
use rustc_hash::FxHashMap;
|
||||
|
||||
use crate::auth::TokenCache;
|
||||
use crate::data::{POICategoryGroup, POIData, PostcodeData, PropertyData};
|
||||
use crate::routes::FeaturesResponse;
|
||||
use crate::utils::GridIndex;
|
||||
|
|
@ -30,6 +33,10 @@ pub struct AppState {
|
|||
pub public_url: String,
|
||||
/// Contents of index.html read at startup, used for crawler OG injection
|
||||
pub index_html: Option<String>,
|
||||
/// Shared HTTP client for proxying to the screenshot sidecar
|
||||
/// Shared HTTP client for proxying to the screenshot sidecar and PocketBase
|
||||
pub http_client: reqwest::Client,
|
||||
/// PocketBase server URL for authentication (e.g. http://localhost:8090)
|
||||
pub pocketbase_url: Option<String>,
|
||||
/// Token validation cache (60s TTL)
|
||||
pub token_cache: Arc<TokenCache>,
|
||||
}
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue