andras/perfect-postcode
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages 2 Wiki Activity Actions

Lots of improvements

Browse source
This commit is contained in:
Andras Schmelczer 2026-02-22 21:09:07 +00:00
parent 205302dbb8
commit eb02b5832b
39 changed files with 699 additions and 271 deletions
Download patch file Download diff file Expand all files Collapse all files

16
server-rs/Cargo.lock generated
View file

@ -2743,6 +2743,7 @@ dependencies = [
"bytes",
"encoding_rs",
"futures-core",
"futures-util",
"h2",
"http",
"http-body",
@ -2767,12 +2768,14 @@ dependencies = [
"tokio",
"tokio-native-tls",
"tokio-rustls",
"tokio-util",
"tower",
"tower-http",
"tower-service",
"url",
"wasm-bindgen",
"wasm-bindgen-futures",
"wasm-streams",
"web-sys",
"webpki-roots",
]
@ -3803,6 +3806,19 @@ dependencies = [
"unicode-ident",
]
[[package]]
name = "wasm-streams"
version = "0.4.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "15053d8d85c7eccdbefef60f06769760a563c7f0a9d6902a13d35c7800b0ad65"
dependencies = [
"futures-util",
"js-sys",
"wasm-bindgen",
"wasm-bindgen-futures",
"web-sys",
]
[[package]]
name = "web-sys"
version = "0.3.85"

2
server-rs/Cargo.toml
View file

@ -22,7 +22,7 @@ tracing-subscriber = { version = "0.3", features = ["env-filter", "fmt"] }
tracing-appender = "0.2"
metrics = "0.24"
metrics-exporter-prometheus = "0.16"
reqwest = { version = "0.12", features = ["rustls-tls", "json"] }
reqwest = { version = "0.12", features = ["rustls-tls", "json", "stream"] }
urlencoding = "2"
rust_xlsxwriter = "0.79"
pmtiles = { version = "0.12", features = ["mmap-async-tokio"] }

21
server-rs/src/data/places.rs
View file

@ -22,18 +22,8 @@ pub struct PlaceData {
fn type_rank(place_type: &str) -> u8 {
match place_type {
"city" => 0,
"borough" => 1,
"town" => 2,
"suburb" => 3,
"quarter" => 4,
"neighbourhood" => 5,
"village" => 6,
"station" => 7,
"island" => 8,
"hamlet" => 9,
"locality" => 10,
"isolated_dwelling" => 11,
_ => 12,
"station" => 1,
_ => 2,
}
}
@ -159,10 +149,7 @@ mod tests {
#[test]
fn type_rank_ordering() {
assert!(type_rank("city") < type_rank("town"));
assert!(type_rank("town") < type_rank("suburb"));
assert!(type_rank("suburb") < type_rank("village"));
assert!(type_rank("village") < type_rank("hamlet"));
assert!(type_rank("hamlet") < type_rank("isolated_dwelling"));
assert!(type_rank("city") < type_rank("station"));
assert!(type_rank("station") < type_rank("unknown"));
}
}

26
server-rs/src/data/travel_time.rs
View file

@ -8,8 +8,15 @@ use polars::lazy::frame::LazyFrame;
use rustc_hash::{FxHashMap, FxHashSet};
use tracing::info;
/// Cached postcode → travel_minutes mapping for a single destination file.
pub type TravelData = Arc<FxHashMap<String, i16>>;
/// Per-postcode travel time data: median and optional best-case (transit only).
#[derive(Clone, Copy)]
pub struct TravelDataRow {
pub minutes: i16,
pub best_minutes: Option<i16>,
}
/// Cached postcode → travel time data for a single destination file.
pub type TravelData = Arc<FxHashMap<String, TravelDataRow>>;
/// Simple LRU cache for travel time data, limited to `capacity` entries.
struct LruCache {
@ -159,12 +166,23 @@ impl TravelTimeStore {
.context("Missing 'travel_minutes' column")?
.i16()
.context("'travel_minutes' is not i16")?;
let best = df
.column("best_minutes")
.ok()
.map(|col| col.i16().expect("'best_minutes' is not i16"));
let mut map = FxHashMap::default();
map.reserve(df.height());
for (pc, min) in postcodes.into_iter().zip(minutes.into_iter()) {
for (i, (pc, min)) in postcodes.into_iter().zip(minutes.into_iter()).enumerate() {
if let (Some(pc), Some(min)) = (pc, min) {
map.insert(pc.to_string(), min);
let best_min = best.as_ref().and_then(|b| b.get(i));
map.insert(
pc.to_string(),
TravelDataRow {
minutes: min,
best_minutes: best_min,
},
);
}
}

7
server-rs/src/main.rs
View file

@ -424,6 +424,7 @@ async fn main() -> anyhow::Result<()> {
let state_invites_create = state.clone();
let state_invite_get = state.clone();
let state_redeem_invite = state.clone();
let state_rightmove = state.clone();
let api = Router::new()
.route(
@ -495,6 +496,10 @@ async fn main() -> anyhow::Result<()> {
"/api/streetview",
get(move |query| routes::get_streetview(state_streetview.clone(), query)),
)
.route(
"/api/rightmove-location",
get(move |query| routes::get_rightmove_typeahead(state_rightmove.clone(), query)),
)
.route(
"/api/subscription",
patch(move |ext, body| {
@ -569,7 +574,7 @@ async fn main() -> anyhow::Result<()> {
let app = if let Some(ref dist) = cli.dist {
api.fallback_service(
ServeDir::new(dist).not_found_service(ServeFile::new(dist.join("index.html"))),
ServeDir::new(dist).fallback(ServeFile::new(dist.join("index.html"))),
)
} else {
api

61
server-rs/src/pocketbase.rs
View file

@ -405,35 +405,49 @@ pub async fn ensure_oauth_providers(
let base_url = base_url.trim_end_matches('/');
let token = auth_superuser(client, base_url, admin_email, admin_password).await?;
// GET current settings
// Set meta.appURL in global settings for OAuth redirects
let app_url = format!("{}/pb", public_url.trim_end_matches('/'));
let settings_url = format!("{base_url}/api/settings");
let patch_resp = client
.patch(&settings_url)
.header("Authorization", format!("Bearer {token}"))
.json(&serde_json::json!({ "meta": { "appURL": app_url } }))
.send()
.await?;
if !patch_resp.status().is_success() {
let status = patch_resp.status();
let text = patch_resp.text().await.unwrap_or_default();
anyhow::bail!("Failed to update PocketBase meta.appURL ({status}): {text}");
}
info!("PocketBase meta.appURL set to {app_url}");
// PocketBase 0.23+: OAuth providers are configured per-collection, not in global settings.
// GET the users collection to update its oauth2 config.
let collection_url = format!("{base_url}/api/collections/users");
let resp = client
.get(&settings_url)
.get(&collection_url)
.header("Authorization", format!("Bearer {token}"))
.send()
.await?;
if !resp.status().is_success() {
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
anyhow::bail!("Failed to fetch PocketBase settings ({status}): {text}");
anyhow::bail!("Failed to fetch users collection ({status}): {text}");
}
let mut settings: serde_json::Value = resp.json().await?;
let mut collection: serde_json::Value = resp.json().await?;
// Set meta.appUrl for OAuth redirect
let app_url = format!("{}/pb", public_url.trim_end_matches('/'));
if let Some(meta) = settings.get_mut("meta") {
meta["appUrl"] = serde_json::json!(app_url);
} else {
settings["meta"] = serde_json::json!({ "appUrl": app_url });
}
let oauth2 = collection
.get_mut("oauth2")
.ok_or_else(|| anyhow::anyhow!("users collection missing oauth2 field"))?;
// Update OAuth2 providers
let providers = settings
.pointer_mut("/oauth2/providers")
// Ensure enabled
oauth2["enabled"] = serde_json::json!(true);
let providers = oauth2
.get_mut("providers")
.and_then(|v| v.as_array_mut())
.ok_or_else(|| anyhow::anyhow!("PocketBase settings missing oauth2.providers array — cannot configure OAuth"))?;
.ok_or_else(|| anyhow::anyhow!("users collection missing oauth2.providers array"))?;
let google = match providers
.iter()
@ -441,7 +455,7 @@ pub async fn ensure_oauth_providers(
{
Some(idx) => &mut providers[idx],
None => {
info!("Google provider not found in PocketBase settings — adding it");
info!("Google provider not found — adding it");
providers.push(serde_json::json!({"name": "google"}));
providers.last_mut().expect("just pushed")
}
@ -449,23 +463,20 @@ pub async fn ensure_oauth_providers(
google["clientId"] = serde_json::json!(google_client_id);
google["clientSecret"] = serde_json::json!(google_client_secret);
google["enabled"] = serde_json::json!(true);
info!("Configured Google OAuth provider");
// PATCH settings back
// PATCH the collection
let patch_resp = client
.patch(&settings_url)
.patch(&collection_url)
.header("Authorization", format!("Bearer {token}"))
.json(&settings)
.json(&serde_json::json!({ "oauth2": oauth2 }))
.send()
.await?;
if !patch_resp.status().is_success() {
let status = patch_resp.status();
let text = patch_resp.text().await.unwrap_or_default();
anyhow::bail!("Failed to update PocketBase settings ({status}): {text}");
anyhow::bail!("Failed to update users collection OAuth ({status}): {text}");
}
info!("PocketBase OAuth settings updated (appUrl: {app_url})");
info!("PocketBase OAuth configured on users collection");
Ok(())
}

2
server-rs/src/routes.rs
View file

@ -19,6 +19,7 @@ mod streetview;
mod stripe_webhook;
mod newsletter;
pub(crate) mod pricing;
mod rightmove_typeahead;
mod subscription;
mod tiles;
pub(crate) mod travel_time;
@ -46,4 +47,5 @@ pub use pricing::get_pricing;
pub use stripe_webhook::post_stripe_webhook;
pub use subscription::patch_subscription;
pub use tiles::{get_style, get_tile, init_tile_reader};
pub use rightmove_typeahead::get_rightmove_typeahead;
pub use travel_modes::get_travel_modes;

28
server-rs/src/routes/hexagon_stats.rs
View file

@ -59,6 +59,8 @@ pub struct HexagonStatsResponse {
pub enum_features: Vec<EnumFeatureStats>,
#[serde(skip_serializing_if = "Vec::is_empty")]
pub price_history: Vec<PricePoint>,
#[serde(skip_serializing_if = "Option::is_none")]
pub central_postcode: Option<String>,
}
#[derive(Deserialize)]
@ -136,6 +138,31 @@ pub async fn get_hexagon_stats(
let total_count = matching_rows.len();
// Find the postcode of the property closest to the hexagon center
let central_postcode = if !matching_rows.is_empty() {
let center: h3o::LatLng = cell.into();
let center_lat = center.lat() as f32;
let center_lon = center.lng() as f32;
let closest_row = matching_rows
.iter()
.copied()
.min_by(|&a, &b| {
let da_lat = state.data.lat[a] - center_lat;
let da_lon = state.data.lon[a] - center_lon;
let db_lat = state.data.lat[b] - center_lat;
let db_lon = state.data.lon[b] - center_lon;
let dist_a = da_lat * da_lat + da_lon * da_lon;
let dist_b = db_lat * db_lat + db_lon * db_lon;
dist_a
.partial_cmp(&dist_b)
.unwrap_or(std::cmp::Ordering::Equal)
})
.expect("matching_rows is non-empty");
Some(state.data.postcode(closest_row).to_string())
} else {
None
};
let price_history = stats::extract_price_history(
&matching_rows,
feature_data,
@ -170,6 +197,7 @@ pub async fn get_hexagon_stats(
numeric_features,
enum_features: enum_features_out,
price_history,
central_postcode,
})
})
.await

22
server-rs/src/routes/hexagons.rs
View file

@ -43,12 +43,13 @@ pub struct HexagonParams {
struct TravelEntry {
mode: String,
slug: String,
use_best: bool,
filter_min: Option<f32>,
filter_max: Option<f32>,
}
/// Parse `travel` param into a list of travel entries.
/// Format: `mode:slug` or `mode:slug:min:max`
/// Format: `mode:slug` or `mode:slug:best` or `mode:slug:min:max` or `mode:slug:best:min:max`
fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mut entries = Vec::new();
let mut seen_keys = Vec::new();
@ -63,12 +64,15 @@ fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
let mode = parts[0].trim().to_string();
let slug = parts[1].trim().to_string();
let (filter_min, filter_max) = if parts.len() >= 4 {
let min: f32 = parts[2]
let use_best = parts.len() >= 3 && parts[2].trim() == "best";
let filter_offset = if use_best { 1 } else { 0 };
let (filter_min, filter_max) = if parts.len() >= 4 + filter_offset {
let min: f32 = parts[2 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter min in '{}'", segment))?;
let max: f32 = parts[3]
let max: f32 = parts[3 + filter_offset]
.trim()
.parse()
.map_err(|_| format!("invalid travel filter max in '{}'", segment))?;
@ -85,6 +89,7 @@ fn parse_travel_entries(travel_str: &str) -> Result<Vec<TravelEntry>, String> {
entries.push(TravelEntry {
mode,
slug,
use_best,
filter_min,
filter_max,
});
@ -286,7 +291,14 @@ pub async fn get_hexagons(
let postcode = pc_interner.resolve(&pc_keys[row]);
travel_minutes.reserve(travel_entries.len());
for (ti, entry) in travel_entries.iter().enumerate() {
let minutes = travel_data[ti].get(postcode).copied();
let row_data = travel_data[ti].get(postcode);
let minutes = row_data.map(|r| {
if entry.use_best {
r.best_minutes.unwrap_or(r.minutes)
} else {
r.minutes
}
});
travel_minutes.push(minutes);
if let (Some(fmin), Some(fmax)) = (entry.filter_min, entry.filter_max) {
match minutes {

19
server-rs/src/routes/pb_proxy.rs
View file

@ -11,10 +11,11 @@ use crate::state::AppState;
/// Dedicated HTTP client for proxying — does not follow redirects so 3xx
/// responses are passed through to the browser (needed for OAuth flows).
/// No overall timeout because SSE (Server-Sent Events) connections used by
/// PocketBase realtime/OAuth2 are long-lived streams.
static PROXY_CLIENT: LazyLock<reqwest::Client> = LazyLock::new(|| {
reqwest::Client::builder()
.redirect(reqwest::redirect::Policy::none())
.timeout(Duration::from_secs(30))
.connect_timeout(Duration::from_secs(5))
.build()
.expect("Failed to build proxy HTTP client")
@ -97,16 +98,12 @@ pub async fn proxy_to_pocketbase(state: Arc<AppState>, req: Request) -> impl Int
}
}
match upstream.bytes().await {
Ok(bytes) => response.body(Body::from(bytes)).unwrap(),
Err(err) => {
warn!("Failed to read upstream response: {err}");
Response::builder()
.status(StatusCode::BAD_GATEWAY)
.body(Body::from("Failed to read upstream response"))
.unwrap()
}
}
// Stream the response body instead of buffering it entirely.
// This is critical for SSE (Server-Sent Events) used by PocketBase's
// realtime system and OAuth2 flow — buffering would hang forever
// since SSE responses never complete.
let body = Body::from_stream(upstream.bytes_stream());
response.body(body).unwrap()
}
Err(err) => {
warn!("PocketBase proxy error: {err}");

1
server-rs/src/routes/postcode_stats.rs
View file

@ -135,6 +135,7 @@ pub async fn get_postcode_stats(
numeric_features,
enum_features: enum_features_out,
price_history,
central_postcode: None,
})
})
.await

83
server-rs/src/routes/rightmove_typeahead.rs Normal file
View file

@ -0,0 +1,83 @@
use std::sync::Arc;
use axum::extract::Query;
use axum::http::StatusCode;
use axum::response::{IntoResponse, Json};
use serde::{Deserialize, Serialize};
use tracing::warn;
use crate::state::AppState;
const TYPEAHEAD_URL: &str = "https://los.rightmove.co.uk/typeahead";
#[derive(Deserialize)]
pub struct TypeaheadParams {
pub postcode: String,
}
#[derive(Serialize)]
pub struct TypeaheadResponse {
pub location_identifier: String,
}
#[derive(Deserialize)]
struct RightmoveMatch {
#[serde(rename = "type")]
match_type: String,
#[serde(rename = "displayName")]
display_name: String,
id: serde_json::Value,
}
#[derive(Deserialize)]
struct RightmoveTypeaheadResponse {
matches: Vec<RightmoveMatch>,
}
pub async fn get_rightmove_typeahead(
state: Arc<AppState>,
Query(params): Query<TypeaheadParams>,
) -> Result<Json<TypeaheadResponse>, axum::response::Response> {
let postcode = params.postcode.trim().to_uppercase();
let resp = state
.http_client
.get(TYPEAHEAD_URL)
.query(&[("query", &postcode), ("limit", &"10".to_string())])
.send()
.await
.map_err(|err| {
warn!(error = %err, "Rightmove typeahead request failed");
(StatusCode::BAD_GATEWAY, "Rightmove typeahead unavailable").into_response()
})?;
let data: RightmoveTypeaheadResponse = resp.json().await.map_err(|err| {
warn!(error = %err, "Failed to parse Rightmove typeahead response");
(StatusCode::BAD_GATEWAY, "Invalid typeahead response").into_response()
})?;
// Look for POSTCODE match first, then OUTCODE
for match_type in &["POSTCODE", "OUTCODE"] {
for m in &data.matches {
if m.match_type == *match_type
&& m.display_name.to_uppercase().replace(' ', "")
== postcode.replace(' ', "")
{
let id = match &m.id {
serde_json::Value::Number(n) => n.to_string(),
serde_json::Value::String(s) => s.clone(),
other => other.to_string(),
};
return Ok(Json(TypeaheadResponse {
location_identifier: format!("{}^{}", match_type, id),
}));
}
}
}
Err((
StatusCode::NOT_FOUND,
format!("No Rightmove location found for: {}", postcode),
)
.into_response())
}