This commit is contained in:
Andras Schmelczer 2026-07-12 20:37:39 +01:00
parent 9e4e65fa2a
commit ca771a7edf
32 changed files with 1467 additions and 109 deletions

View file

@ -75,7 +75,9 @@ pub use poi::{resolve_poi_category_filter, POICategoryGroup, POIData, SchoolMeta
pub use postcode_population::PostcodePopulation;
pub use postcodes::{OutcodeData, PostcodeData};
pub use property::{
precompute_h3, FeatureStats, Histogram, HistoricalPrice, PostcodePoiMetrics, PropertyData,
QuantRef, RenovationEvent, TenureEvent,
combine_nearest_distances, combined_station_feature_name,
combined_station_source_feature_names, precompute_h3, FeatureStats, Histogram, HistoricalPrice,
PostcodePoiMetrics, PropertyData, QuantRef, RenovationEvent, TenureEvent,
COMBINED_STATION_CATEGORY,
};
pub use travel_time::{slugify, TravelTimeStore};

View file

@ -7,7 +7,10 @@ use serde::Serialize;
use tracing::info;
use crate::consts::{NAN_U16, QUANT_SCALE};
use crate::data::{PropertyData, QuantRef};
use crate::data::{
combine_nearest_distances, combined_station_feature_name,
combined_station_source_feature_names, PropertyData, QuantRef,
};
use crate::utils::{normalize_postcode, GridIndex, InternedColumn};
const GRID_CELL_SIZE: f32 = 0.01;
@ -318,8 +321,17 @@ fn build_poi_filter_feature_data(
let mut encoded_columns = 0usize;
for (metric_idx, name) in poi_metrics.feature_names.iter().enumerate() {
let Some(values) = extract_optional_feature_f32(df, name)? else {
continue;
let values = match extract_optional_feature_f32(df, name)? {
Some(values) => values,
// Some POI columns (the combined-station distance) are synthesized
// in memory by PostcodePoiMetrics and never written to the listings
// parquet. Reconstruct them from their source columns so listing POI
// filters match the map exactly instead of silently rejecting every
// row on an all-NaN column.
None => match synthesize_missing_poi_column(df, name, row_count)? {
Some(values) => values,
None => continue,
},
};
for (row, value) in values.into_iter().enumerate() {
let dst = row * num_features + metric_idx;
@ -338,6 +350,45 @@ fn build_poi_filter_feature_data(
Ok(feature_data)
}
/// Reconstruct a POI metric column that `PostcodePoiMetrics` synthesizes in
/// memory and therefore is absent from the listings parquet. Currently only the
/// combined-station distance, derived as the elementwise nearest of its per-mode
/// source columns (the same rule the postcode side table uses). Returns `None`
/// if `name` is not a synthesized column, or if none of its sources are present.
fn synthesize_missing_poi_column(
df: &DataFrame,
name: &str,
row_count: usize,
) -> Result<Option<Vec<Option<f32>>>> {
if name != combined_station_feature_name() {
return Ok(None);
}
let mut sources: Vec<Vec<f32>> = Vec::new();
for source_name in combined_station_source_feature_names() {
if let Some(values) = extract_optional_feature_f32(df, &source_name)? {
sources.push(
values
.into_iter()
.map(|value| value.unwrap_or(f32::NAN))
.collect(),
);
}
}
if sources.is_empty() {
return Ok(None);
}
let source_refs: Vec<&[f32]> = sources.iter().map(Vec::as_slice).collect();
let combined = combine_nearest_distances(&source_refs, row_count);
Ok(Some(
combined
.into_iter()
.map(|value| value.is_finite().then_some(value))
.collect(),
))
}
fn feature_index(property_data: &PropertyData, name: &str) -> Option<usize> {
property_data
.feature_names
@ -741,6 +792,51 @@ mod tests {
assert!(!any_listing.listing_url.is_empty());
}
#[test]
fn synthesizes_combined_station_column_from_source_columns() {
// Mirrors the postcode side table: the combined-station distance is the
// finite minimum of the per-mode source columns present in the parquet.
let df = df![
"Distance to nearest amenity (Rail station) (km)" => [2.0f32, f32::NAN, 5.0],
"Distance to nearest amenity (Tube station) (km)" => [1.0f32, f32::NAN, f32::NAN],
"Distance to nearest amenity (DLR station) (km)" => [3.0f32, 4.0, f32::NAN],
]
.unwrap();
let combined = synthesize_missing_poi_column(&df, &combined_station_feature_name(), 3)
.unwrap()
.expect("combined-station column should be synthesized");
assert_eq!(combined[0], Some(1.0)); // min(2, 1, 3)
assert_eq!(combined[1], Some(4.0)); // only DLR present
assert_eq!(combined[2], Some(5.0)); // only Rail present
}
#[test]
fn does_not_synthesize_non_combined_or_sourceless_columns() {
let df = df![
"Distance to nearest amenity (Rail station) (km)" => [1.0f32],
]
.unwrap();
// A real per-mode column is read from the parquet, never synthesized.
assert!(synthesize_missing_poi_column(
&df,
"Distance to nearest amenity (Rail station) (km)",
1
)
.unwrap()
.is_none());
// The combined column cannot be built when no source columns exist.
let empty = df!["Postcode" => ["AB1 2CD"]].unwrap();
assert!(
synthesize_missing_poi_column(&empty, &combined_station_feature_name(), 1)
.unwrap()
.is_none()
);
}
#[test]
fn extracts_price_history_from_parquet() {
let path = PathBuf::from("src/data/testdata/listings_price_history.parquet");

View file

@ -18,7 +18,10 @@ mod quant;
mod stats;
pub use h3::precompute_h3;
pub use poi_metrics::{PostcodePoiMetrics, COMBINED_STATION_CATEGORY};
pub use poi_metrics::{
combine_nearest_distances, combined_station_feature_name, combined_station_source_feature_names,
PostcodePoiMetrics, COMBINED_STATION_CATEGORY,
};
pub use quant::QuantRef;
pub use stats::{FeatureStats, Histogram};

View file

@ -1445,6 +1445,87 @@ pub async fn ensure_oauth_providers(
Ok(())
}
/// Point the password-reset email at the SPA's own `/reset-password` page.
///
/// PocketBase's default reset template links to `{APP_URL}/_/#/auth/confirm-password-reset/{TOKEN}`,
/// i.e. its superuser UI under `/pb/_/`. `meta.appURL` is `{public_url}/pb` (set in
/// `ensure_oauth_providers` for OAuth redirects), and the `/pb` proxy allowlist deliberately
/// rejects `/_/` (see `routes/pb_proxy.rs`), so the emailed link 404s. We therefore bake the raw
/// `public_url` (NOT `appURL`, which carries the `/pb` prefix) straight into the template body and
/// keep PocketBase's `{TOKEN}` placeholder for it to substitute at send time. The SPA route reads
/// `?token=` and calls `confirmPasswordReset`.
pub async fn ensure_password_reset_template(
client: &Client,
base_url: &str,
admin_email: &str,
admin_password: &str,
public_url: &str,
) -> anyhow::Result<()> {
let base_url = base_url.trim_end_matches('/');
let token = auth_superuser(client, base_url, admin_email, admin_password).await?;
let site = public_url.trim_end_matches('/');
// `{{TOKEN}}` renders to the literal `{TOKEN}` placeholder that PocketBase fills in.
let action_url = format!("{site}/reset-password?token={{TOKEN}}");
let body = format!(
"<p>Hello,</p>\n\
<p>Click the button below to choose a new password for your Perfect Postcode account.</p>\n\
<p><a class=\"btn\" href=\"{action_url}\" target=\"_blank\" rel=\"noopener\">Reset password</a></p>\n\
<p>If you did not request this, you can safely ignore this email.</p>\n\
<p>Thanks,<br/>The Perfect Postcode team</p>"
);
// PocketBase 0.23+: email templates are per-collection. GET the users collection, set the
// reset template, and PATCH it back (mirrors the OAuth config flow above).
let collection_url = format!("{base_url}/api/collections/users");
let resp = client
.get(&collection_url)
.header("Authorization", format!("Bearer {token}"))
.send()
.await?;
if !resp.status().is_success() {
let status = resp.status();
let text = resp.text().await.unwrap_or_default();
anyhow::bail!("Failed to fetch users collection for reset template ({status}): {text}");
}
let mut collection: serde_json::Value = resp.json().await?;
// The field is an object `{subject, body}`; preserve any sibling keys PocketBase may add. If a
// future PocketBase version renames or moves it, warn and skip rather than blocking server
// startup over an email template: a broken reset link is far less bad than a server that won't boot.
let Some(template) = collection
.get_mut("resetPasswordTemplate")
.and_then(|v| v.as_object_mut())
else {
warn!(
"users collection has no resetPasswordTemplate object; \
leaving the reset email on PocketBase's default (which 404s via /pb/_/)"
);
return Ok(());
};
template.insert(
"subject".to_string(),
serde_json::json!("Reset your Perfect Postcode password"),
);
template.insert("body".to_string(), serde_json::json!(body));
let reset_template = collection["resetPasswordTemplate"].clone();
let patch_resp = client
.patch(&collection_url)
.header("Authorization", format!("Bearer {token}"))
.json(&serde_json::json!({ "resetPasswordTemplate": reset_template }))
.send()
.await?;
if !patch_resp.status().is_success() {
let status = patch_resp.status();
let text = patch_resp.text().await.unwrap_or_default();
anyhow::bail!("Failed to set password-reset template ({status}): {text}");
}
info!("PocketBase password-reset email template set to {action_url}");
Ok(())
}
/// Spawn a background task that polls PocketBase every 60 seconds for collection counts
/// and exposes them as Prometheus gauges.
pub fn start_metrics_poller(shared: Arc<crate::state::SharedState>) {

View file

@ -166,6 +166,7 @@ impl AppState {
listing_status: InternedColumn::build(&[]),
listing_date_iso: Vec::new(),
features: Vec::new(),
price_history: Vec::new(),
filter_feature_data: Vec::new(),
poi_filter_feature_data: Vec::new(),
grid: GridIndex::build(&[], &[], 0.01),