fine 2
This commit is contained in:
parent
9e4e65fa2a
commit
ca771a7edf
32 changed files with 1467 additions and 109 deletions
|
|
@ -75,7 +75,9 @@ pub use poi::{resolve_poi_category_filter, POICategoryGroup, POIData, SchoolMeta
|
|||
pub use postcode_population::PostcodePopulation;
|
||||
pub use postcodes::{OutcodeData, PostcodeData};
|
||||
pub use property::{
|
||||
precompute_h3, FeatureStats, Histogram, HistoricalPrice, PostcodePoiMetrics, PropertyData,
|
||||
QuantRef, RenovationEvent, TenureEvent,
|
||||
combine_nearest_distances, combined_station_feature_name,
|
||||
combined_station_source_feature_names, precompute_h3, FeatureStats, Histogram, HistoricalPrice,
|
||||
PostcodePoiMetrics, PropertyData, QuantRef, RenovationEvent, TenureEvent,
|
||||
COMBINED_STATION_CATEGORY,
|
||||
};
|
||||
pub use travel_time::{slugify, TravelTimeStore};
|
||||
|
|
|
|||
|
|
@ -7,7 +7,10 @@ use serde::Serialize;
|
|||
use tracing::info;
|
||||
|
||||
use crate::consts::{NAN_U16, QUANT_SCALE};
|
||||
use crate::data::{PropertyData, QuantRef};
|
||||
use crate::data::{
|
||||
combine_nearest_distances, combined_station_feature_name,
|
||||
combined_station_source_feature_names, PropertyData, QuantRef,
|
||||
};
|
||||
use crate::utils::{normalize_postcode, GridIndex, InternedColumn};
|
||||
|
||||
const GRID_CELL_SIZE: f32 = 0.01;
|
||||
|
|
@ -318,8 +321,17 @@ fn build_poi_filter_feature_data(
|
|||
let mut encoded_columns = 0usize;
|
||||
|
||||
for (metric_idx, name) in poi_metrics.feature_names.iter().enumerate() {
|
||||
let Some(values) = extract_optional_feature_f32(df, name)? else {
|
||||
continue;
|
||||
let values = match extract_optional_feature_f32(df, name)? {
|
||||
Some(values) => values,
|
||||
// Some POI columns (the combined-station distance) are synthesized
|
||||
// in memory by PostcodePoiMetrics and never written to the listings
|
||||
// parquet. Reconstruct them from their source columns so listing POI
|
||||
// filters match the map exactly instead of silently rejecting every
|
||||
// row on an all-NaN column.
|
||||
None => match synthesize_missing_poi_column(df, name, row_count)? {
|
||||
Some(values) => values,
|
||||
None => continue,
|
||||
},
|
||||
};
|
||||
for (row, value) in values.into_iter().enumerate() {
|
||||
let dst = row * num_features + metric_idx;
|
||||
|
|
@ -338,6 +350,45 @@ fn build_poi_filter_feature_data(
|
|||
Ok(feature_data)
|
||||
}
|
||||
|
||||
/// Reconstruct a POI metric column that `PostcodePoiMetrics` synthesizes in
|
||||
/// memory and therefore is absent from the listings parquet. Currently only the
|
||||
/// combined-station distance, derived as the elementwise nearest of its per-mode
|
||||
/// source columns (the same rule the postcode side table uses). Returns `None`
|
||||
/// if `name` is not a synthesized column, or if none of its sources are present.
|
||||
fn synthesize_missing_poi_column(
|
||||
df: &DataFrame,
|
||||
name: &str,
|
||||
row_count: usize,
|
||||
) -> Result<Option<Vec<Option<f32>>>> {
|
||||
if name != combined_station_feature_name() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let mut sources: Vec<Vec<f32>> = Vec::new();
|
||||
for source_name in combined_station_source_feature_names() {
|
||||
if let Some(values) = extract_optional_feature_f32(df, &source_name)? {
|
||||
sources.push(
|
||||
values
|
||||
.into_iter()
|
||||
.map(|value| value.unwrap_or(f32::NAN))
|
||||
.collect(),
|
||||
);
|
||||
}
|
||||
}
|
||||
if sources.is_empty() {
|
||||
return Ok(None);
|
||||
}
|
||||
|
||||
let source_refs: Vec<&[f32]> = sources.iter().map(Vec::as_slice).collect();
|
||||
let combined = combine_nearest_distances(&source_refs, row_count);
|
||||
Ok(Some(
|
||||
combined
|
||||
.into_iter()
|
||||
.map(|value| value.is_finite().then_some(value))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn feature_index(property_data: &PropertyData, name: &str) -> Option<usize> {
|
||||
property_data
|
||||
.feature_names
|
||||
|
|
@ -741,6 +792,51 @@ mod tests {
|
|||
assert!(!any_listing.listing_url.is_empty());
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn synthesizes_combined_station_column_from_source_columns() {
|
||||
// Mirrors the postcode side table: the combined-station distance is the
|
||||
// finite minimum of the per-mode source columns present in the parquet.
|
||||
let df = df![
|
||||
"Distance to nearest amenity (Rail station) (km)" => [2.0f32, f32::NAN, 5.0],
|
||||
"Distance to nearest amenity (Tube station) (km)" => [1.0f32, f32::NAN, f32::NAN],
|
||||
"Distance to nearest amenity (DLR station) (km)" => [3.0f32, 4.0, f32::NAN],
|
||||
]
|
||||
.unwrap();
|
||||
|
||||
let combined = synthesize_missing_poi_column(&df, &combined_station_feature_name(), 3)
|
||||
.unwrap()
|
||||
.expect("combined-station column should be synthesized");
|
||||
|
||||
assert_eq!(combined[0], Some(1.0)); // min(2, 1, 3)
|
||||
assert_eq!(combined[1], Some(4.0)); // only DLR present
|
||||
assert_eq!(combined[2], Some(5.0)); // only Rail present
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn does_not_synthesize_non_combined_or_sourceless_columns() {
|
||||
let df = df![
|
||||
"Distance to nearest amenity (Rail station) (km)" => [1.0f32],
|
||||
]
|
||||
.unwrap();
|
||||
|
||||
// A real per-mode column is read from the parquet, never synthesized.
|
||||
assert!(synthesize_missing_poi_column(
|
||||
&df,
|
||||
"Distance to nearest amenity (Rail station) (km)",
|
||||
1
|
||||
)
|
||||
.unwrap()
|
||||
.is_none());
|
||||
|
||||
// The combined column cannot be built when no source columns exist.
|
||||
let empty = df!["Postcode" => ["AB1 2CD"]].unwrap();
|
||||
assert!(
|
||||
synthesize_missing_poi_column(&empty, &combined_station_feature_name(), 1)
|
||||
.unwrap()
|
||||
.is_none()
|
||||
);
|
||||
}
|
||||
|
||||
#[test]
|
||||
fn extracts_price_history_from_parquet() {
|
||||
let path = PathBuf::from("src/data/testdata/listings_price_history.parquet");
|
||||
|
|
|
|||
|
|
@ -18,7 +18,10 @@ mod quant;
|
|||
mod stats;
|
||||
|
||||
pub use h3::precompute_h3;
|
||||
pub use poi_metrics::{PostcodePoiMetrics, COMBINED_STATION_CATEGORY};
|
||||
pub use poi_metrics::{
|
||||
combine_nearest_distances, combined_station_feature_name, combined_station_source_feature_names,
|
||||
PostcodePoiMetrics, COMBINED_STATION_CATEGORY,
|
||||
};
|
||||
pub use quant::QuantRef;
|
||||
pub use stats::{FeatureStats, Histogram};
|
||||
|
||||
|
|
|
|||
|
|
@ -1445,6 +1445,87 @@ pub async fn ensure_oauth_providers(
|
|||
Ok(())
|
||||
}
|
||||
|
||||
/// Point the password-reset email at the SPA's own `/reset-password` page.
|
||||
///
|
||||
/// PocketBase's default reset template links to `{APP_URL}/_/#/auth/confirm-password-reset/{TOKEN}`,
|
||||
/// i.e. its superuser UI under `/pb/_/`. `meta.appURL` is `{public_url}/pb` (set in
|
||||
/// `ensure_oauth_providers` for OAuth redirects), and the `/pb` proxy allowlist deliberately
|
||||
/// rejects `/_/` (see `routes/pb_proxy.rs`), so the emailed link 404s. We therefore bake the raw
|
||||
/// `public_url` (NOT `appURL`, which carries the `/pb` prefix) straight into the template body and
|
||||
/// keep PocketBase's `{TOKEN}` placeholder for it to substitute at send time. The SPA route reads
|
||||
/// `?token=` and calls `confirmPasswordReset`.
|
||||
pub async fn ensure_password_reset_template(
|
||||
client: &Client,
|
||||
base_url: &str,
|
||||
admin_email: &str,
|
||||
admin_password: &str,
|
||||
public_url: &str,
|
||||
) -> anyhow::Result<()> {
|
||||
let base_url = base_url.trim_end_matches('/');
|
||||
let token = auth_superuser(client, base_url, admin_email, admin_password).await?;
|
||||
|
||||
let site = public_url.trim_end_matches('/');
|
||||
// `{{TOKEN}}` renders to the literal `{TOKEN}` placeholder that PocketBase fills in.
|
||||
let action_url = format!("{site}/reset-password?token={{TOKEN}}");
|
||||
let body = format!(
|
||||
"<p>Hello,</p>\n\
|
||||
<p>Click the button below to choose a new password for your Perfect Postcode account.</p>\n\
|
||||
<p><a class=\"btn\" href=\"{action_url}\" target=\"_blank\" rel=\"noopener\">Reset password</a></p>\n\
|
||||
<p>If you did not request this, you can safely ignore this email.</p>\n\
|
||||
<p>Thanks,<br/>The Perfect Postcode team</p>"
|
||||
);
|
||||
|
||||
// PocketBase 0.23+: email templates are per-collection. GET the users collection, set the
|
||||
// reset template, and PATCH it back (mirrors the OAuth config flow above).
|
||||
let collection_url = format!("{base_url}/api/collections/users");
|
||||
let resp = client
|
||||
.get(&collection_url)
|
||||
.header("Authorization", format!("Bearer {token}"))
|
||||
.send()
|
||||
.await?;
|
||||
if !resp.status().is_success() {
|
||||
let status = resp.status();
|
||||
let text = resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("Failed to fetch users collection for reset template ({status}): {text}");
|
||||
}
|
||||
let mut collection: serde_json::Value = resp.json().await?;
|
||||
|
||||
// The field is an object `{subject, body}`; preserve any sibling keys PocketBase may add. If a
|
||||
// future PocketBase version renames or moves it, warn and skip rather than blocking server
|
||||
// startup over an email template: a broken reset link is far less bad than a server that won't boot.
|
||||
let Some(template) = collection
|
||||
.get_mut("resetPasswordTemplate")
|
||||
.and_then(|v| v.as_object_mut())
|
||||
else {
|
||||
warn!(
|
||||
"users collection has no resetPasswordTemplate object; \
|
||||
leaving the reset email on PocketBase's default (which 404s via /pb/_/)"
|
||||
);
|
||||
return Ok(());
|
||||
};
|
||||
template.insert(
|
||||
"subject".to_string(),
|
||||
serde_json::json!("Reset your Perfect Postcode password"),
|
||||
);
|
||||
template.insert("body".to_string(), serde_json::json!(body));
|
||||
let reset_template = collection["resetPasswordTemplate"].clone();
|
||||
|
||||
let patch_resp = client
|
||||
.patch(&collection_url)
|
||||
.header("Authorization", format!("Bearer {token}"))
|
||||
.json(&serde_json::json!({ "resetPasswordTemplate": reset_template }))
|
||||
.send()
|
||||
.await?;
|
||||
if !patch_resp.status().is_success() {
|
||||
let status = patch_resp.status();
|
||||
let text = patch_resp.text().await.unwrap_or_default();
|
||||
anyhow::bail!("Failed to set password-reset template ({status}): {text}");
|
||||
}
|
||||
|
||||
info!("PocketBase password-reset email template set to {action_url}");
|
||||
Ok(())
|
||||
}
|
||||
|
||||
/// Spawn a background task that polls PocketBase every 60 seconds for collection counts
|
||||
/// and exposes them as Prometheus gauges.
|
||||
pub fn start_metrics_poller(shared: Arc<crate::state::SharedState>) {
|
||||
|
|
|
|||
|
|
@ -166,6 +166,7 @@ impl AppState {
|
|||
listing_status: InternedColumn::build(&[]),
|
||||
listing_date_iso: Vec::new(),
|
||||
features: Vec::new(),
|
||||
price_history: Vec::new(),
|
||||
filter_feature_data: Vec::new(),
|
||||
poi_filter_feature_data: Vec::new(),
|
||||
grid: GridIndex::build(&[], &[], 0.01),
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue