has issues
This commit is contained in:
parent
2e112d7398
commit
c645b0f1d4
96 changed files with 2147083 additions and 5787 deletions
|
|
@ -131,6 +131,51 @@ pub fn resolve_poi_category_filter(category_values: &[String], categories: &str)
|
|||
selected
|
||||
}
|
||||
|
||||
/// Metadata for state-funded school POIs (sourced from the DfE GIAS register).
|
||||
/// Every field is optional because GIAS does not populate every column for every
|
||||
/// establishment type (e.g. nurseries have no sixth form, FE colleges no FSM).
|
||||
#[derive(Serialize, Clone, Default)]
|
||||
pub struct SchoolMetadata {
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub phase: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub r#type: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub type_group: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub age_range: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub gender: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub religious_character: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub admissions_policy: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub nursery_provision: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub sixth_form: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub capacity: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub pupils: Option<u32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub fsm_percent: Option<f32>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub trust: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub address: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub postcode: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub local_authority: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub website: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub telephone: Option<String>,
|
||||
#[serde(skip_serializing_if = "Option::is_none")]
|
||||
pub head_name: Option<String>,
|
||||
}
|
||||
|
||||
pub struct POIData {
|
||||
/// Contiguous buffer holding all POI ID strings end-to-end.
|
||||
id_buffer: String,
|
||||
|
|
@ -149,6 +194,11 @@ pub struct POIData {
|
|||
/// uniform subset when the POI count exceeds the per-request limit.
|
||||
/// Computed once at load time so the same POIs are always chosen for a given viewport.
|
||||
pub priority: Vec<u32>,
|
||||
/// Indirection table: row idx → index into `school_meta`, or u32::MAX when
|
||||
/// the POI is not a school. Keeps the per-row overhead at 4 bytes regardless
|
||||
/// of how many school metadata fields we carry.
|
||||
school_meta_idx: Vec<u32>,
|
||||
school_meta: Vec<SchoolMetadata>,
|
||||
}
|
||||
|
||||
impl POIData {
|
||||
|
|
@ -158,6 +208,16 @@ impl POIData {
|
|||
let length = self.id_lengths[row] as usize;
|
||||
&self.id_buffer[offset..offset + length]
|
||||
}
|
||||
|
||||
/// Get the school metadata for a given row, or None if not a school.
|
||||
pub fn school(&self, row: usize) -> Option<&SchoolMetadata> {
|
||||
let idx = self.school_meta_idx[row];
|
||||
if idx == u32::MAX {
|
||||
None
|
||||
} else {
|
||||
Some(&self.school_meta[idx as usize])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
fn extract_str_col(df: &DataFrame, name: &str) -> anyhow::Result<Vec<String>> {
|
||||
|
|
@ -195,6 +255,146 @@ fn extract_f32_col(df: &DataFrame, name: &str) -> anyhow::Result<Vec<f32>> {
|
|||
.collect()
|
||||
}
|
||||
|
||||
/// Read an optional string column. Returns None when the column itself is missing
|
||||
/// (older POI parquets without the school_* extension); returns Some(vec) of
|
||||
/// length row_count where each entry is None for null cells.
|
||||
fn extract_optional_str_col(
|
||||
df: &DataFrame,
|
||||
name: &str,
|
||||
) -> anyhow::Result<Option<Vec<Option<String>>>> {
|
||||
let column = match df.column(name) {
|
||||
Ok(column) => column,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
let string_column = column
|
||||
.str()
|
||||
.with_context(|| format!("Column '{name}' is not a string column"))?;
|
||||
Ok(Some(
|
||||
string_column
|
||||
.into_iter()
|
||||
.map(|value| value.map(ToString::to_string))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn extract_optional_u32_col(
|
||||
df: &DataFrame,
|
||||
name: &str,
|
||||
) -> anyhow::Result<Option<Vec<Option<u32>>>> {
|
||||
let column = match df.column(name) {
|
||||
Ok(column) => column,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
let cast = column
|
||||
.cast(&DataType::Int64)
|
||||
.with_context(|| format!("Failed to cast column '{name}' to Int64"))?;
|
||||
let int_column = cast
|
||||
.i64()
|
||||
.with_context(|| format!("Column '{name}' is not an integer column"))?;
|
||||
Ok(Some(
|
||||
int_column
|
||||
.into_iter()
|
||||
.map(|value| value.and_then(|v| if v < 0 { None } else { Some(v as u32) }))
|
||||
.collect(),
|
||||
))
|
||||
}
|
||||
|
||||
fn extract_optional_f32_col(
|
||||
df: &DataFrame,
|
||||
name: &str,
|
||||
) -> anyhow::Result<Option<Vec<Option<f32>>>> {
|
||||
let column = match df.column(name) {
|
||||
Ok(column) => column,
|
||||
Err(_) => return Ok(None),
|
||||
};
|
||||
let cast = column
|
||||
.cast(&DataType::Float32)
|
||||
.with_context(|| format!("Failed to cast column '{name}' to Float32"))?;
|
||||
let float_column = cast
|
||||
.f32()
|
||||
.with_context(|| format!("Column '{name}' is not a float32 column"))?;
|
||||
Ok(Some(float_column.into_iter().collect()))
|
||||
}
|
||||
|
||||
fn build_school_meta(
|
||||
row_count: usize,
|
||||
df: &DataFrame,
|
||||
) -> anyhow::Result<(Vec<u32>, Vec<SchoolMetadata>)> {
|
||||
let phase = extract_optional_str_col(df, "school_phase")?;
|
||||
if phase.is_none() {
|
||||
// POI parquet predates the school metadata extension — record an empty
|
||||
// table and a sentinel-filled index, so callers transparently see None.
|
||||
return Ok((vec![u32::MAX; row_count], Vec::new()));
|
||||
}
|
||||
|
||||
let phase = phase.unwrap();
|
||||
let r#type = extract_optional_str_col(df, "school_type")?.unwrap_or_default();
|
||||
let type_group = extract_optional_str_col(df, "school_type_group")?.unwrap_or_default();
|
||||
let age_range = extract_optional_str_col(df, "school_age_range")?.unwrap_or_default();
|
||||
let gender = extract_optional_str_col(df, "school_gender")?.unwrap_or_default();
|
||||
let religious_character =
|
||||
extract_optional_str_col(df, "school_religious_character")?.unwrap_or_default();
|
||||
let admissions_policy =
|
||||
extract_optional_str_col(df, "school_admissions_policy")?.unwrap_or_default();
|
||||
let nursery_provision =
|
||||
extract_optional_str_col(df, "school_nursery_provision")?.unwrap_or_default();
|
||||
let sixth_form = extract_optional_str_col(df, "school_sixth_form")?.unwrap_or_default();
|
||||
let capacity = extract_optional_u32_col(df, "school_capacity")?.unwrap_or_default();
|
||||
let pupils = extract_optional_u32_col(df, "school_pupils")?.unwrap_or_default();
|
||||
let fsm_percent = extract_optional_f32_col(df, "school_fsm_percent")?.unwrap_or_default();
|
||||
let trust = extract_optional_str_col(df, "school_trust")?.unwrap_or_default();
|
||||
let address = extract_optional_str_col(df, "school_address")?.unwrap_or_default();
|
||||
let postcode = extract_optional_str_col(df, "school_postcode")?.unwrap_or_default();
|
||||
let local_authority =
|
||||
extract_optional_str_col(df, "school_local_authority")?.unwrap_or_default();
|
||||
let website = extract_optional_str_col(df, "school_website")?.unwrap_or_default();
|
||||
let telephone = extract_optional_str_col(df, "school_telephone")?.unwrap_or_default();
|
||||
let head_name = extract_optional_str_col(df, "school_head_name")?.unwrap_or_default();
|
||||
|
||||
let fetch_str = |col: &Vec<Option<String>>, row: usize| -> Option<String> {
|
||||
col.get(row).cloned().flatten()
|
||||
};
|
||||
let fetch_u32 =
|
||||
|col: &Vec<Option<u32>>, row: usize| -> Option<u32> { col.get(row).copied().flatten() };
|
||||
let fetch_f32 =
|
||||
|col: &Vec<Option<f32>>, row: usize| -> Option<f32> { col.get(row).copied().flatten() };
|
||||
|
||||
let mut idx = vec![u32::MAX; row_count];
|
||||
let mut meta = Vec::new();
|
||||
for row in 0..row_count {
|
||||
let type_group_val = fetch_str(&type_group, row);
|
||||
let type_val = fetch_str(&r#type, row);
|
||||
// type_group is present for every GIAS row, so use it as the sentinel
|
||||
// for "this POI is a school" — matches the pipeline guarantee.
|
||||
if type_group_val.is_none() && type_val.is_none() {
|
||||
continue;
|
||||
}
|
||||
idx[row] = meta.len() as u32;
|
||||
meta.push(SchoolMetadata {
|
||||
phase: fetch_str(&phase, row),
|
||||
r#type: type_val,
|
||||
type_group: type_group_val,
|
||||
age_range: fetch_str(&age_range, row),
|
||||
gender: fetch_str(&gender, row),
|
||||
religious_character: fetch_str(&religious_character, row),
|
||||
admissions_policy: fetch_str(&admissions_policy, row),
|
||||
nursery_provision: fetch_str(&nursery_provision, row),
|
||||
sixth_form: fetch_str(&sixth_form, row),
|
||||
capacity: fetch_u32(&capacity, row),
|
||||
pupils: fetch_u32(&pupils, row),
|
||||
fsm_percent: fetch_f32(&fsm_percent, row),
|
||||
trust: fetch_str(&trust, row),
|
||||
address: fetch_str(&address, row),
|
||||
postcode: fetch_str(&postcode, row),
|
||||
local_authority: fetch_str(&local_authority, row),
|
||||
website: fetch_str(&website, row),
|
||||
telephone: fetch_str(&telephone, row),
|
||||
head_name: fetch_str(&head_name, row),
|
||||
});
|
||||
}
|
||||
Ok((idx, meta))
|
||||
}
|
||||
|
||||
impl POIData {
|
||||
pub fn load(parquet_path: &Path) -> anyhow::Result<Self> {
|
||||
super::run_polars_io(|| Self::load_inner(parquet_path))
|
||||
|
|
@ -259,6 +459,9 @@ impl POIData {
|
|||
// preventing visual "shuffling" when panning the map.
|
||||
let priority = generate_priorities(row_count);
|
||||
|
||||
let (school_meta_idx, school_meta) = build_school_meta(row_count, &df)?;
|
||||
info!(schools = school_meta.len(), "Loaded GIAS school metadata");
|
||||
|
||||
info!("POI data loading complete.");
|
||||
|
||||
Ok(POIData {
|
||||
|
|
@ -273,6 +476,8 @@ impl POIData {
|
|||
lng,
|
||||
emoji,
|
||||
priority,
|
||||
school_meta_idx,
|
||||
school_meta,
|
||||
})
|
||||
}
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue