Refactor and other improvements
This commit is contained in:
parent
04a78e7bfe
commit
6c90cf3c0f
47 changed files with 2705 additions and 1568 deletions
|
|
@ -8,10 +8,14 @@ use axum::response::Json;
|
|||
use serde::{Deserialize, Serialize};
|
||||
use tracing::{info, warn};
|
||||
|
||||
use crate::consts::{H3_PRECOMPUTE_MAX, H3_REQUEST_MAX, H3_REQUEST_MIN};
|
||||
use crate::parsing::{h3_cell_bounds, parse_filters, row_passes_filters};
|
||||
use crate::parsing::{
|
||||
cell_for_row, h3_cell_bounds, needs_parent, parse_field_set, parse_filters, row_passes_filters,
|
||||
validate_h3_resolution,
|
||||
};
|
||||
use crate::state::AppState;
|
||||
|
||||
use super::stats;
|
||||
|
||||
#[derive(Serialize)]
|
||||
pub struct HistogramStats {
|
||||
pub min: f64,
|
||||
|
|
@ -78,19 +82,8 @@ pub async fn get_hexagon_stats(
|
|||
let cell_u64: u64 = cell.into();
|
||||
|
||||
let resolution = params.resolution;
|
||||
if !(H3_REQUEST_MIN..=H3_REQUEST_MAX).contains(&resolution) {
|
||||
warn!(
|
||||
resolution,
|
||||
"Resolution out of range [{}, {}]", H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
);
|
||||
return Err((
|
||||
StatusCode::BAD_REQUEST,
|
||||
format!(
|
||||
"resolution must be between {} and {}",
|
||||
H3_REQUEST_MIN, H3_REQUEST_MAX
|
||||
),
|
||||
));
|
||||
}
|
||||
validate_h3_resolution(resolution)?;
|
||||
|
||||
let h3_str = params.h3.clone();
|
||||
let filters_str = params.filters.clone();
|
||||
let (parsed_filters, parsed_enum_filters) = parse_filters(
|
||||
|
|
@ -100,48 +93,25 @@ pub async fn get_hexagon_stats(
|
|||
);
|
||||
let num_filters = parsed_filters.len() + parsed_enum_filters.len();
|
||||
|
||||
let fields_specified = params.fields.is_some();
|
||||
let field_set: std::collections::HashSet<String> = params
|
||||
.fields
|
||||
.as_ref()
|
||||
.map(|fields_str| {
|
||||
fields_str
|
||||
.split(',')
|
||||
.map(|field| field.trim().to_string())
|
||||
.filter(|field| !field.is_empty())
|
||||
.collect()
|
||||
})
|
||||
.unwrap_or_default();
|
||||
let (fields_specified, field_set) = parse_field_set(params.fields.as_deref());
|
||||
|
||||
let response = tokio::task::spawn_blocking(move || {
|
||||
let start_time = std::time::Instant::now();
|
||||
let precomputed = &state.h3_cells;
|
||||
let h3_res = h3o::Resolution::try_from(resolution)
|
||||
.map_err(|err| format!("Invalid H3 resolution {}: {}", resolution, err))?;
|
||||
let need_parent = resolution < H3_PRECOMPUTE_MAX;
|
||||
let need_parent = needs_parent(resolution);
|
||||
let num_features = state.data.num_features;
|
||||
let feature_data = &state.data.feature_data;
|
||||
|
||||
let (min_lat, min_lon, max_lat, max_lon) = h3_cell_bounds(cell, 0.001);
|
||||
|
||||
let cell_for_row = |row: usize| -> u64 {
|
||||
let max_cell = precomputed[row];
|
||||
if !need_parent || max_cell == 0 {
|
||||
return max_cell;
|
||||
}
|
||||
h3o::CellIndex::try_from(max_cell)
|
||||
.ok()
|
||||
.and_then(|ci| ci.parent(h3_res))
|
||||
.map(u64::from)
|
||||
.unwrap_or(0)
|
||||
};
|
||||
|
||||
let mut matching_rows: Vec<usize> = Vec::new();
|
||||
state
|
||||
.grid
|
||||
.for_each_in_bounds(min_lat, min_lon, max_lat, max_lon, |row_idx| {
|
||||
let row = row_idx as usize;
|
||||
if cell_for_row(row) == cell_u64
|
||||
if cell_for_row(row, precomputed, h3_res, need_parent) == cell_u64
|
||||
&& row_passes_filters(
|
||||
row,
|
||||
&parsed_filters,
|
||||
|
|
@ -156,149 +126,23 @@ pub async fn get_hexagon_stats(
|
|||
|
||||
let total_count = matching_rows.len();
|
||||
|
||||
// Collect price history (year, price) pairs
|
||||
let price_history = {
|
||||
let year_idx = state
|
||||
.feature_name_to_index
|
||||
.get("Date of last transaction")
|
||||
.copied();
|
||||
let price_idx = state.feature_name_to_index.get("Last known price").copied();
|
||||
match (year_idx, price_idx) {
|
||||
(Some(yi), Some(pi)) => {
|
||||
let mut points: Vec<PricePoint> = matching_rows
|
||||
.iter()
|
||||
.filter_map(|&row| {
|
||||
let year = feature_data[row * num_features + yi];
|
||||
let price = feature_data[row * num_features + pi];
|
||||
if year.is_finite() && price.is_finite() {
|
||||
Some(PricePoint { year, price })
|
||||
} else {
|
||||
None
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
// Cap at 5000 points by evenly sampling
|
||||
if points.len() > 5000 {
|
||||
let step = points.len() as f64 / 5000.0;
|
||||
points = (0..5000)
|
||||
.map(|i| {
|
||||
let idx = (i as f64 * step) as usize;
|
||||
PricePoint {
|
||||
year: points[idx].year,
|
||||
price: points[idx].price,
|
||||
}
|
||||
})
|
||||
.collect();
|
||||
}
|
||||
points
|
||||
}
|
||||
_ => Vec::new(),
|
||||
}
|
||||
};
|
||||
let price_history = stats::extract_price_history(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
num_features,
|
||||
&state.feature_name_to_index,
|
||||
);
|
||||
|
||||
let mut numeric_features = Vec::new();
|
||||
let mut enum_features_out = Vec::new();
|
||||
|
||||
for (feature_index, feature_name) in state.data.feature_names.iter().enumerate() {
|
||||
if fields_specified && !field_set.contains(feature_name.as_str()) {
|
||||
continue;
|
||||
}
|
||||
|
||||
// Check if this is an enum feature
|
||||
if let Some(enum_values) = state.data.enum_values.get(&feature_index) {
|
||||
// Enum feature: count occurrences of each value
|
||||
let mut value_counts = vec![0u64; enum_values.len()];
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
let idx = value as usize;
|
||||
if idx < value_counts.len() {
|
||||
value_counts[idx] += 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
let counts: HashMap<String, u64> = value_counts
|
||||
.iter()
|
||||
.enumerate()
|
||||
.filter(|(_, &count)| count > 0)
|
||||
.map(|(idx, &count)| (enum_values[idx].clone(), count))
|
||||
.collect();
|
||||
|
||||
if !counts.is_empty() {
|
||||
enum_features_out.push(EnumFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
counts,
|
||||
});
|
||||
}
|
||||
} else {
|
||||
// Numeric feature: compute stats and histogram
|
||||
let global_hist = &state.data.feature_stats[feature_index].histogram;
|
||||
let p1 = global_hist.p1;
|
||||
let p99 = global_hist.p99;
|
||||
// Use same bin count as global histogram for consistency
|
||||
let num_bins = global_hist.counts.len();
|
||||
|
||||
let mut count = 0usize;
|
||||
let mut min_value = f32::INFINITY;
|
||||
let mut max_value = f32::NEG_INFINITY;
|
||||
let mut sum = 0.0f64;
|
||||
let mut bins = vec![0u64; num_bins];
|
||||
|
||||
// Compute middle bin width (between p1 and p99)
|
||||
let middle_bins = num_bins.saturating_sub(2);
|
||||
let middle_width = if middle_bins > 0 && p99 > p1 {
|
||||
(p99 - p1) / middle_bins as f32
|
||||
} else {
|
||||
0.0
|
||||
};
|
||||
|
||||
for &row in &matching_rows {
|
||||
let value = feature_data[row * num_features + feature_index];
|
||||
if value.is_finite() {
|
||||
count += 1;
|
||||
if value < min_value {
|
||||
min_value = value;
|
||||
}
|
||||
if value > max_value {
|
||||
max_value = value;
|
||||
}
|
||||
sum += value as f64;
|
||||
|
||||
// Bin using p1/p99 outlier structure
|
||||
let bin = if value < p1 {
|
||||
0 // Low outlier bin
|
||||
} else if value >= p99 {
|
||||
num_bins - 1 // High outlier bin
|
||||
} else if middle_width > 0.0 {
|
||||
// Middle bins (1 to n-2)
|
||||
let middle_bin = ((value - p1) / middle_width) as usize;
|
||||
(1 + middle_bin).min(num_bins - 2)
|
||||
} else {
|
||||
num_bins / 2 // Fallback if p1 == p99
|
||||
};
|
||||
bins[bin] += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if count > 0 {
|
||||
numeric_features.push(NumericFeatureStats {
|
||||
name: feature_name.clone(),
|
||||
count,
|
||||
min: min_value as f64,
|
||||
max: max_value as f64,
|
||||
mean: sum / count as f64,
|
||||
histogram: HistogramStats {
|
||||
min: global_hist.min as f64,
|
||||
max: global_hist.max as f64,
|
||||
p1: p1 as f64,
|
||||
p99: p99 as f64,
|
||||
counts: bins,
|
||||
},
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
let (numeric_features, enum_features_out) = stats::compute_feature_stats(
|
||||
&matching_rows,
|
||||
feature_data,
|
||||
&state.data.feature_names,
|
||||
num_features,
|
||||
&state.data.enum_values,
|
||||
&state.data.feature_stats,
|
||||
fields_specified,
|
||||
&field_set,
|
||||
);
|
||||
|
||||
let elapsed = start_time.elapsed();
|
||||
info!(
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue