#![allow(clippy::min_ident_chars)]

mod aggregation;
mod api_error;
mod auth;
mod bugsink;
mod checkout_sessions;
mod consts;
mod data;
mod features;
mod generated_data_pages;
mod language;
mod metrics;
mod og_middleware;
pub mod parsing;
mod pocketbase;
mod pocketbase_locks;
mod ratelimit;
mod routes;
mod state;
pub mod utils;

// Use jemalloc as the global allocator. glibc malloc keeps freed memory in many
// per-CPU arenas and rarely returns it to the OS, so the transient buffers from
// the rayon/Polars-parallel data load stay resident and inflate steady-state RSS.
// jemalloc's decay-based purging (configured below) hands those pages back.
#[cfg(not(target_env = "msvc"))]
#[global_allocator]
static GLOBAL: tikv_jemallocator::Jemalloc = tikv_jemallocator::Jemalloc;

// Return dirty/muzzy pages to the OS ~1s after they go idle, and run a background
// thread to do so proactively (so RSS drops after the startup load peak without
// waiting for the next allocation). Read by jemalloc at startup via the
// `malloc_conf` symbol (unprefixed on Linux). Can be overridden by the
// `_RJEM_MALLOC_CONF` / `MALLOC_CONF` env var.
#[cfg(not(target_env = "msvc"))]
#[allow(non_upper_case_globals)]
#[used]
#[export_name = "malloc_conf"]
pub static malloc_conf: &[u8] = b"background_thread:true,dirty_decay_ms:1000,muzzy_decay_ms:1000\0";

use std::path::{Path, PathBuf};
use std::sync::Arc;
use std::time::Duration;

use anyhow::{bail, Context};
use axum::http::{header, HeaderValue};
use axum::middleware;
use axum::routing::{any, get, patch, post};
use axum::Router;
use clap::Parser;
use consts::SERVICE_CALL_TIMEOUT;
use tower::limit::ConcurrencyLimitLayer;
use tower::ServiceBuilder;
use tower_http::compression::CompressionLayer;

use tower_http::cors::{AllowHeaders, AllowMethods, CorsLayer};
use tower_http::services::{ServeDir, ServeFile};
use tower_http::trace::TraceLayer;
use tracing::info;
use tracing_subscriber::layer::SubscriberExt;
use tracing_subscriber::util::SubscriberInitExt;
use tracing_subscriber::EnvFilter;

use state::{AppState, SharedState};

fn is_api_path(path: &str) -> bool {
    path.starts_with("/api/")
        || path.starts_with("/pb/")
        || path.starts_with("/s/")
        || matches!(path, "/health" | "/metrics")
}

fn is_fingerprinted_asset(path: &str) -> bool {
    let Some(filename) = path.rsplit('/').next() else {
        return false;
    };
    let Some((stem, extension)) = filename.rsplit_once('.') else {
        return false;
    };
    if !matches!(extension, "css" | "js") {
        return false;
    }
    let Some((_, hash)) = stem.rsplit_once('.') else {
        return false;
    };
    hash.len() >= 8 && hash.bytes().all(|byte| byte.is_ascii_hexdigit())
}

fn is_static_asset_path(path: &str) -> bool {
    path.rsplit('/')
        .next()
        .is_some_and(|segment| segment.contains('.'))
}

async fn static_cache_headers(
    request: axum::extract::Request,
    next: middleware::Next,
) -> axum::response::Response {
    let path = request.uri().path().to_string();
    let mut response = next.run(request).await;

    if is_api_path(&path) || response.headers().contains_key(header::CACHE_CONTROL) {
        return response;
    }

    let cache_control = response
        .headers()
        .get(header::CONTENT_TYPE)
        .and_then(|value| value.to_str().ok())
        .filter(|content_type| content_type.contains("text/html"))
        .map(|_| HeaderValue::from_static("no-cache, must-revalidate"))
        .or_else(|| {
            is_fingerprinted_asset(&path)
                .then(|| HeaderValue::from_static("public, max-age=31536000, immutable"))
        })
        .or_else(|| {
            is_static_asset_path(&path).then(|| HeaderValue::from_static("public, max-age=3600"))
        });

    if let Some(value) = cache_control {
        response.headers_mut().insert(header::CACHE_CONTROL, value);
    }

    response
}


async fn security_headers(
    request: axum::extract::Request,
    next: middleware::Next,
) -> axum::response::Response {
    let mut response = next.run(request).await;
    let headers = response.headers_mut();
    headers
        .entry(header::X_CONTENT_TYPE_OPTIONS)
        .or_insert(HeaderValue::from_static("nosniff"));
    headers
        .entry(header::X_FRAME_OPTIONS)
        .or_insert(HeaderValue::from_static("SAMEORIGIN"));
    headers
        .entry(header::REFERRER_POLICY)
        .or_insert(HeaderValue::from_static("strict-origin-when-cross-origin"));
    response
}

#[cfg(target_os = "linux")]
fn resident_memory_kib() -> Option<u64> {
    let status = std::fs::read_to_string("/proc/self/status").ok()?;
    status.lines().find_map(|line| {
        line.strip_prefix("VmRSS:")?
            .split_whitespace()
            .next()?
            .parse()
            .ok()
    })
}

/// Force jemalloc to return all dirty/muzzy pages to the OS immediately.
///
/// jemalloc keeps freed pages "dirty" for fast reuse and only hands them back to
/// the OS via decay. Under the bursty allocate/free of request handling (and the
/// huge startup-load transients) that decay lags badly, so RSS balloons far above
/// the live working set. `arena.4096.purge` (4096 == `MALLCTL_ARENAS_ALL`) purges
/// every arena synchronously, dropping RSS back down.
#[cfg(target_os = "linux")]
fn jemalloc_purge() {
    const PURGE: &[u8] = b"arena.4096.purge\0";
    // Safety: a write-only mallctl with no input/output buffers; the name is a
    // valid NUL-terminated jemalloc control string.
    unsafe {
        tikv_jemalloc_sys::mallctl(
            PURGE.as_ptr().cast(),
            std::ptr::null_mut(),
            std::ptr::null_mut(),
            std::ptr::null_mut(),
            0,
        );
    }
}

#[cfg(not(target_os = "linux"))]
fn jemalloc_purge() {}

/// Enable jemalloc's background purge thread at runtime. Setting it via
/// `background_thread:true` in `malloc_conf` is unreliable (it can be silently
/// dropped depending on init ordering), so we also flip it explicitly here.
#[cfg(target_os = "linux")]
fn enable_jemalloc_background_thread() {
    let enabled: bool = true;
    // Safety: write-only mallctl of a bool to a valid control name.
    unsafe {
        tikv_jemalloc_sys::mallctl(
            c"background_thread".as_ptr().cast(),
            std::ptr::null_mut(),
            std::ptr::null_mut(),
            std::ptr::addr_of!(enabled) as *mut core::ffi::c_void,
            std::mem::size_of::<bool>(),
        );
    }
}

#[cfg(not(target_os = "linux"))]
fn enable_jemalloc_background_thread() {}

/// Periodically purge jemalloc arenas so request-handling transients are returned
/// to the OS instead of accumulating as dirty pages. A plain OS thread (not a
/// tokio task) keeps the madvise sweep off the async runtime.
#[cfg(target_os = "linux")]
fn spawn_jemalloc_purger() {
    std::thread::Builder::new()
        .name("jemalloc-purge".to_string())
        .spawn(|| loop {
            std::thread::sleep(Duration::from_secs(10));
            jemalloc_purge();
        })
        .ok();
}

#[cfg(not(target_os = "linux"))]
fn spawn_jemalloc_purger() {}

#[cfg(target_os = "linux")]
fn trim_allocator(label: &'static str) {
    let before = resident_memory_kib();
    jemalloc_purge();
    let trimmed = unsafe { libc::malloc_trim(0) };
    let after = resident_memory_kib();
    if let (Some(before), Some(after)) = (before, after) {
        info!(
            label,
            trimmed = trimmed != 0,
            rss_before_mib = format_args!("{:.1}", before as f64 / 1024.0),
            rss_after_mib = format_args!("{:.1}", after as f64 / 1024.0),
            released_mib = format_args!("{:.1}", before.saturating_sub(after) as f64 / 1024.0),
            "Allocator trim"
        );
    }
}

#[cfg(not(target_os = "linux"))]
fn trim_allocator(_label: &'static str) {}

#[derive(Parser)]
#[command(
    name = "perfect-postcode",
    about = "Perfect Postcode property map server"
)]
struct Cli {
    /// Path to properties.parquet (one row per historical property)
    #[arg(long)]
    properties: PathBuf,

    /// Path to postcode.parquet (one row per postcode with area-level data)
    #[arg(long)]
    postcode_features: PathBuf,

    /// Path to the POI parquet file
    #[arg(long)]
    pois: PathBuf,

    /// Path to the places parquet file
    #[arg(long)]
    places: PathBuf,

    /// Path to the postcode boundaries directory
    #[arg(long)]
    postcodes: PathBuf,

    /// Path to the PMTiles file for map tiles
    #[arg(long)]
    tiles: PathBuf,

    /// PMTiles raster basemap for satellite imagery.
    #[arg(long, env = "SATELLITE_TILES")]
    satellite_tiles: PathBuf,

    /// PMTiles raster overlay for high-resolution EA aerial photography.
    #[arg(long, env = "SATELLITE_HIGHRES_TILES")]
    satellite_highres_tiles: PathBuf,

    /// PMTiles raster overlay for high-resolution strategic noise.
    #[arg(long, env = "NOISE_OVERLAY_TILES")]
    noise_overlay_tiles: PathBuf,

    /// PMTiles vector overlay for crime heatmap points.
    #[arg(long, env = "CRIME_HOTSPOT_TILES")]
    crime_hotspot_tiles: PathBuf,

    /// PMTiles vector overlay for Trees Outside Woodland polygons.
    #[arg(long, env = "TREE_OVERLAY_TILES")]
    tree_overlay_tiles: PathBuf,

    /// PMTiles vector overlay for INSPIRE property-border polygons.
    #[arg(long, env = "PROPERTY_BORDER_TILES")]
    property_border_tiles: PathBuf,

    /// Path to the frontend dist directory (optional; disables static serving and OG injection when omitted)
    #[arg(long)]
    dist: Option<PathBuf>,

    /// Dev only: spill the large property arrays (feature matrix + address-search
    /// index) to anonymous files in this directory and memory-map them read-only,
    /// instead of holding them on the heap. Trades a little speed for a much
    /// smaller resident set so a low-memory dev box can run the full dataset; the
    /// mapped pages are file-backed and reclaimable under pressure. Omit in prod.
    #[arg(long, env = "SPILL_DIR")]
    spill_dir: Option<PathBuf>,

    /// URL of the screenshot service (e.g. http://screenshot:8002)
    #[arg(long, env = "SCREENSHOT_URL")]
    screenshot_url: String,

    /// Public-facing URL for absolute og:image URLs
    #[arg(long, env = "PUBLIC_URL")]
    public_url: String,

    /// PocketBase server URL for authentication (e.g. http://localhost:8090)
    #[arg(long, env = "POCKETBASE_URL")]
    pocketbase_url: String,

    /// PocketBase superuser email (for auto-creating collections at startup)
    #[arg(long, env = "POCKETBASE_ADMIN_EMAIL")]
    pocketbase_admin_email: String,

    /// PocketBase superuser password (for auto-creating collections at startup)
    #[arg(long, env = "POCKETBASE_ADMIN_PASSWORD")]
    pocketbase_admin_password: String,

    /// Gemini API key
    #[arg(long, env = "GEMINI_API_KEY")]
    gemini_api_key: String,

    /// Gemini model name (e.g. gemini-2.0-flash)
    #[arg(long, env = "GEMINI_MODEL")]
    gemini_model: String,

    /// Path to precomputed travel times directory (contains mode subdirs with parquet files)
    #[arg(long, env = "TRAVEL_TIMES")]
    travel_times: PathBuf,

    /// Path to a parquet of live online listings (Rightmove etc.) to overlay on the map.
    #[arg(long, env = "ACTUAL_LISTINGS_PATH")]
    actual_listings_path: PathBuf,

    /// Path to a parquet of planned/pipeline development sites (MHCLG brownfield
    /// register + Homes England Land Hub) for the "new developments" layer.
    #[arg(long, env = "DEVELOPMENTS_PATH")]
    developments_path: PathBuf,

    /// Path to the per-LSOA per-year crime parquet (display-only side table for the right pane).
    #[arg(long, env = "CRIME_BY_YEAR_PATH")]
    crime_by_year_path: PathBuf,

    /// Path to the per-incident crime-records parquet (last 7 years, postcode-
    /// sorted) backing the "individual crimes" list. Spilled to disk when
    /// `--spill-dir` is set.
    #[arg(long, env = "CRIME_RECORDS_PATH")]
    crime_records_path: PathBuf,

    /// Path to the precomputed national/per-outcode/per-sector crime-averages
    /// parquet (built by pipeline.transform.area_crime_averages). The right pane
    /// uses it to compare a selection's crime rates against its surroundings.
    #[arg(long, env = "AREA_CRIME_AVERAGES_PATH")]
    area_crime_averages_path: PathBuf,

    /// Path to the per-unit-postcode population parquet (ONS Census 2021 usual
    /// residents; display-only side table for the right pane). Optional: when
    /// absent or missing, the area pane simply omits the population figure.
    #[arg(long, env = "POPULATION_PATH")]
    population_path: Option<PathBuf>,

    /// Google Maps API key for Street View metadata lookups
    #[arg(long, env = "GOOGLE_MAPS_API_KEY")]
    google_maps_api_key: String,

    /// Stripe secret key for checkout sessions
    #[arg(long, env = "STRIPE_SECRET_KEY")]
    stripe_secret_key: String,

    /// Stripe webhook signing secret for verifying webhook signatures
    #[arg(long, env = "STRIPE_WEBHOOK_SECRET")]
    stripe_webhook_secret: String,

    /// Stripe Coupon ID applied when a referral code is used
    #[arg(long, env = "STRIPE_REFERRAL_COUPON_ID")]
    stripe_referral_coupon_id: String,

    /// Google OAuth client ID for PocketBase SSO
    #[arg(long, env = "GOOGLE_OAUTH_CLIENT_ID")]
    google_oauth_client_id: String,

    /// Google OAuth client secret for PocketBase SSO
    #[arg(long, env = "GOOGLE_OAUTH_CLIENT_SECRET")]
    google_oauth_client_secret: String,

    /// Bugsink DSN for backend error reporting
    #[arg(long, env = "BUGSINK_DSN", hide_env_values = true)]
    bugsink_dsn: String,

    /// Bugsink DSN injected into the browser app
    #[arg(long, env = "FRONTEND_BUGSINK_DSN", hide_env_values = true)]
    frontend_bugsink_dsn: String,

    /// Bugsink/Sentry environment name
    #[arg(long, env = "BUGSINK_ENVIRONMENT")]
    bugsink_environment: String,

    /// Bugsink/Sentry release name
    #[arg(long, env = "BUGSINK_RELEASE")]
    bugsink_release: String,

    /// Include default PII in Bugsink events
    #[arg(long, env = "BUGSINK_SEND_DEFAULT_PII", action = clap::ArgAction::Set)]
    bugsink_send_default_pii: bool,
}

async fn capture_server_error_responses(
    request: axum::extract::Request,
    next: middleware::Next,
) -> axum::response::Response {
    let method = request.method().clone();
    let path = request.uri().path().to_owned();
    let response = next.run(request).await;
    let status = response.status();

    if status.is_server_error() {
        sentry::with_scope(
            |scope| {
                scope.set_tag("http.status_code", status.as_u16().to_string());
                scope.set_tag("http.method", method.to_string());
                scope.set_tag("http.route", path.clone());
            },
            || {
                sentry::capture_message(
                    &format!("HTTP {status} response from {method} {path}"),
                    sentry::Level::Error,
                );
            },
        );
    }

    response
}

async fn init_required_tile_reader(
    label: &'static str,
    path: &Path,
) -> anyhow::Result<Arc<routes::TileReader>> {
    if !path.exists() {
        bail!("{label} PMTiles not found: {}", path.display());
    }

    info!("Loading {label} PMTiles from {}", path.display());
    Ok(Arc::new(routes::init_tile_reader(path).await?))
}

#[tokio::main]
async fn main() -> anyhow::Result<()> {
    let cli = Cli::parse();
    let _bugsink_guard = bugsink::init_backend(&bugsink::BackendConfig {
        dsn: Some(cli.bugsink_dsn.clone()),
        environment: Some(cli.bugsink_environment.clone()),
        release: Some(cli.bugsink_release.clone()),
        send_default_pii: cli.bugsink_send_default_pii,
    });
    let bugsink_frontend_config = bugsink::frontend_config(
        Some(cli.frontend_bugsink_dsn.clone()),
        Some(cli.bugsink_environment.clone()),
        Some(cli.bugsink_release.clone()),
        cli.bugsink_send_default_pii,
    );

    let file_appender = tracing_appender::rolling::daily("logs", "server.log");
    let (non_blocking, _guard) = tracing_appender::non_blocking(file_appender);

    let env_filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));

    tracing_subscriber::registry()
        .with(env_filter)
        .with(sentry::integrations::tracing::layer())
        .with(tracing_subscriber::fmt::layer().with_ansi(true))
        .with(
            tracing_subscriber::fmt::layer()
                .with_ansi(false)
                .with_writer(non_blocking),
        )
        .init();

    // Keep jemalloc from hoarding freed memory: run its background purge thread
    // and a periodic explicit purge so load-time and request-time transients are
    // returned to the OS instead of inflating RSS.
    enable_jemalloc_background_thread();
    spawn_jemalloc_purger();

    // Initialize Prometheus metrics
    let metrics_handle = metrics::init_metrics();
    info!("Prometheus metrics initialized");
    if bugsink_frontend_config.is_some() || _bugsink_guard.is_some() {
        info!("Bugsink error reporting configured");
    }

    for (label, path) in [
        ("Properties", &cli.properties),
        ("Postcode features", &cli.postcode_features),
    ] {
        if !path.exists() {
            bail!("{} parquet file not found: {}", label, path.display());
        }
    }

    info!(
        "Loading property data from {}, {}",
        cli.properties.display(),
        cli.postcode_features.display(),
    );
    let spill_dir = cli.spill_dir.as_deref();
    if let Some(dir) = spill_dir {
        info!(
            "Spill-to-disk enabled: large property arrays will be memory-mapped from {}",
            dir.display()
        );
    }
    let property_data =
        data::PropertyData::load(&cli.properties, &cli.postcode_features, spill_dir)?;
    trim_allocator("property data load");
    info!(
        rows = property_data.lat.len(),
        features = property_data.num_features,
        enums = property_data.enum_values.len(),
        "Property data loaded"
    );

    info!("Building spatial grid index (0.01° cells)");
    let grid = utils::GridIndex::build(
        &property_data.lat,
        &property_data.lon,
        consts::GRID_CELL_SIZE,
    );

    info!(
        "Precomputing H3 cells at resolution {}",
        consts::H3_PRECOMPUTE_MAX
    );
    let h3_cells = data::precompute_h3(&property_data.lat, &property_data.lon)?;

    let poi_path = cli.pois;

    if !poi_path.exists() {
        bail!("POI parquet file not found: {}", poi_path.display());
    }

    info!("Loading POI data from {}", poi_path.display());
    let poi_data = data::POIData::load(&poi_path)?;
    trim_allocator("poi data load");
    info!(pois = poi_data.lat.len(), "POI data loaded");

    info!("Building POI spatial grid index");
    let poi_grid = utils::GridIndex::build(&poi_data.lat, &poi_data.lng, consts::GRID_CELL_SIZE);

    // Load place data
    let places_path = &cli.places;
    if !places_path.exists() {
        bail!("Places parquet file not found: {}", places_path.display());
    }
    info!("Loading place data from {}", places_path.display());
    let place_data = data::PlaceData::load(places_path)?;
    trim_allocator("place data load");
    info!(places = place_data.name.len(), "Place data loaded");

    // Load postcode boundaries
    let postcodes_path = &cli.postcodes;
    if !postcodes_path.exists() {
        bail!(
            "Postcode boundaries not found: {}",
            postcodes_path.display()
        );
    }
    info!(
        "Loading postcode boundaries from {}",
        postcodes_path.display()
    );
    let postcode_data = data::PostcodeData::load(postcodes_path)?;
    trim_allocator("postcode boundary load");
    info!(
        postcodes = postcode_data.postcodes.len(),
        "Postcode boundaries loaded"
    );

    let outcode_data = data::OutcodeData::from_postcode_and_place_data(&postcode_data, &place_data);

    // Initialize tile reader
    let tiles_path = &cli.tiles;
    if !tiles_path.exists() {
        bail!("PMTiles file not found: {}", tiles_path.display());
    }
    info!("Loading PMTiles from {}", tiles_path.display());
    let tile_reader = Arc::new(routes::init_tile_reader(tiles_path).await?);
    info!("PMTiles loaded successfully");

    let noise_overlay_reader = init_required_tile_reader("Noise", &cli.noise_overlay_tiles).await?;
    let satellite_reader = init_required_tile_reader("Satellite", &cli.satellite_tiles).await?;
    let satellite_highres_reader =
        init_required_tile_reader("Satellite high-res", &cli.satellite_highres_tiles).await?;
    let crime_hotspot_reader =
        init_required_tile_reader("Crime hotspots", &cli.crime_hotspot_tiles).await?;
    let tree_overlay_reader =
        init_required_tile_reader("Trees outside woodland", &cli.tree_overlay_tiles).await?;
    let property_border_reader =
        init_required_tile_reader("Property borders", &cli.property_border_tiles).await?;

    let feature_name_to_index: rustc_hash::FxHashMap<String, usize> = property_data
        .feature_names
        .iter()
        .enumerate()
        .map(|(idx, name)| (name.clone(), idx))
        .collect();

    let min_keys: Vec<String> = property_data
        .feature_names
        .iter()
        .map(|name| format!("min_{}", name))
        .collect();
    let max_keys: Vec<String> = property_data
        .feature_names
        .iter()
        .map(|name| format!("max_{}", name))
        .collect();
    let avg_keys: Vec<String> = property_data
        .feature_names
        .iter()
        .map(|name| format!("avg_{}", name))
        .collect();

    let poi_category_groups = poi_data.category_groups()?;

    let is_dev = if cli.dist.is_some() {
        info!("Static frontend serving enabled");
        false
    } else {
        info!("No --dist provided; static serving disabled");
        true
    };

    let http_client = reqwest::Client::builder()
        .timeout(Duration::from_secs(SERVICE_CALL_TIMEOUT))
        .connect_timeout(Duration::from_secs(5))
        .referer(false)
        .build()
        .context("Failed to build HTTP client")?;

    info!("Screenshot service configured: {}", cli.screenshot_url);

    let features_response = routes::build_features_response(&property_data);
    info!(
        groups = features_response.groups.len(),
        "Precomputed features response"
    );

    // AI filters system prompt built after travel_time_store is loaded (needs mode counts)

    // Record data loading metrics
    metrics::record_data_stats(
        property_data.lat.len(),
        poi_data.lat.len(),
        postcode_data.postcodes.len(),
    );

    info!("PocketBase configured: {}", cli.pocketbase_url);

    pocketbase::ensure_collections(
        &http_client,
        &cli.pocketbase_url,
        &cli.pocketbase_admin_email,
        &cli.pocketbase_admin_password,
    )
    .await?;
    pocketbase::ensure_oauth_providers(
        &http_client,
        &cli.pocketbase_url,
        &cli.pocketbase_admin_email,
        &cli.pocketbase_admin_password,
        &cli.public_url,
        &cli.google_oauth_client_id,
        &cli.google_oauth_client_secret,
    )
    .await?;
    info!("Gemini configured (model: {})", cli.gemini_model);
    let tt_path = &cli.travel_times;
    if !tt_path.exists() {
        bail!("Travel times directory not found: {}", tt_path.display());
    }
    info!("Loading travel time data from {}", tt_path.display());
    let travel_time_store = {
        let store = data::TravelTimeStore::load(tt_path, 200)?;
        info!(
            modes = store.available_modes.len(),
            "Travel time store loaded"
        );
        Arc::new(store)
    };

    let mode_destinations: Vec<(String, usize)> = travel_time_store
        .available_modes
        .iter()
        .map(|mode| {
            let count = travel_time_store
                .destinations
                .get(mode.as_str())
                .map(|slugs| slugs.len())
                .unwrap_or(0);
            (mode.clone(), count)
        })
        .filter(|(_, count)| *count > 0)
        .collect();
    let ai_filters_system_prompt =
        routes::build_system_prompt(&features_response, &mode_destinations);
    info!("Precomputed AI filters system prompt");

    let token_cache = Arc::new(auth::TokenCache::new());
    let superuser_token_cache = Arc::new(pocketbase::SuperuserTokenCache::new());

    let actual_listings = {
        let path = &cli.actual_listings_path;
        if !path.exists() {
            bail!("Actual listings parquet not found: {}", path.display());
        }
        info!("Loading actual listings from {}", path.display());
        let listings = data::ActualListingData::load(path, &property_data)?;
        trim_allocator("actual listings load");
        info!(rows = listings.lat.len(), "Actual listings loaded");
        Arc::new(listings)
    };

    let developments = {
        let path = &cli.developments_path;
        if !path.exists() {
            bail!("Development sites parquet not found: {}", path.display());
        }
        info!("Loading development sites from {}", path.display());
        let data = data::DevelopmentData::load(path)?;
        trim_allocator("development sites load");
        info!(rows = data.lat.len(), "Development sites loaded");
        Arc::new(data)
    };

    let crime_by_year = {
        let path = &cli.crime_by_year_path;
        if !path.exists() {
            bail!("Crime-by-year parquet not found: {}", path.display());
        }
        let data = data::CrimeByYearData::load(path)?;
        trim_allocator("crime-by-year load");
        Arc::new(data)
    };

    let crime_records = {
        let path = &cli.crime_records_path;
        if !path.exists() {
            bail!("Crime-records parquet not found: {}", path.display());
        }
        let data = data::CrimeRecords::load(path, spill_dir)?;
        trim_allocator("crime-records load");
        Arc::new(data)
    };

    let population = match &cli.population_path {
        Some(path) if path.exists() => {
            let data = data::PostcodePopulation::load(path)?;
            trim_allocator("postcode population load");
            Arc::new(data)
        }
        Some(path) => {
            tracing::warn!(
                "Population parquet not found at {}; area pane will omit population",
                path.display()
            );
            Arc::new(data::PostcodePopulation::empty())
        }
        None => Arc::new(data::PostcodePopulation::empty()),
    };

    let area_crime_averages = {
        let path = &cli.area_crime_averages_path;
        if !path.exists() {
            bail!("Area-crime-averages parquet not found: {}", path.display());
        }
        let data = data::AreaCrimeAverages::load(path)?;
        trim_allocator("area crime averages");
        Arc::new(data)
    };

    let app_state = AppState {
        data: property_data,
        grid,
        h3_cells,
        poi_data: Arc::new(poi_data),
        poi_grid: Arc::new(poi_grid),
        place_data: Arc::new(place_data),
        postcode_data: Arc::new(postcode_data),
        outcode_data: Arc::new(outcode_data),
        feature_name_to_index,
        min_keys,
        max_keys,
        avg_keys,
        poi_category_groups: Arc::new(poi_category_groups),
        features_response,
        screenshot_url: cli.screenshot_url,
        public_url: cli.public_url,
        is_dev,
        http_client,
        pocketbase_url: cli.pocketbase_url,
        pocketbase_admin_email: cli.pocketbase_admin_email,
        pocketbase_admin_password: cli.pocketbase_admin_password,
        gemini_api_key: cli.gemini_api_key,
        gemini_model: cli.gemini_model,
        travel_time_store,
        actual_listings,
        developments,
        crime_by_year,
        crime_records,
        population,
        area_crime_averages,
        token_cache,
        superuser_token_cache,
        ai_filters_system_prompt,
        google_maps_api_key: cli.google_maps_api_key,
        stripe_secret_key: cli.stripe_secret_key,
        stripe_webhook_secret: cli.stripe_webhook_secret,
        stripe_referral_coupon_id: cli.stripe_referral_coupon_id,
        bugsink_frontend_config,
        demo_rate_limiter: Arc::new(ratelimit::DemoRateLimiter::new()),
    };

    let shared = Arc::new(SharedState::new(app_state));

    // Start background PocketBase metrics poller (users, saved searches counts)
    pocketbase::start_metrics_poller(shared.clone());

    let initial_state = shared.load_state();
    let cors = CorsLayer::new()
        .allow_origin(
            initial_state
                .public_url
                .parse::<axum::http::HeaderValue>()
                .expect("public_url must be a valid header value"),
        )
        .allow_methods(AllowMethods::mirror_request())
        .allow_headers(AllowHeaders::mirror_request())
        .allow_credentials(true);

    // Handlers use Axum's State extractor to get Arc<SharedState>, then call
    // load_state() to get the current Arc<AppState>.
    let s_crawler = shared.clone();

    let reader_tile = tile_reader.clone();
    let reader_style = tile_reader.clone();
    let reader_satellite = satellite_reader.clone();
    let reader_satellite_highres = satellite_highres_reader.clone();
    let reader_noise_overlay = noise_overlay_reader.clone();
    let reader_crime_hotspot = crime_hotspot_reader.clone();
    let reader_tree_overlay = tree_overlay_reader.clone();
    let reader_property_border = property_border_reader.clone();
    let public_url_tiles = initial_state.public_url.clone();

    let api = Router::new()
        .route(
            "/api/features",
            get(routes::get_features).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/hexagons",
            get(routes::get_hexagons).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/postcodes",
            get(routes::get_postcodes).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/postcode/{postcode}",
            get(routes::get_postcode_lookup).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/nearest-postcode",
            get(routes::get_nearest_postcode).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/pois",
            get(routes::get_pois).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/actual-listings",
            get(routes::get_actual_listings).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/developments",
            get(routes::get_developments).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/poi-categories",
            get(routes::get_poi_categories).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/places",
            get(routes::get_places).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/travel-modes",
            get(routes::get_travel_modes).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/travel-destinations",
            get(routes::get_travel_destinations).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/journey",
            get(routes::get_journey).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/hexagon-properties",
            get(routes::get_hexagon_properties).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/filter-counts",
            get(routes::get_filter_counts).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/hexagon-stats",
            get(routes::get_hexagon_stats).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/postcode-stats",
            get(routes::get_postcode_stats).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/postcode-properties",
            get(routes::get_postcode_properties).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/crime-records",
            get(routes::get_crime_records).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/screenshot",
            get(routes::get_screenshot).layer(ConcurrencyLimitLayer::new(3)),
        )
        .route(
            "/api/export",
            get(routes::get_export).layer(ConcurrencyLimitLayer::new(3)),
        )
        .route(
            "/api/me",
            get(routes::get_me).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/shorten",
            post(routes::post_shorten).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/share-links",
            get(routes::get_share_links).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/ai-filters",
            post(routes::post_ai_filters).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/streetview",
            get(routes::get_streetview).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/rightmove-search",
            get(routes::get_rightmove_redirect).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/newsletter",
            patch(routes::patch_newsletter).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/pricing",
            get(routes::get_pricing).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/checkout",
            post(routes::post_checkout).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/stripe-webhook",
            post(routes::post_stripe_webhook).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/invites",
            get(routes::get_invites)
                .post(routes::post_invites)
                .layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/invite/{code}",
            get(routes::get_invite).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/api/redeem-invite",
            post(routes::post_redeem_invite).layer(ConcurrencyLimitLayer::new(5)),
        )
        .route(
            "/s/{code}",
            get(routes::get_short_url).layer(ConcurrencyLimitLayer::new(10)),
        )
        .route(
            "/api/telemetry",
            post(routes::post_telemetry).layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/pb/api/realtime",
            any(routes::proxy_to_pocketbase).layer(ConcurrencyLimitLayer::new(50)),
        )
        .route(
            "/pb/{*rest}",
            any(routes::proxy_to_pocketbase).layer(ConcurrencyLimitLayer::new(10)),
        )
        // Tile routes use a different state type, kept as closures
        .route(
            "/api/tiles/{z}/{x}/{y}",
            get(move |path| routes::get_tile(axum::extract::State(reader_tile.clone()), path))
                .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/tiles/style.json",
            get(move |query| {
                let pu = public_url_tiles.clone();
                routes::get_style(axum::extract::State(reader_style.clone()), pu, query)
            })
            .layer(ConcurrencyLimitLayer::new(20)),
        )
        .route(
            "/api/tiles/satellite/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_satellite.clone(),
                    routes::OverlayTileFormat::RasterJpeg,
                    "satellite",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/tiles/satellite-highres/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_satellite_highres.clone(),
                    routes::OverlayTileFormat::RasterWebp,
                    "satellite-highres",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/overlays/noise/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_noise_overlay.clone(),
                    routes::OverlayTileFormat::RasterPng,
                    "noise",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/overlays/crime-hotspots/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_crime_hotspot.clone(),
                    routes::OverlayTileFormat::VectorMvtGzip,
                    "crime-hotspots",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/overlays/trees-outside-woodlands/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_tree_overlay.clone(),
                    routes::OverlayTileFormat::VectorMvtGzip,
                    "trees-outside-woodlands",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route(
            "/api/overlays/property-borders/{z}/{x}/{y}",
            get(move |path| {
                routes::get_overlay_tile(
                    reader_property_border.clone(),
                    routes::OverlayTileFormat::VectorMvtGzip,
                    "property-borders",
                    path,
                )
            })
            .layer(ConcurrencyLimitLayer::new(30)),
        )
        .route("/health", get(|| async { "ok" }))
        .route(
            "/metrics",
            get(move |headers, connect_info| {
                metrics::metrics_handler(metrics_handle.clone(), headers, connect_info)
            }),
        )
        .with_state(shared.clone());

    let app = if let Some(ref dist) = cli.dist {
        api.fallback_service(ServeDir::new(dist).fallback(ServeFile::new(dist.join("index.html"))))
    } else {
        api
    }
    .layer(middleware::from_fn(metrics::track_metrics))
    .layer(middleware::from_fn(ratelimit::demo_guard_middleware))
    .layer(middleware::from_fn(auth::auth_middleware))
    .layer(middleware::from_fn(
        move |req: axum::extract::Request, next: middleware::Next| {
            let st = s_crawler.load_state();
            async move {
                // Inject state into request extensions for auth + OG middleware
                let (mut parts, body) = req.into_parts();
                parts.extensions.insert(st);
                let req = axum::extract::Request::from_parts(parts, body);
                og_middleware::og_middleware(req, next).await
            }
        },
    ))
    .layer(middleware::from_fn(static_cache_headers))
    .layer(middleware::from_fn(security_headers))
    .layer(middleware::from_fn(capture_server_error_responses))
    .layer(cors)
    .layer(CompressionLayer::new().zstd(true).gzip(true))
    .layer(TraceLayer::new_for_http())
    .layer(
        ServiceBuilder::new()
            .layer(sentry::integrations::tower::NewSentryLayer::<
                axum::extract::Request,
            >::new_from_top())
            .layer(sentry::integrations::tower::SentryHttpLayer::new()),
    );

    // NOTE: we deliberately do NOT mlockall() here. Locking MCL_CURRENT|MCL_FUTURE
    // pinned the allocator's entire mapped heap, including jemalloc's freed/dirty
    // pages, resident and non-reclaimable, inflating RSS from the ~10GB working
    // set to ~40GB and defeating the allocator's page-return entirely. The hot
    // working set stays resident naturally; freed pages are returned to the OS.

    trim_allocator("startup complete");

    let addr = consts::SERVER_ADDRESS;
    let listener = tokio::net::TcpListener::bind(addr)
        .await
        .with_context(|| format!("Failed to bind to {addr}"))?;
    info!("Server listening on {}", addr);
    axum::serve(
        listener,
        app.into_make_service_with_connect_info::<std::net::SocketAddr>(),
    )
    .with_graceful_shutdown(shutdown_signal())
    .await
    .context("Server error")?;
    info!("Server shut down cleanly");
    Ok(())
}

/// Resolves on SIGTERM or SIGINT so in-flight requests (exports, checkouts)
/// can drain before the process exits. The realtime SSE proxy connections
/// never complete, so a watchdog force-exits before Docker's default 10s
/// stop grace period elapses and it sends SIGKILL.
async fn shutdown_signal() {
    let ctrl_c = async {
        tokio::signal::ctrl_c()
            .await
            .expect("Failed to install SIGINT handler");
    };

    #[cfg(unix)]
    let terminate = async {
        tokio::signal::unix::signal(tokio::signal::unix::SignalKind::terminate())
            .expect("Failed to install SIGTERM handler")
            .recv()
            .await;
    };

    #[cfg(not(unix))]
    let terminate = std::future::pending::<()>();

    tokio::select! {
        () = ctrl_c => {},
        () = terminate => {},
    }
    info!("Shutdown signal received; draining in-flight requests");

    tokio::spawn(async {
        tokio::time::sleep(Duration::from_secs(8)).await;
        tracing::warn!("Graceful shutdown drain timed out after 8s; forcing exit");
        std::process::exit(0);
    });
}