Compare commits
3 commits
44b4e0d72f
...
719dd27600
| Author | SHA1 | Date | |
|---|---|---|---|
| 719dd27600 | |||
| 3e307c2fff | |||
| df63764a9f |
10 changed files with 158 additions and 210 deletions
|
|
@ -13,85 +13,15 @@ jobs:
|
||||||
- name: Checkout
|
- name: Checkout
|
||||||
uses: actions/checkout@v4
|
uses: actions/checkout@v4
|
||||||
|
|
||||||
- name: Install Docker CLI
|
- name: Build and publish main image
|
||||||
run: |
|
uses: http://forgejo:3000/andras/ci-actions/docker-publish@main
|
||||||
ARCH=$(uname -m)
|
|
||||||
curl -fsSL "https://download.docker.com/linux/static/stable/${ARCH}/docker-27.5.1.tgz" \
|
|
||||||
| tar xz --strip-components=1 -C /usr/local/bin docker/docker
|
|
||||||
docker --version
|
|
||||||
|
|
||||||
- name: Set up Docker Buildx
|
|
||||||
uses: https://github.com/docker/setup-buildx-action@v3
|
|
||||||
with:
|
|
||||||
driver-opts: |
|
|
||||||
network=host
|
|
||||||
|
|
||||||
- name: Resolve registry vars
|
|
||||||
id: registry
|
|
||||||
env:
|
|
||||||
CONTAINER_REGISTRY_HOST: ${{ vars.CONTAINER_REGISTRY_HOST }}
|
|
||||||
run: |
|
|
||||||
host="${CONTAINER_REGISTRY_HOST:-${{ gitea.server_url }}}"
|
|
||||||
host="${host#https://}"
|
|
||||||
host="${host#http://}"
|
|
||||||
host="${host%/}"
|
|
||||||
if [ "$host" = "forgejo:3000" ]; then
|
|
||||||
host="127.0.0.1:13000"
|
|
||||||
fi
|
|
||||||
repo=$(echo "${{ gitea.repository }}" | tr '[:upper:]' '[:lower:]')
|
|
||||||
owner="${repo%%/*}"
|
|
||||||
{
|
|
||||||
echo "host=${host}"
|
|
||||||
echo "owner=${owner}"
|
|
||||||
echo "image=${host}/${repo}"
|
|
||||||
echo "screenshot_image=${host}/${repo}-screenshot"
|
|
||||||
} >> "$GITHUB_OUTPUT"
|
|
||||||
|
|
||||||
- name: Log in to Forgejo Container Registry
|
|
||||||
uses: https://github.com/docker/login-action@v3
|
|
||||||
with:
|
|
||||||
registry: ${{ steps.registry.outputs.host }}
|
|
||||||
username: ${{ steps.registry.outputs.owner }}
|
|
||||||
password: ${{ secrets.FORGEJO_PACKAGE_TOKEN }}
|
|
||||||
|
|
||||||
- name: Extract metadata (main)
|
|
||||||
id: meta
|
|
||||||
uses: https://github.com/docker/metadata-action@v5
|
|
||||||
with:
|
|
||||||
images: ${{ steps.registry.outputs.image }}
|
|
||||||
tags: |
|
|
||||||
type=sha,format=short
|
|
||||||
type=raw,value=latest,enable={{is_default_branch}}
|
|
||||||
type=semver,pattern={{version}}
|
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
|
||||||
|
|
||||||
- name: Build and push (main)
|
|
||||||
uses: https://github.com/docker/build-push-action@v6
|
|
||||||
with:
|
with:
|
||||||
context: .
|
context: .
|
||||||
push: true
|
token: ${{ secrets.FORGEJO_PACKAGE_TOKEN }}
|
||||||
tags: ${{ steps.meta.outputs.tags }}
|
|
||||||
labels: ${{ steps.meta.outputs.labels }}
|
|
||||||
cache-from: type=registry,ref=${{ steps.registry.outputs.image }}:buildcache
|
|
||||||
cache-to: type=registry,ref=${{ steps.registry.outputs.image }}:buildcache,mode=max
|
|
||||||
|
|
||||||
- name: Extract metadata (screenshot)
|
- name: Build and publish screenshot image
|
||||||
id: meta-screenshot
|
uses: http://forgejo:3000/andras/ci-actions/docker-publish@main
|
||||||
uses: https://github.com/docker/metadata-action@v5
|
|
||||||
with:
|
|
||||||
images: ${{ steps.registry.outputs.screenshot_image }}
|
|
||||||
tags: |
|
|
||||||
type=sha,format=short
|
|
||||||
type=raw,value=latest,enable={{is_default_branch}}
|
|
||||||
type=semver,pattern={{version}}
|
|
||||||
type=semver,pattern={{major}}.{{minor}}
|
|
||||||
|
|
||||||
- name: Build and push (screenshot)
|
|
||||||
uses: https://github.com/docker/build-push-action@v6
|
|
||||||
with:
|
with:
|
||||||
context: ./screenshot
|
context: ./screenshot
|
||||||
push: true
|
image-suffix: -screenshot
|
||||||
tags: ${{ steps.meta-screenshot.outputs.tags }}
|
token: ${{ secrets.FORGEJO_PACKAGE_TOKEN }}
|
||||||
labels: ${{ steps.meta-screenshot.outputs.labels }}
|
|
||||||
cache-from: type=registry,ref=${{ steps.registry.outputs.screenshot_image }}:buildcache
|
|
||||||
cache-to: type=registry,ref=${{ steps.registry.outputs.screenshot_image }}:buildcache,mode=max
|
|
||||||
|
|
|
||||||
|
|
@ -47,4 +47,4 @@ EXPOSE 8001
|
||||||
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
|
HEALTHCHECK --interval=30s --timeout=5s --start-period=120s --retries=3 \
|
||||||
CMD curl -f http://localhost:8001/health || exit 1
|
CMD curl -f http://localhost:8001/health || exit 1
|
||||||
ENTRYPOINT ["./property-map-server"]
|
ENTRYPOINT ["./property-map-server"]
|
||||||
CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--dist", "/app/frontend/dist"]
|
CMD ["--properties", "/app/data/properties.parquet", "--postcode-features", "/app/data/postcode.parquet", "--pois", "/app/data/filtered_uk_pois.parquet", "--places", "/app/data/places.parquet", "--tiles", "/app/data/uk.pmtiles", "--postcodes", "/app/data/postcode_boundaries", "--travel-times", "/app/data/travel-times", "--satellite-tiles", "/app/data/satellite.pmtiles", "--satellite-highres-tiles", "/app/data/satellite_highres.pmtiles", "--noise-overlay-tiles", "/app/data/noise_lden_10m.pmtiles", "--crime-hotspot-tiles", "/app/data/crime_hotspots.pmtiles", "--tree-overlay-tiles", "/app/data/trees_outside_woodlands.pmtiles", "--property-border-tiles", "/app/data/property_borders.pmtiles", "--dist", "/app/frontend/dist"]
|
||||||
|
|
|
||||||
|
|
@ -11,7 +11,7 @@ services:
|
||||||
command: >
|
command: >
|
||||||
bash -c "
|
bash -c "
|
||||||
cargo install cargo-watch &&
|
cargo install cargo-watch &&
|
||||||
cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data4/properties.parquet --postcode-features /app/property-data4/postcode.parquet --pois /app/property-data4/filtered_uk_pois.parquet --places /app/property-data4/places.parquet --tiles /app/property-data4/uk.pmtiles --postcodes /app/property-data4/postcode_boundaries --travel-times /app/property-data4/travel-times'
|
cargo watch --poll -i logs/ -x 'run -- --properties /app/property-data4/properties.parquet --postcode-features /app/property-data4/postcode.parquet --pois /app/property-data4/filtered_uk_pois.parquet --places /app/property-data4/places.parquet --tiles /app/property-data4/uk.pmtiles --postcodes /app/property-data4/postcode_boundaries --travel-times /app/property-data4/travel-times --satellite-tiles /app/property-data4/satellite.pmtiles --satellite-highres-tiles /app/property-data4/satellite_highres.pmtiles --noise-overlay-tiles /app/property-data4/noise_lden_10m.pmtiles --crime-hotspot-tiles /app/property-data4/crime_hotspots.pmtiles --tree-overlay-tiles /app/property-data4/trees_outside_woodlands.pmtiles --property-border-tiles /app/property-data4/property_borders.pmtiles'
|
||||||
"
|
"
|
||||||
ports:
|
ports:
|
||||||
- "8001:8001"
|
- "8001:8001"
|
||||||
|
|
@ -51,8 +51,6 @@ services:
|
||||||
BUGSINK_ENVIRONMENT: ${BUGSINK_ENVIRONMENT:-development}
|
BUGSINK_ENVIRONMENT: ${BUGSINK_ENVIRONMENT:-development}
|
||||||
BUGSINK_RELEASE: ${BUGSINK_RELEASE:-}
|
BUGSINK_RELEASE: ${BUGSINK_RELEASE:-}
|
||||||
BUGSINK_SEND_DEFAULT_PII: ${BUGSINK_SEND_DEFAULT_PII:-false}
|
BUGSINK_SEND_DEFAULT_PII: ${BUGSINK_SEND_DEFAULT_PII:-false}
|
||||||
ACTUAL_LISTINGS_PATH: /app/finder/data/online_listings_buy_enriched.parquet
|
|
||||||
CRIME_BY_YEAR_PATH: /app/property-data4/crime_by_postcode_by_year.parquet
|
|
||||||
depends_on:
|
depends_on:
|
||||||
screenshot:
|
screenshot:
|
||||||
condition: service_healthy
|
condition: service_healthy
|
||||||
|
|
|
||||||
|
|
@ -183,9 +183,19 @@ def search_vap_tiles(aoi: dict, timeout: float = 60.0) -> list[VapTile]:
|
||||||
def _download_and_extract(
|
def _download_and_extract(
|
||||||
tile: VapTile, ecw_dir: Path, key: str, timeout: float, retries: int
|
tile: VapTile, ecw_dir: Path, key: str, timeout: float, retries: int
|
||||||
) -> list[Path]:
|
) -> list[Path]:
|
||||||
"""Download one survey zip and extract its ECW raster(s)."""
|
"""Download one survey zip and extract its ECW raster(s).
|
||||||
|
|
||||||
|
The Defra endpoint sometimes answers a download with a 200 whose body is an
|
||||||
|
error/HTML page or a truncated stream rather than a ZIP (rate limiting, a
|
||||||
|
rotated key, a transient backend hiccup). We treat a non-ZIP body the same as
|
||||||
|
a network error -- retry it -- and, if every attempt fails, skip just this
|
||||||
|
tile rather than aborting the whole multi-tile run: missing coverage simply
|
||||||
|
shows the Sentinel-2 base through, and the caller still hard-fails if *no*
|
||||||
|
ECWs were extracted at all.
|
||||||
|
"""
|
||||||
url = f"{tile.uri}?subscription-key={key}"
|
url = f"{tile.uri}?subscription-key={key}"
|
||||||
zip_path = ecw_dir / f"{tile.os_tile_id}.zip"
|
zip_path = ecw_dir / f"{tile.os_tile_id}.zip"
|
||||||
|
last_err: Exception | None = None
|
||||||
for attempt in range(retries + 1):
|
for attempt in range(retries + 1):
|
||||||
try:
|
try:
|
||||||
with urllib.request.urlopen(
|
with urllib.request.urlopen(
|
||||||
|
|
@ -193,10 +203,25 @@ def _download_and_extract(
|
||||||
timeout=timeout,
|
timeout=timeout,
|
||||||
) as response, zip_path.open("wb") as out:
|
) as response, zip_path.open("wb") as out:
|
||||||
shutil.copyfileobj(response, out, length=1 << 20)
|
shutil.copyfileobj(response, out, length=1 << 20)
|
||||||
break
|
return _extract_ecws(zip_path, tile, ecw_dir)
|
||||||
except (urllib.error.URLError, TimeoutError, ConnectionError) as err:
|
except (
|
||||||
if attempt == retries:
|
urllib.error.URLError,
|
||||||
raise RuntimeError(f"Failed to download {url}: {err}") from err
|
TimeoutError,
|
||||||
|
ConnectionError,
|
||||||
|
zipfile.BadZipFile,
|
||||||
|
) as err:
|
||||||
|
last_err = err
|
||||||
|
zip_path.unlink(missing_ok=True)
|
||||||
|
print(
|
||||||
|
f" {tile.os_tile_id}: download failed after {retries + 1} attempt(s), "
|
||||||
|
f"skipping ({last_err})",
|
||||||
|
flush=True,
|
||||||
|
)
|
||||||
|
return []
|
||||||
|
|
||||||
|
|
||||||
|
def _extract_ecws(zip_path: Path, tile: VapTile, ecw_dir: Path) -> list[Path]:
|
||||||
|
"""Extract every ECW raster from a downloaded survey zip; delete the zip."""
|
||||||
extracted: list[Path] = []
|
extracted: list[Path] = []
|
||||||
with zipfile.ZipFile(zip_path) as archive:
|
with zipfile.ZipFile(zip_path) as archive:
|
||||||
for member in archive.infolist():
|
for member in archive.infolist():
|
||||||
|
|
|
||||||
|
|
@ -1,3 +1,6 @@
|
||||||
|
import io
|
||||||
|
import zipfile
|
||||||
|
|
||||||
from pipeline.download import satellite_highres
|
from pipeline.download import satellite_highres
|
||||||
from pipeline.download.satellite_highres import (
|
from pipeline.download.satellite_highres import (
|
||||||
VapTile,
|
VapTile,
|
||||||
|
|
@ -95,3 +98,76 @@ def test_select_best_rgb_breaks_resolution_ties_by_year() -> None:
|
||||||
def test_select_best_rgb_empty_when_no_rgb() -> None:
|
def test_select_best_rgb_empty_when_no_rgb() -> None:
|
||||||
payload = {"results": [_result("lidar_composite_dtm", "2022", "1", "TQ2575")]}
|
payload = {"results": [_result("lidar_composite_dtm", "2022", "1", "TQ2575")]}
|
||||||
assert select_best_rgb_tiles(parse_search_results(payload)) == []
|
assert select_best_rgb_tiles(parse_search_results(payload)) == []
|
||||||
|
|
||||||
|
|
||||||
|
_TILE = VapTile(
|
||||||
|
product_id=satellite_highres.VAP_RGB_PRODUCT,
|
||||||
|
year=2008,
|
||||||
|
resolution_m=0.4,
|
||||||
|
os_tile_id="TQ2575",
|
||||||
|
uri="https://example.invalid/tile",
|
||||||
|
label="t",
|
||||||
|
)
|
||||||
|
|
||||||
|
|
||||||
|
def _zip_with_ecw() -> bytes:
|
||||||
|
"""A minimal valid survey zip carrying one ECW member."""
|
||||||
|
buffer = io.BytesIO()
|
||||||
|
with zipfile.ZipFile(buffer, "w") as archive:
|
||||||
|
archive.writestr("TQ2575.ecw", b"ecw-pixels")
|
||||||
|
archive.writestr("readme.txt", b"ignored")
|
||||||
|
return buffer.getvalue()
|
||||||
|
|
||||||
|
|
||||||
|
def _fake_urlopen(bodies: list[bytes]):
|
||||||
|
"""Return a urlopen stand-in that yields each body in turn as the response."""
|
||||||
|
queue = list(bodies)
|
||||||
|
|
||||||
|
def opener(_request, timeout=None): # noqa: ANN001 - matches urlopen signature
|
||||||
|
return io.BytesIO(queue.pop(0))
|
||||||
|
|
||||||
|
return opener
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_extracts_ecw_from_valid_zip(tmp_path, monkeypatch) -> None:
|
||||||
|
monkeypatch.setattr(
|
||||||
|
satellite_highres.urllib.request,
|
||||||
|
"urlopen",
|
||||||
|
_fake_urlopen([_zip_with_ecw()]),
|
||||||
|
)
|
||||||
|
paths = satellite_highres._download_and_extract(
|
||||||
|
_TILE, tmp_path, key="k", timeout=1.0, retries=2
|
||||||
|
)
|
||||||
|
assert [p.name for p in paths] == ["TQ2575_TQ2575.ecw"]
|
||||||
|
assert paths[0].read_bytes() == b"ecw-pixels"
|
||||||
|
# The transient zip is cleaned up regardless.
|
||||||
|
assert not (tmp_path / "TQ2575.zip").exists()
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_retries_past_a_corrupt_body(tmp_path, monkeypatch) -> None:
|
||||||
|
# First response is a non-zip error page; the retry serves a real zip.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
satellite_highres.urllib.request,
|
||||||
|
"urlopen",
|
||||||
|
_fake_urlopen([b"<html>rate limited</html>", _zip_with_ecw()]),
|
||||||
|
)
|
||||||
|
paths = satellite_highres._download_and_extract(
|
||||||
|
_TILE, tmp_path, key="k", timeout=1.0, retries=2
|
||||||
|
)
|
||||||
|
assert [p.name for p in paths] == ["TQ2575_TQ2575.ecw"]
|
||||||
|
|
||||||
|
|
||||||
|
def test_download_skips_tile_when_all_attempts_are_corrupt(
|
||||||
|
tmp_path, monkeypatch
|
||||||
|
) -> None:
|
||||||
|
# Every attempt returns a non-zip body: the tile is skipped, not raised.
|
||||||
|
monkeypatch.setattr(
|
||||||
|
satellite_highres.urllib.request,
|
||||||
|
"urlopen",
|
||||||
|
_fake_urlopen([b"not a zip"] * 3),
|
||||||
|
)
|
||||||
|
paths = satellite_highres._download_and_extract(
|
||||||
|
_TILE, tmp_path, key="k", timeout=1.0, retries=2
|
||||||
|
)
|
||||||
|
assert paths == []
|
||||||
|
assert list(tmp_path.glob("*")) == [] # no leftover zip or partial ecw
|
||||||
|
|
|
||||||
|
|
@ -21,10 +21,6 @@ pub struct FrontendConfig {
|
||||||
pub send_default_pii: bool,
|
pub send_default_pii: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn env_nonempty(name: &str) -> Option<String> {
|
|
||||||
std::env::var(name).ok().and_then(nonempty)
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn nonempty(value: String) -> Option<String> {
|
pub fn nonempty(value: String) -> Option<String> {
|
||||||
let trimmed = value.trim();
|
let trimmed = value.trim();
|
||||||
(!trimmed.is_empty()).then(|| trimmed.to_owned())
|
(!trimmed.is_empty()).then(|| trimmed.to_owned())
|
||||||
|
|
|
||||||
|
|
@ -40,14 +40,6 @@ pub struct CrimeByYearData {
|
||||||
}
|
}
|
||||||
|
|
||||||
impl CrimeByYearData {
|
impl CrimeByYearData {
|
||||||
pub fn empty() -> Self {
|
|
||||||
Self {
|
|
||||||
crime_types: Vec::new(),
|
|
||||||
years_by_type: Vec::new(),
|
|
||||||
series_by_postcode: FxHashMap::default(),
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
pub fn load(path: &Path) -> anyhow::Result<Self> {
|
||||||
run_polars_io(|| Self::load_inner(path))
|
run_polars_io(|| Self::load_inner(path))
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -251,29 +251,29 @@ struct Cli {
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
tiles: PathBuf,
|
tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles raster basemap for satellite imagery.
|
/// PMTiles raster basemap for satellite imagery.
|
||||||
#[arg(long, env = "SATELLITE_TILES")]
|
#[arg(long, env = "SATELLITE_TILES")]
|
||||||
satellite_tiles: Option<PathBuf>,
|
satellite_tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles raster overlay for high-resolution EA aerial photography.
|
/// PMTiles raster overlay for high-resolution EA aerial photography.
|
||||||
#[arg(long, env = "SATELLITE_HIGHRES_TILES")]
|
#[arg(long, env = "SATELLITE_HIGHRES_TILES")]
|
||||||
satellite_highres_tiles: Option<PathBuf>,
|
satellite_highres_tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles raster overlay for high-resolution strategic noise.
|
/// PMTiles raster overlay for high-resolution strategic noise.
|
||||||
#[arg(long, env = "NOISE_OVERLAY_TILES")]
|
#[arg(long, env = "NOISE_OVERLAY_TILES")]
|
||||||
noise_overlay_tiles: Option<PathBuf>,
|
noise_overlay_tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles vector overlay for crime heatmap points.
|
/// PMTiles vector overlay for crime heatmap points.
|
||||||
#[arg(long, env = "CRIME_HOTSPOT_TILES")]
|
#[arg(long, env = "CRIME_HOTSPOT_TILES")]
|
||||||
crime_hotspot_tiles: Option<PathBuf>,
|
crime_hotspot_tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles vector overlay for Trees Outside Woodland polygons.
|
/// PMTiles vector overlay for Trees Outside Woodland polygons.
|
||||||
#[arg(long, env = "TREE_OVERLAY_TILES")]
|
#[arg(long, env = "TREE_OVERLAY_TILES")]
|
||||||
tree_overlay_tiles: Option<PathBuf>,
|
tree_overlay_tiles: PathBuf,
|
||||||
|
|
||||||
/// Optional PMTiles vector overlay for INSPIRE property-border polygons.
|
/// PMTiles vector overlay for INSPIRE property-border polygons.
|
||||||
#[arg(long, env = "PROPERTY_BORDER_TILES")]
|
#[arg(long, env = "PROPERTY_BORDER_TILES")]
|
||||||
property_border_tiles: Option<PathBuf>,
|
property_border_tiles: PathBuf,
|
||||||
|
|
||||||
/// Path to the frontend dist directory (optional; disables static serving and OG injection when omitted)
|
/// Path to the frontend dist directory (optional; disables static serving and OG injection when omitted)
|
||||||
#[arg(long)]
|
#[arg(long)]
|
||||||
|
|
@ -311,13 +311,13 @@ struct Cli {
|
||||||
#[arg(long, env = "TRAVEL_TIMES")]
|
#[arg(long, env = "TRAVEL_TIMES")]
|
||||||
travel_times: PathBuf,
|
travel_times: PathBuf,
|
||||||
|
|
||||||
/// Optional path to a parquet of live online listings (Rightmove etc.) to overlay on the map.
|
/// Path to a parquet of live online listings (Rightmove etc.) to overlay on the map.
|
||||||
#[arg(long, env = "ACTUAL_LISTINGS_PATH")]
|
#[arg(long, env = "ACTUAL_LISTINGS_PATH")]
|
||||||
actual_listings_path: Option<PathBuf>,
|
actual_listings_path: PathBuf,
|
||||||
|
|
||||||
/// Optional path to the per-LSOA per-year crime parquet (display-only side table for the right pane).
|
/// Path to the per-LSOA per-year crime parquet (display-only side table for the right pane).
|
||||||
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
|
#[arg(long, env = "CRIME_BY_YEAR_PATH")]
|
||||||
crime_by_year_path: Option<PathBuf>,
|
crime_by_year_path: PathBuf,
|
||||||
|
|
||||||
/// Google Maps API key for Street View metadata lookups
|
/// Google Maps API key for Street View metadata lookups
|
||||||
#[arg(long, env = "GOOGLE_MAPS_API_KEY")]
|
#[arg(long, env = "GOOGLE_MAPS_API_KEY")]
|
||||||
|
|
@ -345,22 +345,22 @@ struct Cli {
|
||||||
|
|
||||||
/// Bugsink DSN for backend error reporting
|
/// Bugsink DSN for backend error reporting
|
||||||
#[arg(long, env = "BUGSINK_DSN", hide_env_values = true)]
|
#[arg(long, env = "BUGSINK_DSN", hide_env_values = true)]
|
||||||
bugsink_dsn: Option<String>,
|
bugsink_dsn: String,
|
||||||
|
|
||||||
/// Bugsink DSN injected into the browser app; falls back to BUGSINK_DSN when omitted
|
/// Bugsink DSN injected into the browser app
|
||||||
#[arg(long, env = "FRONTEND_BUGSINK_DSN", hide_env_values = true)]
|
#[arg(long, env = "FRONTEND_BUGSINK_DSN", hide_env_values = true)]
|
||||||
frontend_bugsink_dsn: Option<String>,
|
frontend_bugsink_dsn: String,
|
||||||
|
|
||||||
/// Bugsink/Sentry environment name
|
/// Bugsink/Sentry environment name
|
||||||
#[arg(long, env = "BUGSINK_ENVIRONMENT")]
|
#[arg(long, env = "BUGSINK_ENVIRONMENT")]
|
||||||
bugsink_environment: Option<String>,
|
bugsink_environment: String,
|
||||||
|
|
||||||
/// Bugsink/Sentry release name
|
/// Bugsink/Sentry release name
|
||||||
#[arg(long, env = "BUGSINK_RELEASE")]
|
#[arg(long, env = "BUGSINK_RELEASE")]
|
||||||
bugsink_release: Option<String>,
|
bugsink_release: String,
|
||||||
|
|
||||||
/// Include default PII in Bugsink events
|
/// Include default PII in Bugsink events
|
||||||
#[arg(long, env = "BUGSINK_SEND_DEFAULT_PII", default_value_t = false)]
|
#[arg(long, env = "BUGSINK_SEND_DEFAULT_PII", action = clap::ArgAction::Set)]
|
||||||
bugsink_send_default_pii: bool,
|
bugsink_send_default_pii: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -404,49 +404,19 @@ async fn init_required_tile_reader(
|
||||||
Ok(Arc::new(routes::init_tile_reader(path).await?))
|
Ok(Arc::new(routes::init_tile_reader(path).await?))
|
||||||
}
|
}
|
||||||
|
|
||||||
fn configured_or_default_overlay_path(
|
|
||||||
configured: &Option<PathBuf>,
|
|
||||||
tiles_path: &Path,
|
|
||||||
file_name: &str,
|
|
||||||
) -> PathBuf {
|
|
||||||
if let Some(path) = configured {
|
|
||||||
return path.clone();
|
|
||||||
}
|
|
||||||
|
|
||||||
tiles_path
|
|
||||||
.parent()
|
|
||||||
.map(|parent| parent.join(file_name))
|
|
||||||
.unwrap_or_else(|| PathBuf::from(file_name))
|
|
||||||
}
|
|
||||||
|
|
||||||
#[tokio::main]
|
#[tokio::main]
|
||||||
async fn main() -> anyhow::Result<()> {
|
async fn main() -> anyhow::Result<()> {
|
||||||
let cli = Cli::parse();
|
let cli = Cli::parse();
|
||||||
let bugsink_environment = cli
|
|
||||||
.bugsink_environment
|
|
||||||
.clone()
|
|
||||||
.or_else(|| bugsink::env_nonempty("SENTRY_ENVIRONMENT"));
|
|
||||||
let bugsink_release = cli
|
|
||||||
.bugsink_release
|
|
||||||
.clone()
|
|
||||||
.or_else(|| bugsink::env_nonempty("SENTRY_RELEASE"));
|
|
||||||
let backend_bugsink_dsn = cli
|
|
||||||
.bugsink_dsn
|
|
||||||
.clone()
|
|
||||||
.or_else(|| bugsink::env_nonempty("SENTRY_DSN"));
|
|
||||||
let _bugsink_guard = bugsink::init_backend(&bugsink::BackendConfig {
|
let _bugsink_guard = bugsink::init_backend(&bugsink::BackendConfig {
|
||||||
dsn: backend_bugsink_dsn.clone(),
|
dsn: Some(cli.bugsink_dsn.clone()),
|
||||||
environment: bugsink_environment.clone(),
|
environment: Some(cli.bugsink_environment.clone()),
|
||||||
release: bugsink_release.clone(),
|
release: Some(cli.bugsink_release.clone()),
|
||||||
send_default_pii: cli.bugsink_send_default_pii,
|
send_default_pii: cli.bugsink_send_default_pii,
|
||||||
});
|
});
|
||||||
let bugsink_frontend_config = bugsink::frontend_config(
|
let bugsink_frontend_config = bugsink::frontend_config(
|
||||||
cli.frontend_bugsink_dsn
|
Some(cli.frontend_bugsink_dsn.clone()),
|
||||||
.clone()
|
Some(cli.bugsink_environment.clone()),
|
||||||
.or_else(|| bugsink::env_nonempty("PUBLIC_BUGSINK_DSN"))
|
Some(cli.bugsink_release.clone()),
|
||||||
.or(backend_bugsink_dsn),
|
|
||||||
bugsink_environment.clone(),
|
|
||||||
bugsink_release.clone(),
|
|
||||||
cli.bugsink_send_default_pii,
|
cli.bugsink_send_default_pii,
|
||||||
);
|
);
|
||||||
|
|
||||||
|
|
@ -569,44 +539,17 @@ async fn main() -> anyhow::Result<()> {
|
||||||
let tile_reader = Arc::new(routes::init_tile_reader(tiles_path).await?);
|
let tile_reader = Arc::new(routes::init_tile_reader(tiles_path).await?);
|
||||||
info!("PMTiles loaded successfully");
|
info!("PMTiles loaded successfully");
|
||||||
|
|
||||||
let noise_overlay_tiles = configured_or_default_overlay_path(
|
let noise_overlay_reader =
|
||||||
&cli.noise_overlay_tiles,
|
init_required_tile_reader("Noise", &cli.noise_overlay_tiles).await?;
|
||||||
tiles_path,
|
let satellite_reader = init_required_tile_reader("Satellite", &cli.satellite_tiles).await?;
|
||||||
"noise_lden_10m.pmtiles",
|
|
||||||
);
|
|
||||||
let satellite_tiles =
|
|
||||||
configured_or_default_overlay_path(&cli.satellite_tiles, tiles_path, "satellite.pmtiles");
|
|
||||||
let satellite_highres_tiles = configured_or_default_overlay_path(
|
|
||||||
&cli.satellite_highres_tiles,
|
|
||||||
tiles_path,
|
|
||||||
"satellite_highres.pmtiles",
|
|
||||||
);
|
|
||||||
let crime_hotspot_tiles = configured_or_default_overlay_path(
|
|
||||||
&cli.crime_hotspot_tiles,
|
|
||||||
tiles_path,
|
|
||||||
"crime_hotspots.pmtiles",
|
|
||||||
);
|
|
||||||
let tree_overlay_tiles = configured_or_default_overlay_path(
|
|
||||||
&cli.tree_overlay_tiles,
|
|
||||||
tiles_path,
|
|
||||||
"trees_outside_woodlands.pmtiles",
|
|
||||||
);
|
|
||||||
let property_border_tiles = configured_or_default_overlay_path(
|
|
||||||
&cli.property_border_tiles,
|
|
||||||
tiles_path,
|
|
||||||
"property_borders.pmtiles",
|
|
||||||
);
|
|
||||||
|
|
||||||
let noise_overlay_reader = init_required_tile_reader("Noise", &noise_overlay_tiles).await?;
|
|
||||||
let satellite_reader = init_required_tile_reader("Satellite", &satellite_tiles).await?;
|
|
||||||
let satellite_highres_reader =
|
let satellite_highres_reader =
|
||||||
init_required_tile_reader("Satellite high-res", &satellite_highres_tiles).await?;
|
init_required_tile_reader("Satellite high-res", &cli.satellite_highres_tiles).await?;
|
||||||
let crime_hotspot_reader =
|
let crime_hotspot_reader =
|
||||||
init_required_tile_reader("Crime hotspots", &crime_hotspot_tiles).await?;
|
init_required_tile_reader("Crime hotspots", &cli.crime_hotspot_tiles).await?;
|
||||||
let tree_overlay_reader =
|
let tree_overlay_reader =
|
||||||
init_required_tile_reader("Trees outside woodland", &tree_overlay_tiles).await?;
|
init_required_tile_reader("Trees outside woodland", &cli.tree_overlay_tiles).await?;
|
||||||
let property_border_reader =
|
let property_border_reader =
|
||||||
init_required_tile_reader("Property borders", &property_border_tiles).await?;
|
init_required_tile_reader("Property borders", &cli.property_border_tiles).await?;
|
||||||
|
|
||||||
let feature_name_to_index: rustc_hash::FxHashMap<String, usize> = property_data
|
let feature_name_to_index: rustc_hash::FxHashMap<String, usize> = property_data
|
||||||
.feature_names
|
.feature_names
|
||||||
|
|
@ -720,7 +663,8 @@ async fn main() -> anyhow::Result<()> {
|
||||||
let superuser_token_cache = Arc::new(pocketbase::SuperuserTokenCache::new());
|
let superuser_token_cache = Arc::new(pocketbase::SuperuserTokenCache::new());
|
||||||
let share_cache = Arc::new(licensing::ShareBoundsCache::new());
|
let share_cache = Arc::new(licensing::ShareBoundsCache::new());
|
||||||
|
|
||||||
let actual_listings = if let Some(path) = cli.actual_listings_path.as_ref() {
|
let actual_listings = {
|
||||||
|
let path = &cli.actual_listings_path;
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
bail!("Actual listings parquet not found: {}", path.display());
|
bail!("Actual listings parquet not found: {}", path.display());
|
||||||
}
|
}
|
||||||
|
|
@ -728,22 +672,17 @@ async fn main() -> anyhow::Result<()> {
|
||||||
let listings = data::ActualListingData::load(path, &property_data)?;
|
let listings = data::ActualListingData::load(path, &property_data)?;
|
||||||
trim_allocator("actual listings load");
|
trim_allocator("actual listings load");
|
||||||
info!(rows = listings.lat.len(), "Actual listings loaded");
|
info!(rows = listings.lat.len(), "Actual listings loaded");
|
||||||
Some(Arc::new(listings))
|
Arc::new(listings)
|
||||||
} else {
|
|
||||||
info!("ACTUAL_LISTINGS_PATH not set; live listings overlay disabled");
|
|
||||||
None
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let crime_by_year = if let Some(path) = cli.crime_by_year_path.as_ref() {
|
let crime_by_year = {
|
||||||
|
let path = &cli.crime_by_year_path;
|
||||||
if !path.exists() {
|
if !path.exists() {
|
||||||
bail!("Crime-by-year parquet not found: {}", path.display());
|
bail!("Crime-by-year parquet not found: {}", path.display());
|
||||||
}
|
}
|
||||||
let data = data::CrimeByYearData::load(path)?;
|
let data = data::CrimeByYearData::load(path)?;
|
||||||
trim_allocator("crime-by-year load");
|
trim_allocator("crime-by-year load");
|
||||||
Arc::new(data)
|
Arc::new(data)
|
||||||
} else {
|
|
||||||
info!("CRIME_BY_YEAR_PATH not set; crime-over-time chart disabled");
|
|
||||||
Arc::new(data::CrimeByYearData::empty())
|
|
||||||
};
|
};
|
||||||
|
|
||||||
let app_state = AppState {
|
let app_state = AppState {
|
||||||
|
|
|
||||||
|
|
@ -65,14 +65,7 @@ pub async fn get_actual_listings(
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
let Some(actual_listings) = state.actual_listings.clone() else {
|
let actual_listings = state.actual_listings.clone();
|
||||||
return Ok(Json(ActualListingsResponse {
|
|
||||||
listings: Vec::new(),
|
|
||||||
total: 0,
|
|
||||||
offset,
|
|
||||||
truncated: false,
|
|
||||||
}));
|
|
||||||
};
|
|
||||||
let (south, west, north, east) =
|
let (south, west, north, east) =
|
||||||
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
require_bounds(params.bounds).map_err(IntoResponse::into_response)?;
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -44,10 +44,9 @@ pub struct AppState {
|
||||||
pub poi_category_groups: Arc<Vec<POICategoryGroup>>,
|
pub poi_category_groups: Arc<Vec<POICategoryGroup>>,
|
||||||
/// Precomputed travel time data store
|
/// Precomputed travel time data store
|
||||||
pub travel_time_store: Arc<TravelTimeStore>,
|
pub travel_time_store: Arc<TravelTimeStore>,
|
||||||
/// Optional real-world listings (e.g. Rightmove / Zoopla data) loaded from ACTUAL_LISTINGS_PATH.
|
/// Real-world listings (e.g. Rightmove / Zoopla data) loaded from ACTUAL_LISTINGS_PATH.
|
||||||
pub actual_listings: Option<Arc<ActualListingData>>,
|
pub actual_listings: Arc<ActualListingData>,
|
||||||
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
|
/// Per-LSOA per-year crime counts used by the right pane to plot trends.
|
||||||
/// Empty when the side parquet was not supplied.
|
|
||||||
pub crime_by_year: Arc<CrimeByYearData>,
|
pub crime_by_year: Arc<CrimeByYearData>,
|
||||||
/// Token validation cache (60s TTL)
|
/// Token validation cache (60s TTL)
|
||||||
pub token_cache: Arc<TokenCache>,
|
pub token_cache: Arc<TokenCache>,
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue