Fun changes

pipeline/download/election_results.py

@@ -0,0 +1,104 @@
+import argparse
+from pathlib import Path
+
+import httpx
+import polars as pl
+
+# UK Parliament publishes candidate-level results for the 2024 General Election.
+# One row per candidate per constituency — we aggregate to per-constituency stats.
+URL = "https://electionresults.parliament.uk/general-elections/6/candidacies.csv"
+
+# Map party names to a smaller set for the enum feature and vote share columns.
+# Only parties that won seats in England are kept; the rest become "Other parties".
+PARTY_MAP = {
+    "Labour": "Labour",
+    "Conservative": "Conservative",
+    "Liberal Democrat": "Liberal Democrat",
+    "Reform UK": "Reform UK",
+    "Green Party": "Green",
+}
+
+
+def download_and_convert(output_path: Path) -> None:
+    print("Downloading 2024 General Election results...")
+    response = httpx.get(URL, follow_redirects=True, timeout=60)
+    response.raise_for_status()
+
+    df = pl.read_csv(response.content)
+    print(f"Raw shape: {df.shape}")
+
+    # Filter to England only (constituency codes starting with E14)
+    df = df.filter(pl.col("Constituency geographic code").str.starts_with("E14"))
+
+    # Map party names to our output groups
+    df = df.with_columns(
+        pl.col("Main party name")
+        .replace_strict(PARTY_MAP, default="Other parties")
+        .alias("party_group"),
+    )
+
+    # ── Per-constituency winner stats ──
+    winners = df.filter(pl.col("Candidate result position") == 1).select(
+        pl.col("Constituency geographic code").alias("pcon"),
+        pl.col("party_group").alias("winning_party"),
+        (pl.col("Majority") / pl.col("Election valid vote count") * 100)
+        .round(1)
+        .alias("majority_pct"),
+        (pl.col("Election valid vote count") / pl.col("Electorate") * 100)
+        .round(1)
+        .alias("turnout_pct"),
+    )
+
+    # ── Per-party vote share percentages ──
+    # Sum votes per party group per constituency, then pivot to wide format
+    party_votes = (
+        df.group_by("Constituency geographic code", "party_group")
+        .agg(pl.col("Candidate vote count").sum())
+        .rename({"Constituency geographic code": "pcon"})
+    )
+    total_votes = (
+        df.group_by("Constituency geographic code")
+        .agg(pl.col("Candidate vote count").sum().alias("total_votes"))
+        .rename({"Constituency geographic code": "pcon"})
+    )
+    party_pct = (
+        party_votes.join(total_votes, on="pcon")
+        .with_columns(
+            (pl.col("Candidate vote count") / pl.col("total_votes") * 100)
+            .round(1)
+            .alias("vote_pct"),
+        )
+        .pivot(on="party_group", index="pcon", values="vote_pct")
+    )
+
+    # Rename columns to "% Party" format
+    rename_map = {col: f"% {col}" for col in party_pct.columns if col != "pcon"}
+    party_pct = party_pct.rename(rename_map)
+
+    # Join winner stats with party vote shares
+    result = winners.join(party_pct, on="pcon", how="left")
+
+    print(f"Constituencies: {result.height}")
+    print(f"Columns: {result.columns}")
+    print(
+        f"Party breakdown:\n{result['winning_party'].value_counts().sort('count', descending=True)}"
+    )
+
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    result.write_parquet(output_path, compression="zstd")
+    print(f"Saved to {output_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(
+        description="Download 2024 General Election results by constituency"
+    )
+    parser.add_argument(
+        "--output", type=Path, required=True, help="Output parquet file path"
+    )
+    args = parser.parse_args()
+    download_and_convert(args.output)
+
+
+if __name__ == "__main__":
+    main()

pipeline/download/ethnicity.py

@@ -57,11 +57,33 @@ def download_and_convert(output_path: Path) -> None:
         pl.col("Ethnicity").replace_strict(group_map).alias("group"),
     )
 
-    # Sum percentages within each group per local authority
-    wide = (
-        detailed.group_by("Geography_code", "group")
-        .agg(pl.col("Value1").sum().round(1))
-        .pivot(on="group", index="Geography_code", values="Value1")
+    # Sum percentages within each group per local authority (keep full precision)
+    grouped = detailed.group_by("Geography_code", "group").agg(pl.col("Value1").sum())
+    wide = grouped.pivot(on="group", index="Geography_code", values="Value1")
+
+    # Normalize so each row sums to exactly 100%, then round using largest-remainder
+    # method to preserve the sum. Independent rounding of 6 values can drift ±0.3.
+    group_cols = [c for c in wide.columns if c != "Geography_code"]
+    row_total = sum(pl.col(c) for c in group_cols)
+    # Scale each group so they sum to exactly 100
+    wide = wide.with_columns(
+        [(pl.col(c) / row_total * 100.0).alias(c) for c in group_cols]
+    )
+    # Round to 1 decimal, then adjust the largest group to absorb residual
+    rounded_cols = [pl.col(c).round(1).alias(c) for c in group_cols]
+    wide = wide.with_columns(rounded_cols)
+    rounded_sum = sum(pl.col(c) for c in group_cols)
+    residual = (100.0 - rounded_sum).round(1)
+    # Find which group is largest per row and add the residual there
+    largest_col = pl.concat_list(group_cols).list.arg_max()
+    wide = wide.with_columns(
+        [
+            pl.when(largest_col == i)
+            .then(pl.col(c) + residual)
+            .otherwise(pl.col(c))
+            .alias(c)
+            for i, c in enumerate(group_cols)
+        ]
     )
 
     # Rename columns to be descriptive

pipeline/download/naptan.py

@@ -17,8 +17,8 @@ STOP_TYPES = {
     "BCT": "Bus stop",
     "BCE": "Bus station",
     "TXR": "Taxi rank",
-    "TMU": "Metro or Tram stop",
-    "MET": "Metro or Tram stop",
+    "TMU": "Tube station",
+    "MET": "Tube station",
 }