Separate chinese

pipeline/download/ethnicity.py

@@ -18,16 +18,50 @@ def download_and_convert(output_path: Path) -> None:
     df = pl.read_csv(response.content)
     print(f"Raw shape: {df.head(100)}")
 
-    # Keep only broad ethnicity categories (5+1), exclude "All" totals
+    # Use the detailed 19+1 breakdown to get sub-categories for Asian ethnicity,
-    df = df.filter(
+    # then aggregate back to the broad groups plus South Asian / East Asian split.
-        (pl.col("Ethnicity_type") == "ONS 2021 5+1") & (pl.col("Ethnicity") != "All")
+    detailed = df.filter(
+        (pl.col("Ethnicity_type") == "ONS 2021 19+1") & (pl.col("Ethnicity") != "All")
     )
 
-    # Pivot: one row per local authority, columns = ethnicity percentages
+    # Map detailed categories to our output groups
-    wide = df.pivot(
+    group_map = {
-        on="Ethnicity",
+        # White
-        index="Geography_code",
+        "White British": "White",
-        values="Value1",
+        "White Irish": "White",
+        "Gypsy Or Irish Traveller": "White",
+        "Roma": "White",
+        "Any Other White Background": "White",
+        # South Asian
+        "Indian": "South Asian",
+        "Pakistani": "South Asian",
+        "Bangladeshi": "South Asian",
+        "Any Other Asian Background": "South Asian",
+        # East Asian
+        "Chinese": "East Asian",
+        # Black
+        "Black African": "Black",
+        "Black Caribbean": "Black",
+        "Any Other Black Background": "Black",
+        # Mixed
+        "Mixed White And Asian": "Mixed",
+        "Mixed White And Black African": "Mixed",
+        "Mixed White And Black Caribbean": "Mixed",
+        "Any Other Mixed/Multiple Ethnic Background": "Mixed",
+        # Other
+        "Arab": "Other",
+        "Any Other Ethnic Background": "Other",
+    }
+
+    detailed = detailed.with_columns(
+        pl.col("Ethnicity").replace_strict(group_map).alias("group"),
+    )
+
+    # Sum percentages within each group per local authority
+    wide = (
+        detailed.group_by("Geography_code", "group")
+        .agg(pl.col("Value1").sum().round(1))
+        .pivot(on="group", index="Geography_code", values="Value1")
     )
 
     # Rename columns to be descriptive