{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# EPC Data Analysis\n",
    "\n",
    "Exploratory analysis of the Energy Performance Certificate (EPC) dataset (~29M rows, converted from CSV to parquet for memory efficiency)."
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:26.543050Z",
     "iopub.status.busy": "2026-01-28T22:06:26.542977Z",
     "iopub.status.idle": "2026-01-28T22:06:27.083569Z",
     "shell.execute_reply": "2026-01-28T22:06:27.083187Z"
    }
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Rows: 28,847,961\n",
      "Columns: LMK_KEY\n",
      "ADDRESS1\n",
      "ADDRESS2\n",
      "ADDRESS3\n",
      "POSTCODE\n",
      "BUILDING_REFERENCE_NUMBER\n",
      "CURRENT_ENERGY_RATING\n",
      "POTENTIAL_ENERGY_RATING\n",
      "CURRENT_ENERGY_EFFICIENCY\n",
      "POTENTIAL_ENERGY_EFFICIENCY\n",
      "PROPERTY_TYPE\n",
      "BUILT_FORM\n",
      "INSPECTION_DATE\n",
      "LOCAL_AUTHORITY\n",
      "CONSTITUENCY\n",
      "COUNTY\n",
      "LODGEMENT_DATE\n",
      "TRANSACTION_TYPE\n",
      "ENVIRONMENT_IMPACT_CURRENT\n",
      "ENVIRONMENT_IMPACT_POTENTIAL\n",
      "ENERGY_CONSUMPTION_CURRENT\n",
      "ENERGY_CONSUMPTION_POTENTIAL\n",
      "CO2_EMISSIONS_CURRENT\n",
      "CO2_EMISS_CURR_PER_FLOOR_AREA\n",
      "CO2_EMISSIONS_POTENTIAL\n",
      "LIGHTING_COST_CURRENT\n",
      "LIGHTING_COST_POTENTIAL\n",
      "HEATING_COST_CURRENT\n",
      "HEATING_COST_POTENTIAL\n",
      "HOT_WATER_COST_CURRENT\n",
      "HOT_WATER_COST_POTENTIAL\n",
      "TOTAL_FLOOR_AREA\n",
      "ENERGY_TARIFF\n",
      "MAINS_GAS_FLAG\n",
      "FLOOR_LEVEL\n",
      "FLAT_TOP_STOREY\n",
      "FLAT_STOREY_COUNT\n",
      "MAIN_HEATING_CONTROLS\n",
      "MULTI_GLAZE_PROPORTION\n",
      "GLAZED_TYPE\n",
      "GLAZED_AREA\n",
      "EXTENSION_COUNT\n",
      "NUMBER_HABITABLE_ROOMS\n",
      "NUMBER_HEATED_ROOMS\n",
      "LOW_ENERGY_LIGHTING\n",
      "NUMBER_OPEN_FIREPLACES\n",
      "HOTWATER_DESCRIPTION\n",
      "HOT_WATER_ENERGY_EFF\n",
      "HOT_WATER_ENV_EFF\n",
      "FLOOR_DESCRIPTION\n",
      "FLOOR_ENERGY_EFF\n",
      "FLOOR_ENV_EFF\n",
      "WINDOWS_DESCRIPTION\n",
      "WINDOWS_ENERGY_EFF\n",
      "WINDOWS_ENV_EFF\n",
      "WALLS_DESCRIPTION\n",
      "WALLS_ENERGY_EFF\n",
      "WALLS_ENV_EFF\n",
      "SECONDHEAT_DESCRIPTION\n",
      "SHEATING_ENERGY_EFF\n",
      "SHEATING_ENV_EFF\n",
      "ROOF_DESCRIPTION\n",
      "ROOF_ENERGY_EFF\n",
      "ROOF_ENV_EFF\n",
      "MAINHEAT_DESCRIPTION\n",
      "MAINHEAT_ENERGY_EFF\n",
      "MAINHEAT_ENV_EFF\n",
      "MAINHEATCONT_DESCRIPTION\n",
      "MAINHEATC_ENERGY_EFF\n",
      "MAINHEATC_ENV_EFF\n",
      "LIGHTING_DESCRIPTION\n",
      "LIGHTING_ENERGY_EFF\n",
      "LIGHTING_ENV_EFF\n",
      "MAIN_FUEL\n",
      "WIND_TURBINE_COUNT\n",
      "HEAT_LOSS_CORRIDOR\n",
      "UNHEATED_CORRIDOR_LENGTH\n",
      "FLOOR_HEIGHT\n",
      "PHOTO_SUPPLY\n",
      "SOLAR_WATER_HEATING_FLAG\n",
      "MECHANICAL_VENTILATION\n",
      "ADDRESS\n",
      "LOCAL_AUTHORITY_LABEL\n",
      "CONSTITUENCY_LABEL\n",
      "POSTTOWN\n",
      "CONSTRUCTION_AGE_BAND\n",
      "LODGEMENT_DATETIME\n",
      "TENURE\n",
      "FIXED_LIGHTING_OUTLETS_COUNT\n",
      "LOW_ENERGY_FIXED_LIGHT_COUNT\n",
      "UPRN\n",
      "UPRN_SOURCE\n",
      "REPORT_TYPE\n"
     ]
    },
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (3, 93)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>LMK_KEY</th><th>ADDRESS1</th><th>ADDRESS2</th><th>ADDRESS3</th><th>POSTCODE</th><th>BUILDING_REFERENCE_NUMBER</th><th>CURRENT_ENERGY_RATING</th><th>POTENTIAL_ENERGY_RATING</th><th>CURRENT_ENERGY_EFFICIENCY</th><th>POTENTIAL_ENERGY_EFFICIENCY</th><th>PROPERTY_TYPE</th><th>BUILT_FORM</th><th>INSPECTION_DATE</th><th>LOCAL_AUTHORITY</th><th>CONSTITUENCY</th><th>COUNTY</th><th>LODGEMENT_DATE</th><th>TRANSACTION_TYPE</th><th>ENVIRONMENT_IMPACT_CURRENT</th><th>ENVIRONMENT_IMPACT_POTENTIAL</th><th>ENERGY_CONSUMPTION_CURRENT</th><th>ENERGY_CONSUMPTION_POTENTIAL</th><th>CO2_EMISSIONS_CURRENT</th><th>CO2_EMISS_CURR_PER_FLOOR_AREA</th><th>CO2_EMISSIONS_POTENTIAL</th><th>LIGHTING_COST_CURRENT</th><th>LIGHTING_COST_POTENTIAL</th><th>HEATING_COST_CURRENT</th><th>HEATING_COST_POTENTIAL</th><th>HOT_WATER_COST_CURRENT</th><th>HOT_WATER_COST_POTENTIAL</th><th>TOTAL_FLOOR_AREA</th><th>ENERGY_TARIFF</th><th>MAINS_GAS_FLAG</th><th>FLOOR_LEVEL</th><th>FLAT_TOP_STOREY</th><th>FLAT_STOREY_COUNT</th><th>&hellip;</th><th>WALLS_ENERGY_EFF</th><th>WALLS_ENV_EFF</th><th>SECONDHEAT_DESCRIPTION</th><th>SHEATING_ENERGY_EFF</th><th>SHEATING_ENV_EFF</th><th>ROOF_DESCRIPTION</th><th>ROOF_ENERGY_EFF</th><th>ROOF_ENV_EFF</th><th>MAINHEAT_DESCRIPTION</th><th>MAINHEAT_ENERGY_EFF</th><th>MAINHEAT_ENV_EFF</th><th>MAINHEATCONT_DESCRIPTION</th><th>MAINHEATC_ENERGY_EFF</th><th>MAINHEATC_ENV_EFF</th><th>LIGHTING_DESCRIPTION</th><th>LIGHTING_ENERGY_EFF</th><th>LIGHTING_ENV_EFF</th><th>MAIN_FUEL</th><th>WIND_TURBINE_COUNT</th><th>HEAT_LOSS_CORRIDOR</th><th>UNHEATED_CORRIDOR_LENGTH</th><th>FLOOR_HEIGHT</th><th>PHOTO_SUPPLY</th><th>SOLAR_WATER_HEATING_FLAG</th><th>MECHANICAL_VENTILATION</th><th>ADDRESS</th><th>LOCAL_AUTHORITY_LABEL</th><th>CONSTITUENCY_LABEL</th><th>POSTTOWN</th><th>CONSTRUCTION_AGE_BAND</th><th>LODGEMENT_DATETIME</th><th>TENURE</th><th>FIXED_LIGHTING_OUTLETS_COUNT</th><th>LOW_ENERGY_FIXED_LIGHT_COUNT</th><th>UPRN</th><th>UPRN_SOURCE</th><th>REPORT_TYPE</th></tr><tr><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>str</td><td>i64</td><td>i64</td><td>str</td><td>str</td><td>date</td><td>str</td><td>str</td><td>str</td><td>date</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>i64</td><td>f64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>i64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>f64</td><td>&hellip;</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>i64</td><td>str</td><td>f64</td><td>f64</td><td>f64</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>str</td><td>datetime[μs]</td><td>str</td><td>i64</td><td>i64</td><td>i64</td><td>str</td><td>i64</td></tr></thead><tbody><tr><td>&quot;0013ac0dc0f80dd06448efce74287d…</td><td>&quot;43 GAYAL CROFT&quot;</td><td>&quot;MILTON KEYNES&quot;</td><td>null</td><td>&quot;MK5 7HX&quot;</td><td>10003769440</td><td>&quot;C&quot;</td><td>&quot;B&quot;</td><td>75</td><td>89</td><td>&quot;House&quot;</td><td>&quot;Mid-Terrace&quot;</td><td>2022-08-22</td><td>&quot;E06000042&quot;</td><td>&quot;E14000822&quot;</td><td>null</td><td>2022-12-08</td><td>&quot;rental&quot;</td><td>75</td><td>90</td><td>175</td><td>61</td><td>1.9</td><td>31</td><td>0.7</td><td>55</td><td>44</td><td>340</td><td>342</td><td>98</td><td>63</td><td>61.0</td><td>&quot;Single&quot;</td><td>&quot;Y&quot;</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;None&quot;</td><td>&quot;N/A&quot;</td><td>&quot;N/A&quot;</td><td>&quot;Pitched, insulated (assumed)&quot;</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;Boiler and radiators, mains ga…</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;Programmer, room thermostat an…</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;Low energy lighting in 75% of …</td><td>&quot;Very Good&quot;</td><td>&quot;Very Good&quot;</td><td>&quot;mains gas (not community)&quot;</td><td>0</td><td>null</td><td>null</td><td>2.4</td><td>0.0</td><td>&quot;N&quot;</td><td>&quot;natural&quot;</td><td>&quot;43 GAYAL CROFT, MILTON KEYNES&quot;</td><td>&quot;Milton Keynes&quot;</td><td>&quot;Milton Keynes South&quot;</td><td>null</td><td>&quot;England and Wales: 1996-2002&quot;</td><td>2022-12-08 22:29:20</td><td>&quot;Rented (social)&quot;</td><td>8</td><td>null</td><td>25018528</td><td>&quot;Energy Assessor&quot;</td><td>100</td></tr><tr><td>&quot;20daddd4e5ce9abcca2efab2270af5…</td><td>&quot;30 Ditton Road&quot;</td><td>null</td><td>null</td><td>&quot;DA6 8JL&quot;</td><td>10003624703</td><td>&quot;D&quot;</td><td>&quot;B&quot;</td><td>59</td><td>84</td><td>&quot;House&quot;</td><td>&quot;Mid-Terrace&quot;</td><td>2022-11-03</td><td>&quot;E09000004&quot;</td><td>&quot;E14000558&quot;</td><td>null</td><td>2022-11-03</td><td>&quot;marketed sale&quot;</td><td>51</td><td>81</td><td>279</td><td>93</td><td>4.6</td><td>49</td><td>1.6</td><td>80</td><td>80</td><td>755</td><td>452</td><td>117</td><td>68</td><td>93.0</td><td>&quot;Unknown&quot;</td><td>&quot;Y&quot;</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>&quot;Poor&quot;</td><td>&quot;Poor&quot;</td><td>&quot;None&quot;</td><td>&quot;N/A&quot;</td><td>&quot;N/A&quot;</td><td>&quot;Pitched, 100 mm loft insulatio…</td><td>&quot;Average&quot;</td><td>&quot;Average&quot;</td><td>&quot;Boiler and radiators, mains ga…</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;Programmer and room thermostat&quot;</td><td>&quot;Average&quot;</td><td>&quot;Average&quot;</td><td>&quot;Low energy lighting in all fix…</td><td>&quot;Very Good&quot;</td><td>&quot;Very Good&quot;</td><td>&quot;mains gas (not community)&quot;</td><td>0</td><td>null</td><td>null</td><td>2.64</td><td>0.0</td><td>&quot;N&quot;</td><td>&quot;natural&quot;</td><td>&quot;30 Ditton Road&quot;</td><td>&quot;Bexley&quot;</td><td>&quot;Bexleyheath and Crayford&quot;</td><td>&quot;BEXLEYHEATH&quot;</td><td>&quot;England and Wales: 1950-1966&quot;</td><td>2022-11-03 16:38:48</td><td>&quot;Owner-occupied&quot;</td><td>11</td><td>null</td><td>100020205591</td><td>&quot;Energy Assessor&quot;</td><td>100</td></tr><tr><td>&quot;0019793879cf7ab92be22ba552e992…</td><td>&quot;The Paddocks&quot;</td><td>&quot;Bridge Road&quot;</td><td>&quot;Stoke Bruerne&quot;</td><td>&quot;NN12 7SE&quot;</td><td>10003816660</td><td>&quot;D&quot;</td><td>&quot;C&quot;</td><td>65</td><td>74</td><td>&quot;House&quot;</td><td>&quot;Detached&quot;</td><td>2022-12-16</td><td>&quot;E06000062&quot;</td><td>&quot;E14000942&quot;</td><td>null</td><td>2022-12-16</td><td>&quot;marketed sale&quot;</td><td>57</td><td>67</td><td>146</td><td>103</td><td>8.3</td><td>38</td><td>6.4</td><td>130</td><td>130</td><td>1008</td><td>959</td><td>184</td><td>93</td><td>217.0</td><td>&quot;Single&quot;</td><td>&quot;N&quot;</td><td>null</td><td>null</td><td>null</td><td>&hellip;</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;None&quot;</td><td>&quot;N/A&quot;</td><td>&quot;N/A&quot;</td><td>&quot;Pitched, insulated at rafters&quot;</td><td>&quot;Average&quot;</td><td>&quot;Average&quot;</td><td>&quot;Boiler and radiators, oil&quot;</td><td>&quot;Average&quot;</td><td>&quot;Good&quot;</td><td>&quot;Programmer, room thermostat an…</td><td>&quot;Good&quot;</td><td>&quot;Good&quot;</td><td>&quot;Low energy lighting in all fix…</td><td>&quot;Very Good&quot;</td><td>&quot;Very Good&quot;</td><td>&quot;oil (not community)&quot;</td><td>0</td><td>null</td><td>null</td><td>2.36</td><td>0.0</td><td>&quot;N&quot;</td><td>&quot;natural&quot;</td><td>&quot;The Paddocks, Bridge Road, Sto…</td><td>&quot;West Northamptonshire&quot;</td><td>&quot;South Northamptonshire&quot;</td><td>&quot;TOWCESTER&quot;</td><td>&quot;England and Wales: 1983-1990&quot;</td><td>2022-12-16 10:47:55</td><td>&quot;Owner-occupied&quot;</td><td>22</td><td>null</td><td>10023964427</td><td>&quot;Energy Assessor&quot;</td><td>100</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (3, 93)\n",
       "┌───────────┬───────────┬───────────┬───────────┬───┬───────────┬───────────┬───────────┬──────────┐\n",
       "│ LMK_KEY   ┆ ADDRESS1  ┆ ADDRESS2  ┆ ADDRESS3  ┆ … ┆ LOW_ENERG ┆ UPRN      ┆ UPRN_SOUR ┆ REPORT_T │\n",
       "│ ---       ┆ ---       ┆ ---       ┆ ---       ┆   ┆ Y_FIXED_L ┆ ---       ┆ CE        ┆ YPE      │\n",
       "│ str       ┆ str       ┆ str       ┆ str       ┆   ┆ IGHT_COUN ┆ i64       ┆ ---       ┆ ---      │\n",
       "│           ┆           ┆           ┆           ┆   ┆ T         ┆           ┆ str       ┆ i64      │\n",
       "│           ┆           ┆           ┆           ┆   ┆ ---       ┆           ┆           ┆          │\n",
       "│           ┆           ┆           ┆           ┆   ┆ i64       ┆           ┆           ┆          │\n",
       "╞═══════════╪═══════════╪═══════════╪═══════════╪═══╪═══════════╪═══════════╪═══════════╪══════════╡\n",
       "│ 0013ac0dc ┆ 43 GAYAL  ┆ MILTON    ┆ null      ┆ … ┆ null      ┆ 25018528  ┆ Energy    ┆ 100      │\n",
       "│ 0f80dd064 ┆ CROFT     ┆ KEYNES    ┆           ┆   ┆           ┆           ┆ Assessor  ┆          │\n",
       "│ 48efce742 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ 87d…      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ 20daddd4e ┆ 30 Ditton ┆ null      ┆ null      ┆ … ┆ null      ┆ 100020205 ┆ Energy    ┆ 100      │\n",
       "│ 5ce9abcca ┆ Road      ┆           ┆           ┆   ┆           ┆ 591       ┆ Assessor  ┆          │\n",
       "│ 2efab2270 ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ af5…      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ 001979387 ┆ The       ┆ Bridge    ┆ Stoke     ┆ … ┆ null      ┆ 100239644 ┆ Energy    ┆ 100      │\n",
       "│ 9cf7ab92b ┆ Paddocks  ┆ Road      ┆ Bruerne   ┆   ┆           ┆ 27        ┆ Assessor  ┆          │\n",
       "│ e22ba552e ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "│ 992…      ┆           ┆           ┆           ┆   ┆           ┆           ┆           ┆          │\n",
       "└───────────┴───────────┴───────────┴───────────┴───┴───────────┴───────────┴───────────┴──────────┘"
      ]
     },
     "execution_count": 1,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "import polars as pl\n",
    "import matplotlib.pyplot as plt\n",
    "import matplotlib.ticker as ticker\n",
    "\n",
    "plt.rcParams[\"figure.figsize\"] = (12, 5)\n",
    "plt.rcParams[\"figure.dpi\"] = 100\n",
    "\n",
    "PARQUET = \"../data/epc/certificates.parquet\"\n",
    "\n",
    "\n",
    "def scan():\n",
    "    return pl.scan_parquet(PARQUET)\n",
    "\n",
    "\n",
    "# Quick row count\n",
    "row_count = scan().select(pl.len()).collect().item()\n",
    "print(f\"Rows: {row_count:,}\")\n",
    "print(f\"Columns: {'\\n'.join(scan().collect_schema().names())}\")\n",
    "\n",
    "scan().head(3).collect()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Schema Overview"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:27.101365Z",
     "iopub.status.busy": "2026-01-28T22:06:27.101128Z",
     "iopub.status.idle": "2026-01-28T22:06:29.053942Z",
     "shell.execute_reply": "2026-01-28T22:06:29.053508Z"
    }
   },
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div><style>\n",
       ".dataframe > thead > tr,\n",
       ".dataframe > tbody > tr {\n",
       "  text-align: right;\n",
       "  white-space: pre-wrap;\n",
       "}\n",
       "</style>\n",
       "<small>shape: (93, 4)</small><table border=\"1\" class=\"dataframe\"><thead><tr><th>column</th><th>dtype</th><th>null_count</th><th>null_pct</th></tr><tr><td>str</td><td>str</td><td>i64</td><td>f64</td></tr></thead><tbody><tr><td>&quot;LMK_KEY&quot;</td><td>&quot;String&quot;</td><td>0</td><td>0.0</td></tr><tr><td>&quot;ADDRESS1&quot;</td><td>&quot;String&quot;</td><td>49</td><td>0.0</td></tr><tr><td>&quot;ADDRESS2&quot;</td><td>&quot;String&quot;</td><td>14135472</td><td>49.0</td></tr><tr><td>&quot;ADDRESS3&quot;</td><td>&quot;String&quot;</td><td>26056812</td><td>90.3</td></tr><tr><td>&quot;POSTCODE&quot;</td><td>&quot;String&quot;</td><td>0</td><td>0.0</td></tr><tr><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td><td>&hellip;</td></tr><tr><td>&quot;FIXED_LIGHTING_OUTLETS_COUNT&quot;</td><td>&quot;Int64&quot;</td><td>11903726</td><td>41.3</td></tr><tr><td>&quot;LOW_ENERGY_FIXED_LIGHT_COUNT&quot;</td><td>&quot;Int64&quot;</td><td>19926976</td><td>69.1</td></tr><tr><td>&quot;UPRN&quot;</td><td>&quot;Int64&quot;</td><td>646277</td><td>2.2</td></tr><tr><td>&quot;UPRN_SOURCE&quot;</td><td>&quot;String&quot;</td><td>646277</td><td>2.2</td></tr><tr><td>&quot;REPORT_TYPE&quot;</td><td>&quot;Int64&quot;</td><td>0</td><td>0.0</td></tr></tbody></table></div>"
      ],
      "text/plain": [
       "shape: (93, 4)\n",
       "┌──────────────────────────────┬────────┬────────────┬──────────┐\n",
       "│ column                       ┆ dtype  ┆ null_count ┆ null_pct │\n",
       "│ ---                          ┆ ---    ┆ ---        ┆ ---      │\n",
       "│ str                          ┆ str    ┆ i64        ┆ f64      │\n",
       "╞══════════════════════════════╪════════╪════════════╪══════════╡\n",
       "│ LMK_KEY                      ┆ String ┆ 0          ┆ 0.0      │\n",
       "│ ADDRESS1                     ┆ String ┆ 49         ┆ 0.0      │\n",
       "│ ADDRESS2                     ┆ String ┆ 14135472   ┆ 49.0     │\n",
       "│ ADDRESS3                     ┆ String ┆ 26056812   ┆ 90.3     │\n",
       "│ POSTCODE                     ┆ String ┆ 0          ┆ 0.0      │\n",
       "│ …                            ┆ …      ┆ …          ┆ …        │\n",
       "│ FIXED_LIGHTING_OUTLETS_COUNT ┆ Int64  ┆ 11903726   ┆ 41.3     │\n",
       "│ LOW_ENERGY_FIXED_LIGHT_COUNT ┆ Int64  ┆ 19926976   ┆ 69.1     │\n",
       "│ UPRN                         ┆ Int64  ┆ 646277     ┆ 2.2      │\n",
       "│ UPRN_SOURCE                  ┆ String ┆ 646277     ┆ 2.2      │\n",
       "│ REPORT_TYPE                  ┆ Int64  ┆ 0          ┆ 0.0      │\n",
       "└──────────────────────────────┴────────┴────────────┴──────────┘"
      ]
     },
     "execution_count": 2,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "schema = scan().collect_schema()\n",
    "\n",
    "# Compute null counts in batches of columns to avoid OOM\n",
    "batch_size = 10\n",
    "all_nulls = {}\n",
    "col_names = schema.names()\n",
    "for i in range(0, len(col_names), batch_size):\n",
    "    batch = col_names[i : i + batch_size]\n",
    "    result = scan().select([pl.col(c).null_count().alias(c) for c in batch]).collect()\n",
    "    for c in batch:\n",
    "        all_nulls[c] = result[c].item()\n",
    "\n",
    "schema_info = pl.DataFrame(\n",
    "    {\n",
    "        \"column\": col_names,\n",
    "        \"dtype\": [str(d) for d in schema.dtypes()],\n",
    "        \"null_count\": [all_nulls[c] for c in col_names],\n",
    "        \"null_pct\": [round(all_nulls[c] / row_count * 100, 1) for c in col_names],\n",
    "    }\n",
    ")\n",
    "schema_info"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Energy Rating Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:29.054902Z",
     "iopub.status.busy": "2026-01-28T22:06:29.054824Z",
     "iopub.status.idle": "2026-01-28T22:06:29.241274Z",
     "shell.execute_reply": "2026-01-28T22:06:29.240893Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 35080 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "rating_order = [\"A\", \"B\", \"C\", \"D\", \"E\", \"F\", \"G\"]\n",
    "rating_counts = (\n",
    "    scan()\n",
    "    .filter(pl.col(\"CURRENT_ENERGY_RATING\").is_in(rating_order))\n",
    "    .group_by(\"CURRENT_ENERGY_RATING\")\n",
    "    .len()\n",
    "    .collect()\n",
    ")\n",
    "rc = {r[\"CURRENT_ENERGY_RATING\"]: r[\"len\"] for r in rating_counts.iter_rows(named=True)}\n",
    "\n",
    "colors = [\"#1a9641\", \"#52b947\", \"#a6d96a\", \"#fee08b\", \"#f09e4a\", \"#d7191c\", \"#a50026\"]\n",
    "values = [rc.get(r, 0) for r in rating_order]\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "bars = ax.bar(rating_order, values, color=colors)\n",
    "ax.set_xlabel(\"Current Energy Rating\")\n",
    "ax.set_ylabel(\"Number of Properties\")\n",
    "ax.set_title(\"Distribution of Current Energy Ratings\")\n",
    "ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f\"{x / 1e6:.1f}M\"))\n",
    "for bar, v in zip(bars, values):\n",
    "    ax.text(\n",
    "        bar.get_x() + bar.get_width() / 2,\n",
    "        bar.get_height(),\n",
    "        f\"{v / 1e6:.1f}M\",\n",
    "        ha=\"center\",\n",
    "        va=\"bottom\",\n",
    "        fontsize=9,\n",
    "    )\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Property Type Breakdown"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:29.243014Z",
     "iopub.status.busy": "2026-01-28T22:06:29.242929Z",
     "iopub.status.idle": "2026-01-28T22:06:29.388503Z",
     "shell.execute_reply": "2026-01-28T22:06:29.388068Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 23216 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "prop_type = (\n",
    "    scan().group_by(\"PROPERTY_TYPE\").len().sort(\"len\", descending=True).collect()\n",
    ")\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.barh(prop_type[\"PROPERTY_TYPE\"].to_list()[::-1], prop_type[\"len\"].to_list()[::-1])\n",
    "ax.set_xlabel(\"Count\")\n",
    "ax.set_title(\"Properties by Type\")\n",
    "ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f\"{x / 1e6:.1f}M\"))\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## PROPERTY_TYPE and BUILT_FORM Cardinality"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "PROPERTY_TYPE — 5 distinct values:\n",
      "  House                            17,437,884\n",
      "  Flat                              8,236,696\n",
      "  Bungalow                          2,448,109\n",
      "  Maisonette                          710,695\n",
      "  Park home                            14,577\n",
      "\n",
      "BUILT_FORM — 9 distinct values:\n",
      "  Semi-Detached                     8,777,318\n",
      "  Mid-Terrace                       7,972,697\n",
      "  Detached                          6,428,144\n",
      "  End-Terrace                       4,132,264\n",
      "  NO DATA!                            548,769\n",
      "  Enclosed End-Terrace                450,732\n",
      "  Enclosed Mid-Terrace                377,198\n",
      "  Not Recorded                        155,103\n",
      "  None                                  5,736\n",
      "\n"
     ]
    }
   ],
   "source": [
    "for col_name in [\"PROPERTY_TYPE\", \"BUILT_FORM\"]:\n",
    "    counts = scan().group_by(col_name).len().sort(\"len\", descending=True).collect()\n",
    "    print(f\"{col_name} — {len(counts)} distinct values:\")\n",
    "    for row in counts.iter_rows(named=True):\n",
    "        print(f\"  {row[col_name]!s:30s} {row['len']:>12,}\")\n",
    "    print()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Energy Rating by Construction Age Band"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:29.941986Z",
     "iopub.status.busy": "2026-01-28T22:06:29.941893Z",
     "iopub.status.idle": "2026-01-28T22:06:30.515904Z",
     "shell.execute_reply": "2026-01-28T22:06:30.515522Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 62948 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "age_eff = (\n",
    "    scan()\n",
    "    .filter(pl.col(\"CONSTRUCTION_AGE_BAND\").is_not_null())\n",
    "    .group_by(\"CONSTRUCTION_AGE_BAND\")\n",
    "    .agg(\n",
    "        pl.col(\"CURRENT_ENERGY_EFFICIENCY\").mean().alias(\"mean_efficiency\"),\n",
    "        pl.col(\"CURRENT_ENERGY_EFFICIENCY\").median().alias(\"median_efficiency\"),\n",
    "        pl.len().alias(\"count\"),\n",
    "    )\n",
    "    .filter(pl.col(\"count\") > 10000)\n",
    "    .collect()\n",
    ")\n",
    "\n",
    "# Define sort order for known age bands\n",
    "age_band_order = [\n",
    "    \"before 1900\",\n",
    "    \"1900-1929\",\n",
    "    \"1930-1949\",\n",
    "    \"1950-1966\",\n",
    "    \"1967-1975\",\n",
    "    \"1976-1982\",\n",
    "    \"1983-1990\",\n",
    "    \"1991-1995\",\n",
    "    \"1996-2002\",\n",
    "    \"2003-2006\",\n",
    "    \"2007-2011\",\n",
    "    \"2007 onwards\",\n",
    "    \"2012 onwards\",\n",
    "]\n",
    "\n",
    "\n",
    "def age_sort_key(band):\n",
    "    for i, ab in enumerate(age_band_order):\n",
    "        if ab in str(band):\n",
    "            return i\n",
    "    return 99\n",
    "\n",
    "\n",
    "age_eff = (\n",
    "    age_eff.with_columns(\n",
    "        pl.col(\"CONSTRUCTION_AGE_BAND\")\n",
    "        .map_elements(age_sort_key, return_dtype=pl.Int64)\n",
    "        .alias(\"sort_key\")\n",
    "    )\n",
    "    .sort(\"sort_key\")\n",
    "    .filter(pl.col(\"sort_key\") < 99)\n",
    ")\n",
    "\n",
    "labels = age_eff[\"CONSTRUCTION_AGE_BAND\"].to_list()\n",
    "# Shorten labels\n",
    "short_labels = [label.replace(\"England and Wales: \", \"\") for label in labels]\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(short_labels, age_eff[\"mean_efficiency\"].to_list(), label=\"Mean\", alpha=0.7)\n",
    "ax.bar(short_labels, age_eff[\"median_efficiency\"].to_list(), label=\"Median\", alpha=0.5)\n",
    "ax.set_xlabel(\"Construction Age Band\")\n",
    "ax.set_ylabel(\"Energy Efficiency Score\")\n",
    "ax.set_title(\"Energy Efficiency by Construction Age\")\n",
    "ax.legend()\n",
    "plt.xticks(rotation=45, ha=\"right\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Floor Area Distribution by Property Type"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:31.277179Z",
     "iopub.status.busy": "2026-01-28T22:06:31.277103Z",
     "iopub.status.idle": "2026-01-28T22:06:31.967032Z",
     "shell.execute_reply": "2026-01-28T22:06:31.966642Z"
    }
   },
   "outputs": [
    {
     "name": "stderr",
     "output_type": "stream",
     "text": [
      "/tmp/ipykernel_1997061/992856044.py:36: MatplotlibDeprecationWarning: The 'labels' parameter of boxplot() has been renamed 'tick_labels' since Matplotlib 3.9; support for the old name will be dropped in 3.11.\n",
      "  ax.boxplot(data, labels=labels, vert=True)\n"
     ]
    },
    {
     "data": {
      "image/png": "[encoded data, 39052 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "top_types = (\n",
    "    scan()\n",
    "    .group_by(\"PROPERTY_TYPE\")\n",
    "    .len()\n",
    "    .sort(\"len\", descending=True)\n",
    "    .head(5)\n",
    "    .collect()[\"PROPERTY_TYPE\"]\n",
    "    .to_list()\n",
    ")\n",
    "\n",
    "# Sample per property type to avoid loading all data\n",
    "fig, ax = plt.subplots()\n",
    "data = []\n",
    "labels = []\n",
    "for pt in top_types:\n",
    "    vals = (\n",
    "        scan()\n",
    "        .filter(\n",
    "            (pl.col(\"PROPERTY_TYPE\") == pt)\n",
    "            & pl.col(\"TOTAL_FLOOR_AREA\").is_not_null()\n",
    "            & (pl.col(\"TOTAL_FLOOR_AREA\").cast(pl.Float64, strict=False) > 0)\n",
    "        )\n",
    "        .select(pl.col(\"TOTAL_FLOOR_AREA\").cast(pl.Float64, strict=False))\n",
    "        .collect()\n",
    "        .to_series()\n",
    "        .drop_nulls()\n",
    "    )\n",
    "    p95 = vals.quantile(0.95)\n",
    "    vals = vals.filter(vals < p95)\n",
    "    # Sample if too large\n",
    "    if len(vals) > 100_000:\n",
    "        vals = vals.sample(n=100_000, seed=42)\n",
    "    data.append(vals.to_list())\n",
    "    labels.append(pt)\n",
    "\n",
    "ax.boxplot(data, labels=labels, vert=True)\n",
    "ax.set_ylabel(\"Total Floor Area (m²)\")\n",
    "ax.set_title(\"Floor Area Distribution by Property Type (95th pctl)\")\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Lodgement Trends Over Time"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:31.967996Z",
     "iopub.status.busy": "2026-01-28T22:06:31.967918Z",
     "iopub.status.idle": "2026-01-28T22:06:32.076565Z",
     "shell.execute_reply": "2026-01-28T22:06:32.076202Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 26224 characters]",
      "text/plain": [
       "<Figure size 1200x500 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "lodgements = (\n",
    "    scan()\n",
    "    .filter(pl.col(\"LODGEMENT_DATE\").is_not_null())\n",
    "    .with_columns(pl.col(\"LODGEMENT_DATE\").dt.year().alias(\"year\"))\n",
    "    .filter(pl.col(\"year\").is_between(2008, 2025))\n",
    "    .group_by(\"year\")\n",
    "    .len()\n",
    "    .sort(\"year\")\n",
    "    .collect()\n",
    ")\n",
    "\n",
    "fig, ax = plt.subplots()\n",
    "ax.bar(lodgements[\"year\"].to_list(), lodgements[\"len\"].to_list())\n",
    "ax.set_xlabel(\"Year\")\n",
    "ax.set_ylabel(\"Certificates Lodged\")\n",
    "ax.set_title(\"EPC Lodgements by Year\")\n",
    "ax.yaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f\"{x / 1e6:.1f}M\"))\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## Main Fuel Type Distribution"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {
    "execution": {
     "iopub.execute_input": "2026-01-28T22:06:32.398389Z",
     "iopub.status.busy": "2026-01-28T22:06:32.398313Z",
     "iopub.status.idle": "2026-01-28T22:06:32.909646Z",
     "shell.execute_reply": "2026-01-28T22:06:32.909308Z"
    }
   },
   "outputs": [
    {
     "data": {
      "image/png": "[encoded data, 67644 characters]",
      "text/plain": [
       "<Figure size 1200x600 with 1 Axes>"
      ]
     },
     "metadata": {},
     "output_type": "display_data"
    }
   ],
   "source": [
    "fuel = (\n",
    "    scan()\n",
    "    .filter(pl.col(\"MAIN_FUEL\").is_not_null() & (pl.col(\"MAIN_FUEL\") != \"\"))\n",
    "    .group_by(\"MAIN_FUEL\")\n",
    "    .len()\n",
    "    .sort(\"len\", descending=True)\n",
    "    .head(10)\n",
    "    .collect()\n",
    ")\n",
    "\n",
    "fig, ax = plt.subplots(figsize=(12, 6))\n",
    "ax.barh(fuel[\"MAIN_FUEL\"].to_list()[::-1], fuel[\"len\"].to_list()[::-1])\n",
    "ax.set_xlabel(\"Count\")\n",
    "ax.set_title(\"Top 10 Main Fuel Types\")\n",
    "ax.xaxis.set_major_formatter(ticker.FuncFormatter(lambda x, _: f\"{x / 1e6:.1f}M\"))\n",
    "plt.tight_layout()\n",
    "plt.show()"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "property-map",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.12"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 4
}
