All good
This commit is contained in:
parent
6ea544a0f6
commit
6cc7288126
45 changed files with 929 additions and 1043 deletions
|
|
@ -3,6 +3,8 @@ from pathlib import Path
|
|||
import numpy as np
|
||||
import polars as pl
|
||||
|
||||
from pipeline.local_temp import local_tmp_dir
|
||||
|
||||
from .memory import release_memory
|
||||
|
||||
|
||||
|
|
@ -17,7 +19,9 @@ def load_uprns(uprn_path: Path) -> tuple[pl.DataFrame, dict[str, tuple[int, int]
|
|||
print("Loading UPRN lookup...")
|
||||
|
||||
# Sort via streaming sink to avoid polars doubling memory during in-memory sort
|
||||
with tempfile.NamedTemporaryFile(suffix=".parquet", delete=False) as tmp:
|
||||
with tempfile.NamedTemporaryFile(
|
||||
suffix=".parquet", delete=False, dir=local_tmp_dir()
|
||||
) as tmp:
|
||||
tmp_path = Path(tmp.name)
|
||||
(
|
||||
pl.scan_parquet(uprn_path)
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue