340 lines
10 KiB
Python
340 lines
10 KiB
Python
import polars as pl
|
|
|
|
from pipeline.utils import fuzzy_join_on_postcode, normalize_postcode_key
|
|
from pipeline.utils.fuzzy_join import _numbers_compatible
|
|
|
|
|
|
def test_fuzzy_join_on_postcode_matches_addresses_within_postcode():
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_id": ["flat", "house", "unmatched"],
|
|
"left_address": [
|
|
"Flat 2, 10 High Street",
|
|
"12 High Street",
|
|
"99 Other Road",
|
|
],
|
|
"left_postcode": ["AB1 2CD", "AB1 2CD", "AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_id": ["flat_epc", "house_epc", "other_postcode"],
|
|
"right_address": [
|
|
"10 HIGH STREET FLAT 2",
|
|
"12 High-Street",
|
|
"99 Other Road",
|
|
],
|
|
"right_postcode": [" AB1 2CD ", "AB1 2CD", "ZZ9 9ZZ"],
|
|
}
|
|
)
|
|
|
|
result = (
|
|
fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
)
|
|
.sort("left_id")
|
|
.collect()
|
|
)
|
|
|
|
assert result.select("left_id", "right_id").to_dicts() == [
|
|
{"left_id": "flat", "right_id": "flat_epc"},
|
|
{"left_id": "house", "right_id": "house_epc"},
|
|
{"left_id": "unmatched", "right_id": None},
|
|
]
|
|
|
|
|
|
def test_fuzzy_join_on_postcode_requires_matching_numbers():
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["10 High Street"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["11 High Street"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == [None]
|
|
|
|
|
|
def test_fuzzy_join_on_postcode_rejects_low_score_same_number_matches():
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["1 Example Street"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["1 Totally Different Road"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == [None]
|
|
|
|
|
|
def test_fuzzy_join_on_postcode_rejects_blank_and_invalid_match_keys():
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_id": ["blank", "number_only", "valid"],
|
|
"left_address": [" ", "10", "10 High Street"],
|
|
"left_postcode": ["AB1 2CD", "AB1 2CD", "AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["", "10", "10 High Street"],
|
|
"right_postcode": ["AB1 2CD", "AB1 2CD", "AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = (
|
|
fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
)
|
|
.sort("left_id")
|
|
.collect()
|
|
)
|
|
|
|
assert result.select("left_id", "right_address").to_dicts() == [
|
|
{"left_id": "blank", "right_address": None},
|
|
{"left_id": "number_only", "right_address": None},
|
|
{"left_id": "valid", "right_address": "10 High Street"},
|
|
]
|
|
|
|
|
|
def test_fuzzy_join_rejects_mid_score_number_less_match():
|
|
# "THE COACH HOUSE" vs "THE OLD COACH HOUSE" scores 88 via token_sort_ratio:
|
|
# above the old MIN_FUZZY_SCORE of 60 (so it used to falsely match) but below
|
|
# the number-less threshold of 90, so it must NOT match now.
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["The Coach House"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["The Old Coach House"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == [None]
|
|
|
|
|
|
def test_fuzzy_join_matches_numbered_pair_at_baseline_threshold():
|
|
# "10 ACACIA AVENUE" vs "FLAT A 10 ACACIA AVENUE" scores exactly 82 and the
|
|
# house number is compatible, so the numbered baseline (>= 82) still matches.
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["10 Acacia Avenue"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["Flat A, 10 Acacia Avenue"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == ["Flat A, 10 Acacia Avenue"]
|
|
|
|
|
|
def test_fuzzy_join_matches_high_score_number_less_pair():
|
|
# A number-less pair that clears the 90 threshold (here an exact token match,
|
|
# score 100) must still match.
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["The Old Rectory"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["THE OLD RECTORY"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == ["THE OLD RECTORY"]
|
|
|
|
|
|
def test_numbers_compatible_treats_letter_suffix_as_part_of_the_number():
|
|
# 8A, 8B and plain 8 are three different properties on the same street;
|
|
# digit-only extraction collapsed all three to {8} and let them match.
|
|
assert not _numbers_compatible("8A HIGH STREET", "8B HIGH STREET")
|
|
assert not _numbers_compatible("8A HIGH STREET", "8 HIGH STREET")
|
|
assert _numbers_compatible("8A HIGH STREET", "8A HIGH STREET")
|
|
|
|
|
|
def test_numbers_compatible_requires_equal_sets_not_subset():
|
|
# Subset logic let the whole-building record "188 ..." absorb its flat
|
|
# "FLAT 1 188 ..." ({188} is a subset of {1, 188}); the sets must be equal.
|
|
assert not _numbers_compatible("FLAT 1 188 GREAT NORTH WAY", "188 GREAT NORTH WAY")
|
|
assert _numbers_compatible(
|
|
"FLAT 1 188 GREAT NORTH WAY", "188 GREAT NORTH WAY FLAT 1"
|
|
)
|
|
|
|
|
|
def test_numbers_compatible_number_less_and_one_sided_pairs():
|
|
# Neither side numbered -> compatible (gated by the stricter no-numbers
|
|
# score threshold instead); exactly one side numbered -> incompatible.
|
|
assert _numbers_compatible("ROSE COTTAGE", "ROSE COTTAGE")
|
|
assert not _numbers_compatible("ROSE COTTAGE", "8 HIGH STREET")
|
|
|
|
|
|
def test_fuzzy_join_rejects_wrong_letter_suffix_match():
|
|
# End-to-end guard for the 8A/8B class of wrong-property matches: the only
|
|
# candidate in the postcode bucket differs solely in the number suffix, so
|
|
# the row must stay unmatched rather than borrow the neighbour's record.
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_address": ["8A High Street"],
|
|
"left_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": ["8B High Street"],
|
|
"right_postcode": ["AB1 2CD"],
|
|
}
|
|
)
|
|
|
|
result = fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
).collect()
|
|
|
|
assert result["right_address"].to_list() == [None]
|
|
|
|
|
|
def test_fuzzy_join_emits_match_score_column():
|
|
# The audit column carries the token_sort_ratio of the accepted match:
|
|
# 100 for an exact (post-normalisation) address match, the raw fuzzy score
|
|
# otherwise, and null for unmatched rows.
|
|
left = pl.LazyFrame(
|
|
{
|
|
"left_id": ["exact", "fuzzy", "unmatched"],
|
|
"left_address": [
|
|
"10 High Street",
|
|
"10 Acacia Avenue",
|
|
"99 Other Road",
|
|
],
|
|
"left_postcode": ["AB1 2CD", "EF3 4GH", "ZZ9 9ZZ"],
|
|
}
|
|
)
|
|
right = pl.LazyFrame(
|
|
{
|
|
"right_address": [
|
|
"10 HIGH STREET",
|
|
# Scores exactly 82 against "10 Acacia Avenue" (see
|
|
# test_fuzzy_join_matches_numbered_pair_at_baseline_threshold).
|
|
"Flat A, 10 Acacia Avenue",
|
|
],
|
|
"right_postcode": ["AB1 2CD", "EF3 4GH"],
|
|
}
|
|
)
|
|
|
|
result = (
|
|
fuzzy_join_on_postcode(
|
|
left=left,
|
|
right=right,
|
|
left_address_col="left_address",
|
|
right_address_col="right_address",
|
|
left_postcode_col="left_postcode",
|
|
right_postcode_col="right_postcode",
|
|
)
|
|
.sort("left_id")
|
|
.collect()
|
|
)
|
|
|
|
assert result.schema["_match_score"] == pl.UInt8
|
|
assert result.select("left_id", "_match_score").to_dicts() == [
|
|
{"left_id": "exact", "_match_score": 100},
|
|
{"left_id": "fuzzy", "_match_score": 82},
|
|
{"left_id": "unmatched", "_match_score": None},
|
|
]
|
|
|
|
|
|
def test_normalize_postcode_key_requires_full_postcode():
|
|
df = pl.DataFrame(
|
|
{
|
|
"postcode": [
|
|
" SW1A 1AA ",
|
|
"sw1a-1aa",
|
|
"",
|
|
"SW1A",
|
|
"12345",
|
|
"not a postcode",
|
|
]
|
|
}
|
|
)
|
|
|
|
result = df.select(normalize_postcode_key(pl.col("postcode")).alias("key"))
|
|
|
|
assert result["key"].to_list() == ["SW1A1AA", "SW1A1AA", None, None, None, None]
|