Small clean up
This commit is contained in:
parent
b9f9c3ae50
commit
94ebd0f614
10 changed files with 22 additions and 21 deletions
|
|
@ -100,6 +100,7 @@ services:
|
||||||
environment:
|
environment:
|
||||||
PB_ADMIN_EMAIL: *pb-email
|
PB_ADMIN_EMAIL: *pb-email
|
||||||
PB_ADMIN_PASSWORD: *pb-password
|
PB_ADMIN_PASSWORD: *pb-password
|
||||||
|
PB_TRUSTED_PROXY: server
|
||||||
healthcheck:
|
healthcheck:
|
||||||
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8090/api/health"]
|
test: ["CMD", "wget", "--no-verbose", "--tries=1", "--spider", "http://localhost:8090/api/health"]
|
||||||
interval: 10s
|
interval: 10s
|
||||||
|
|
|
||||||
|
|
@ -14,7 +14,7 @@ SEED = 42
|
||||||
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
# Schedule: hour of day (UTC) to auto-run scrape. Set to -1 to disable.
|
||||||
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
SCHEDULE_HOUR = int(os.environ.get("SCHEDULE_HOUR", "3"))
|
||||||
# Whether to run a scrape immediately on startup
|
# Whether to run a scrape immediately on startup
|
||||||
RUN_ON_STARTUP = os.environ.get("RUN_ON_STARTUP", "true").lower() in ("1", "true", "yes")
|
RUN_ON_STARTUP = False
|
||||||
|
|
||||||
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
TYPEAHEAD_URL = "https://los.rightmove.co.uk/typeahead"
|
||||||
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
SEARCH_URL = "https://www.rightmove.co.uk/api/property-search/listing/search"
|
||||||
|
|
@ -26,12 +26,12 @@ PROPERTY_TYPE_MAP = {
|
||||||
"Terraced": "Terraced",
|
"Terraced": "Terraced",
|
||||||
"End of Terrace": "Terraced",
|
"End of Terrace": "Terraced",
|
||||||
"Mid Terrace": "Terraced",
|
"Mid Terrace": "Terraced",
|
||||||
"Flat": "Flat",
|
"Flat": "Flats/Maisonettes",
|
||||||
"Maisonette": "Flat",
|
"Maisonette": "Flats/Maisonettes",
|
||||||
"Studio": "Flat",
|
"Studio": "Flats/Maisonettes",
|
||||||
"Apartment": "Flat",
|
"Apartment": "Flats/Maisonettes",
|
||||||
"Penthouse": "Flat",
|
"Penthouse": "Flats/Maisonettes",
|
||||||
"Ground Flat": "Flat",
|
"Ground Flat": "Flats/Maisonettes",
|
||||||
"Detached Bungalow": "Detached",
|
"Detached Bungalow": "Detached",
|
||||||
"Semi-Detached Bungalow": "Semi-Detached",
|
"Semi-Detached Bungalow": "Semi-Detached",
|
||||||
"Town House": "Terraced",
|
"Town House": "Terraced",
|
||||||
|
|
@ -49,7 +49,7 @@ PROPERTY_TYPE_MAP = {
|
||||||
"Coach House": "Other",
|
"Coach House": "Other",
|
||||||
"Character Property": "Other",
|
"Character Property": "Other",
|
||||||
"Cluster House": "Other",
|
"Cluster House": "Other",
|
||||||
"Retirement Property": "Flat",
|
"Retirement Property": "Flats/Maisonettes",
|
||||||
"Plot": "Other",
|
"Plot": "Other",
|
||||||
"Garages": "Other",
|
"Garages": "Other",
|
||||||
"Mews": "Terraced",
|
"Mews": "Terraced",
|
||||||
|
|
|
||||||
|
|
@ -149,7 +149,7 @@ export const STACKED_ENUM_GROUPS: Record<
|
||||||
label: 'Property type',
|
label: 'Property type',
|
||||||
feature: 'Property type',
|
feature: 'Property type',
|
||||||
components: ['Property type'],
|
components: ['Property type'],
|
||||||
valueOrder: ['Detached', 'Semi-Detached', 'Terraced', 'Flat'],
|
valueOrder: ['Detached', 'Semi-Detached', 'Terraced', 'Flats/Maisonettes'],
|
||||||
valueColors: ['#8b5cf6', '#3b82f6', '#14b8a6', '#f59e0b'],
|
valueColors: ['#8b5cf6', '#3b82f6', '#14b8a6', '#f59e0b'],
|
||||||
},
|
},
|
||||||
{
|
{
|
||||||
|
|
|
||||||
|
|
@ -21,8 +21,7 @@ const PROPERTY_TYPE_MAP: Record<
|
||||||
'End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
'End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||||
'Enclosed Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
'Enclosed Mid-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||||
'Enclosed End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
'Enclosed End-Terrace': { rightmove: 'terraced', onthemarket: 'terraced', zoopla: 'terraced' },
|
||||||
Flat: { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' },
|
'Flats/Maisonettes': { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' },
|
||||||
Maisonette: { rightmove: 'flat', onthemarket: 'flats', zoopla: 'flat' },
|
|
||||||
Bungalow: { rightmove: 'bungalow', onthemarket: 'bungalow', zoopla: 'bungalow' },
|
Bungalow: { rightmove: 'bungalow', onthemarket: 'bungalow', zoopla: 'bungalow' },
|
||||||
'Park home': { rightmove: 'park-home', onthemarket: 'property', zoopla: '' },
|
'Park home': { rightmove: 'park-home', onthemarket: 'property', zoopla: '' },
|
||||||
};
|
};
|
||||||
|
|
|
||||||
|
|
@ -265,6 +265,8 @@ def _build(
|
||||||
.when(has_epc)
|
.when(has_epc)
|
||||||
.then(pl.col("epc_property_type"))
|
.then(pl.col("epc_property_type"))
|
||||||
.otherwise(pl.col("pp_property_type"))
|
.otherwise(pl.col("pp_property_type"))
|
||||||
|
# Unify EPC's "Flat"/"Maisonette" with price-paid's "Flats/Maisonettes"
|
||||||
|
.replace({"Flat": "Flats/Maisonettes", "Maisonette": "Flats/Maisonettes"})
|
||||||
.alias("property_type")
|
.alias("property_type")
|
||||||
)
|
)
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -18,7 +18,7 @@ TERRACE_TYPES = [
|
||||||
"Enclosed End-Terrace",
|
"Enclosed End-Terrace",
|
||||||
"Terraced",
|
"Terraced",
|
||||||
]
|
]
|
||||||
FLAT_TYPES = ["Flats/Maisonettes", "Flat", "Maisonette"]
|
FLAT_TYPES = ["Flats/Maisonettes"]
|
||||||
TYPE_GROUPS = ["Detached", "Semi-Detached", "Terraced", "Flats", "Bungalow"]
|
TYPE_GROUPS = ["Detached", "Semi-Detached", "Terraced", "Flats", "Bungalow"]
|
||||||
SHRINKAGE_K = 50
|
SHRINKAGE_K = 50
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -944,9 +944,9 @@ pub static ENUM_FEATURE_GROUPS: &[EnumFeatureGroup] = &[
|
||||||
},
|
},
|
||||||
EnumFeatureConfig {
|
EnumFeatureConfig {
|
||||||
name: "Property type",
|
name: "Property type",
|
||||||
order: Some(&["Detached", "Semi-Detached", "Terraced", "Flat"]),
|
order: Some(&["Detached", "Semi-Detached", "Terraced", "Flats/Maisonettes"]),
|
||||||
description: "Type of property: detached, semi-detached, terraced, or flat",
|
description: "Type of property: detached, semi-detached, terraced, or flat/maisonette",
|
||||||
detail: "From HM Land Registry Price Paid data. The broad property type classification: Detached, Semi-Detached, Terraced, or Flat/Maisonette.",
|
detail: "From HM Land Registry Price Paid data. The broad property type classification: Detached, Semi-Detached, Terraced, or Flats/Maisonettes.",
|
||||||
source: "price-paid",
|
source: "price-paid",
|
||||||
},
|
},
|
||||||
EnumFeatureConfig {
|
EnumFeatureConfig {
|
||||||
|
|
|
||||||
|
|
@ -329,7 +329,6 @@ async fn main() -> anyhow::Result<()> {
|
||||||
&cli.google_oauth_client_secret,
|
&cli.google_oauth_client_secret,
|
||||||
)
|
)
|
||||||
.await?;
|
.await?;
|
||||||
|
|
||||||
info!(
|
info!(
|
||||||
"Ollama configured: {} (model: {})",
|
"Ollama configured: {} (model: {})",
|
||||||
cli.ollama_url, cli.ollama_model
|
cli.ollama_url, cli.ollama_model
|
||||||
|
|
|
||||||
|
|
@ -145,7 +145,7 @@ mod tests {
|
||||||
"Detached".into(),
|
"Detached".into(),
|
||||||
"Semi".into(),
|
"Semi".into(),
|
||||||
"Terraced".into(),
|
"Terraced".into(),
|
||||||
"Flat".into(),
|
"Flats/Maisonettes".into(),
|
||||||
],
|
],
|
||||||
);
|
);
|
||||||
map
|
map
|
||||||
|
|
@ -273,7 +273,7 @@ mod tests {
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_enum_with_unknown_value() {
|
fn parse_enum_with_unknown_value() {
|
||||||
let (_numeric, enums) = parse_filters(
|
let (_numeric, enums) = parse_filters(
|
||||||
Some("Type:Detached|Unknown|Flat"),
|
Some("Type:Detached|Unknown|Flats/Maisonettes"),
|
||||||
&extended_feature_map(),
|
&extended_feature_map(),
|
||||||
&extended_enum_values(),
|
&extended_enum_values(),
|
||||||
)
|
)
|
||||||
|
|
@ -281,14 +281,14 @@ mod tests {
|
||||||
|
|
||||||
assert_eq!(enums.len(), 1);
|
assert_eq!(enums.len(), 1);
|
||||||
assert!(enums[0].allowed.contains(&(0.0_f32).to_bits())); // Detached
|
assert!(enums[0].allowed.contains(&(0.0_f32).to_bits())); // Detached
|
||||||
assert!(enums[0].allowed.contains(&(3.0_f32).to_bits())); // Flat
|
assert!(enums[0].allowed.contains(&(3.0_f32).to_bits())); // Flats/Maisonettes
|
||||||
assert_eq!(enums[0].allowed.len(), 2);
|
assert_eq!(enums[0].allowed.len(), 2);
|
||||||
}
|
}
|
||||||
|
|
||||||
#[test]
|
#[test]
|
||||||
fn parse_filter_with_whitespace() {
|
fn parse_filter_with_whitespace() {
|
||||||
let (numeric, enums) = parse_filters(
|
let (numeric, enums) = parse_filters(
|
||||||
Some("Price : 100000 : 500000 , Type : Detached | Flat"),
|
Some("Price : 100000 : 500000 , Type : Detached | Flats/Maisonettes"),
|
||||||
&extended_feature_map(),
|
&extended_feature_map(),
|
||||||
&extended_enum_values(),
|
&extended_enum_values(),
|
||||||
)
|
)
|
||||||
|
|
|
||||||
|
|
@ -171,7 +171,7 @@ pub fn build_system_prompt(features: &FeaturesResponse) -> String {
|
||||||
{\"name\": \"Last known price\", \"bound\": \"max\", \"value\": 300000}, \
|
{\"name\": \"Last known price\", \"bound\": \"max\", \"value\": 300000}, \
|
||||||
{\"name\": \"Number of bedrooms & living rooms\", \"bound\": \"min\", \"value\": 4}, \
|
{\"name\": \"Number of bedrooms & living rooms\", \"bound\": \"min\", \"value\": 4}, \
|
||||||
{\"name\": \"Max available download speed (Mbps)\", \"bound\": \"min\", \"value\": 100}], \
|
{\"name\": \"Max available download speed (Mbps)\", \"bound\": \"min\", \"value\": 100}], \
|
||||||
\"enum_filters\": [{\"name\": \"Property type\", \"values\": [\"Flat\"]}], \
|
\"enum_filters\": [{\"name\": \"Property type\", \"values\": [\"Flats/Maisonettes\"]}], \
|
||||||
\"notes\": \"No filter for: beach proximity\"}"
|
\"notes\": \"No filter for: beach proximity\"}"
|
||||||
.to_string(),
|
.to_string(),
|
||||||
);
|
);
|
||||||
|
|
|
||||||
Loading…
Add table
Add a link
Reference in a new issue