From 5319a46ff79dc73f0bbfd84849d293ab48e86903 Mon Sep 17 00:00:00 2001 From: Andras Schmelczer Date: Sun, 26 Jun 2022 12:08:19 +0200 Subject: [PATCH] Improve get_sentences --- .vscode/settings.json | 2 ++ setup.cfg | 2 +- src/great_ai/utilities/__init__.py | 6 ++-- src/great_ai/utilities/get_sentences.py | 46 ++++++++++++++++--------- tests/utilities/test_get_sentences.py | 42 ++++++++++++++++++++++ 5 files changed, 77 insertions(+), 21 deletions(-) diff --git a/.vscode/settings.json b/.vscode/settings.json index 9780c7f..b9d0b65 100644 --- a/.vscode/settings.json +++ b/.vscode/settings.json @@ -4,6 +4,7 @@ "basereload", "boto", "botocore", + "Convolutional", "datatable", "displaylogo", "downsample", @@ -39,6 +40,7 @@ "sklearn", "starlette", "sublinear", + "syntok", "Tfidf", "threadsafe", "ticktext", diff --git a/setup.cfg b/setup.cfg index 265c1ad..7af40a1 100644 --- a/setup.cfg +++ b/setup.cfg @@ -26,8 +26,8 @@ install_requires = scikit-learn == 1.1.1 matplotlib >= 3.5.0 numpy >= 1.22.0 + syntok >= 1.4.0 langcodes[data] >= 3.3.0 - segtok >= 1.5.11 langdetect >= 1.0.9 tinydb >= 4.7.0 pandas >= 1.4.0 diff --git a/src/great_ai/utilities/__init__.py b/src/great_ai/utilities/__init__.py index 984d698..7d63c08 100644 --- a/src/great_ai/utilities/__init__.py +++ b/src/great_ai/utilities/__init__.py @@ -1,10 +1,10 @@ from .clean import clean -from .parallel_map import parallel_map -from .unique import unique from .config_file import ConfigFile, ParseError +from .evaluate_ranking import evaluate_ranking from .get_sentences import get_sentences from .language import english_name_of_language, is_english, predict_language from .lemmatize import lemmatize from .logger import get_logger -from .evaluate_ranking import evaluate_ranking from .match_names import match_names +from .parallel_map import parallel_map +from .unique import unique diff --git a/src/great_ai/utilities/get_sentences.py b/src/great_ai/utilities/get_sentences.py index de0ad25..1a507dc 100644 --- a/src/great_ai/utilities/get_sentences.py +++ b/src/great_ai/utilities/get_sentences.py @@ -1,30 +1,42 @@ -from typing import List, cast +import re +from string import punctuation +from typing import List -from segtok.segmenter import split_multi +from syntok.segmenter import segment +from syntok.tokenizer import Tokenizer from .data import sentence_ending_punctuations +punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*") + def get_sentences( - text: str, ignore_partial: bool = False, true_case: bool = False + text: str, + ignore_partial: bool = False, + true_case: bool = False, + remove_punctuation: bool = False, ) -> List[str]: - if text.strip() == "": - return [] + tokenizer = Tokenizer( + emit_hyphen_or_underscore_sep=True, replace_not_contraction=False + ) + token_stream = tokenizer.tokenize(text) - possible_sentences = [ - cast(str, s).strip() for s in split_multi(text) if cast(str, s).strip() + def process(sentence: str) -> str: + if true_case: + sentence = sentence[0].lower() + sentence[1:] # very crude method + if remove_punctuation: + sentence = re.sub(punctuations_pattern, " ", sentence) + return sentence.strip() + + sentences = [ + process(tokenizer.to_text(sentence)) for sentence in segment(token_stream) ] if ignore_partial: - possible_sentences = [ - s - for s in possible_sentences - if s[0].isupper() and s[-1] in sentence_ending_punctuations + sentences = [ + sentence + for sentence in sentences + if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations ] - if true_case: - possible_sentences = [ - s[0].lower() + s[1:] for s in possible_sentences # very crude method - ] - - return possible_sentences + return sentences diff --git a/tests/utilities/test_get_sentences.py b/tests/utilities/test_get_sentences.py index da2f772..7d864f3 100644 --- a/tests/utilities/test_get_sentences.py +++ b/tests/utilities/test_get_sentences.py @@ -11,6 +11,48 @@ class TestGetSentences(unittest.TestCase): assert get_sentences(text) == expected assert get_sentences(text, ignore_partial=True) == expected[0:2] + def test_complex(self) -> None: + text = """ + This is a complete sentence. So is this. + End of paragraph. + + + Negation contractions (like don't or ain't) are resolved. + + However this is not a sent + """ + + expected = [ + "This is a complete sentence.", + "So is this.", + "End of paragraph.", + "Negation contractions (like don't or ain't) are resolved.", + "However this is not a sent", + ] + + print(get_sentences(text, ignore_partial=True)) + + assert get_sentences(text) == expected + assert get_sentences(text, ignore_partial=True) == expected[:-1] + + def test_true_casing(self) -> None: + text = "This is also referred to as a Convolutional Neural Network (CNN)." + expected = ["this is also referred to as a Convolutional Neural Network (CNN)."] + + assert get_sentences(text, true_case=True) == expected + + def test_remove_punctuation(self) -> None: + text = "Also, we --- the authors --- have to find less intrusive, and higher potential procedures. " + expected = [ + "Also we the authors have to find less intrusive and higher potential procedures" + ] + + assert get_sentences(text, remove_punctuation=True) == expected + def test_empty(self) -> None: assert get_sentences("") == [] + assert get_sentences(" ") == [] + assert get_sentences(" \n ") == [] assert get_sentences("", ignore_partial=True) == [] + assert get_sentences("", true_case=True) == [] + assert get_sentences("", remove_punctuation=True) == []