Improve get_sentences

.vscode/settings.json

@@ -4,6 +4,7 @@
     "basereload",
     "boto",
     "botocore",
+    "Convolutional",
     "datatable",
     "displaylogo",
     "downsample",
@@ -39,6 +40,7 @@
     "sklearn",
     "starlette",
     "sublinear",
+    "syntok",
     "Tfidf",
     "threadsafe",
     "ticktext",

setup.cfg

@@ -26,8 +26,8 @@ install_requires =
     scikit-learn == 1.1.1
     matplotlib >= 3.5.0
     numpy >= 1.22.0
+    syntok >= 1.4.0
     langcodes[data] >= 3.3.0
-    segtok >= 1.5.11
     langdetect >= 1.0.9
     tinydb >= 4.7.0
     pandas >= 1.4.0

src/great_ai/utilities/__init__.py

@@ -1,10 +1,10 @@
 from .clean import clean
-from .parallel_map import parallel_map
-from .unique import unique
 from .config_file import ConfigFile, ParseError
+from .evaluate_ranking import evaluate_ranking
 from .get_sentences import get_sentences
 from .language import english_name_of_language, is_english, predict_language
 from .lemmatize import lemmatize
 from .logger import get_logger
-from .evaluate_ranking import evaluate_ranking
 from .match_names import match_names
+from .parallel_map import parallel_map
+from .unique import unique

src/great_ai/utilities/get_sentences.py

@@ -1,30 +1,42 @@
-from typing import List, cast
+import re
+from string import punctuation
+from typing import List
 
-from segtok.segmenter import split_multi
+from syntok.segmenter import segment
+from syntok.tokenizer import Tokenizer
 
 from .data import sentence_ending_punctuations
 
+punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*")
+
 
 def get_sentences(
-    text: str, ignore_partial: bool = False, true_case: bool = False
+    text: str,
+    ignore_partial: bool = False,
+    true_case: bool = False,
+    remove_punctuation: bool = False,
 ) -> List[str]:
-    if text.strip() == "":
+    tokenizer = Tokenizer(
-        return []
+        emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
+    )
+    token_stream = tokenizer.tokenize(text)
 
-    possible_sentences = [
+    def process(sentence: str) -> str:
-        cast(str, s).strip() for s in split_multi(text) if cast(str, s).strip()
+        if true_case:
+            sentence = sentence[0].lower() + sentence[1:]  # very crude method
+        if remove_punctuation:
+            sentence = re.sub(punctuations_pattern, " ", sentence)
+        return sentence.strip()
+
+    sentences = [
+        process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
     ]
 
     if ignore_partial:
-        possible_sentences = [
+        sentences = [
-            s
+            sentence
-            for s in possible_sentences
+            for sentence in sentences
-            if s[0].isupper() and s[-1] in sentence_ending_punctuations
+            if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
         ]
 
-    if true_case:
+    return sentences
-        possible_sentences = [
-            s[0].lower() + s[1:] for s in possible_sentences  # very crude method
-        ]
-
-    return possible_sentences

tests/utilities/test_get_sentences.py

@@ -11,6 +11,48 @@ class TestGetSentences(unittest.TestCase):
         assert get_sentences(text) == expected
         assert get_sentences(text, ignore_partial=True) == expected[0:2]
 
+    def test_complex(self) -> None:
+        text = """
+        This is a complete sentence. So is this.
+        End of paragraph.
+
+
+        Negation contractions (like don't or ain't) are resolved.
+
+        However this is not a sent
+        """
+
+        expected = [
+            "This is a complete sentence.",
+            "So is this.",
+            "End of paragraph.",
+            "Negation contractions (like don't or ain't) are resolved.",
+            "However this is not a sent",
+        ]
+
+        print(get_sentences(text, ignore_partial=True))
+
+        assert get_sentences(text) == expected
+        assert get_sentences(text, ignore_partial=True) == expected[:-1]
+
+    def test_true_casing(self) -> None:
+        text = "This is also referred to as a Convolutional Neural Network (CNN)."
+        expected = ["this is also referred to as a Convolutional Neural Network (CNN)."]
+
+        assert get_sentences(text, true_case=True) == expected
+
+    def test_remove_punctuation(self) -> None:
+        text = "Also, we --- the authors --- have to find less intrusive, and higher potential procedures. "
+        expected = [
+            "Also we the authors have to find less intrusive and higher potential procedures"
+        ]
+
+        assert get_sentences(text, remove_punctuation=True) == expected
+
     def test_empty(self) -> None:
         assert get_sentences("") == []
+        assert get_sentences(" ") == []
+        assert get_sentences("  \n ") == []
         assert get_sentences("", ignore_partial=True) == []
+        assert get_sentences("", true_case=True) == []
+        assert get_sentences("", remove_punctuation=True) == []