Move files

src/great_ai/utilities/get_sentences.py

@@ -1,42 +0,0 @@
-import re
-from string import punctuation
-from typing import List
-
-from syntok.segmenter import segment
-from syntok.tokenizer import Tokenizer
-
-from .data import sentence_ending_punctuations
-
-punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*")
-
-
-def get_sentences(
-    text: str,
-    ignore_partial: bool = False,
-    true_case: bool = False,
-    remove_punctuation: bool = False,
-) -> List[str]:
-    tokenizer = Tokenizer(
-        emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
-    )
-    token_stream = tokenizer.tokenize(text)
-
-    def process(sentence: str) -> str:
-        if true_case:
-            sentence = sentence[0].lower() + sentence[1:]  # very crude method
-        if remove_punctuation:
-            sentence = re.sub(punctuations_pattern, " ", sentence)
-        return sentence.strip()
-
-    sentences = [
-        process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
-    ]
-
-    if ignore_partial:
-        sentences = [
-            sentence
-            for sentence in sentences
-            if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
-        ]
-
-    return sentences