42 lines
1.1 KiB
Python
42 lines
1.1 KiB
Python
import re
|
|
from string import punctuation
|
|
from typing import List
|
|
|
|
from syntok.segmenter import segment
|
|
from syntok.tokenizer import Tokenizer
|
|
|
|
from .data import sentence_ending_punctuations
|
|
|
|
punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*")
|
|
|
|
|
|
def get_sentences(
|
|
text: str,
|
|
ignore_partial: bool = False,
|
|
true_case: bool = False,
|
|
remove_punctuation: bool = False,
|
|
) -> List[str]:
|
|
tokenizer = Tokenizer(
|
|
emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
|
|
)
|
|
token_stream = tokenizer.tokenize(text)
|
|
|
|
def process(sentence: str) -> str:
|
|
if true_case:
|
|
sentence = sentence[0].lower() + sentence[1:] # very crude method
|
|
if remove_punctuation:
|
|
sentence = re.sub(punctuations_pattern, " ", sentence)
|
|
return sentence.strip()
|
|
|
|
sentences = [
|
|
process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
|
|
]
|
|
|
|
if ignore_partial:
|
|
sentences = [
|
|
sentence
|
|
for sentence in sentences
|
|
if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
|
|
]
|
|
|
|
return sentences
|