68 lines
2.1 KiB
Python
68 lines
2.1 KiB
Python
import re
|
|
from string import punctuation
|
|
from typing import List
|
|
|
|
from syntok.segmenter import segment
|
|
from syntok.tokenizer import Tokenizer
|
|
|
|
from .data import sentence_ending_punctuations
|
|
|
|
punctuations_pattern = re.compile(f"\\s*[{re.escape(punctuation)}]+\\s*")
|
|
|
|
|
|
def get_sentences(
|
|
text: str,
|
|
ignore_partial: bool = False,
|
|
true_case: bool = False,
|
|
remove_punctuation: bool = False,
|
|
) -> List[str]:
|
|
"""Return the list of sentences found in the input text.
|
|
|
|
Use [syntok](https://github.com/fnl/syntok) to segment the sentences. Further
|
|
processing can be enabled with optional arguments.
|
|
|
|
Examples:
|
|
>>> get_sentences('This is a sentence. This is a half')
|
|
['This is a sentence.', 'This is a half']
|
|
|
|
>>> get_sentences('This is a sentence. This is a half', ignore_partial=True)
|
|
['This is a sentence.']
|
|
|
|
>>> get_sentences('I like Apple.', true_case=True, remove_punctuation=True)
|
|
['i like Apple']
|
|
|
|
Args:
|
|
text: Text to be segmented into sentences.
|
|
ignore_partial: Filter out sentences that are not capitalised/don't end with a
|
|
punctuation.
|
|
true_case: Crude method: lowercase the first word of each sentence.
|
|
remove_punctuation: Remove all kinds of punctuation.
|
|
|
|
Returns:
|
|
The found sentences (with partial sentences optionally filtered out).
|
|
"""
|
|
|
|
tokenizer = Tokenizer(
|
|
emit_hyphen_or_underscore_sep=True, replace_not_contraction=False
|
|
)
|
|
token_stream = tokenizer.tokenize(text)
|
|
|
|
def process(sentence: str) -> str:
|
|
if true_case:
|
|
sentence = sentence[0].lower() + sentence[1:] # very crude method
|
|
if remove_punctuation:
|
|
sentence = re.sub(punctuations_pattern, " ", sentence)
|
|
return sentence.strip()
|
|
|
|
sentences = [
|
|
process(tokenizer.to_text(sentence)) for sentence in segment(token_stream)
|
|
]
|
|
|
|
if ignore_partial:
|
|
sentences = [
|
|
sentence
|
|
for sentence in sentences
|
|
if sentence[0].isupper() and sentence[-1] in sentence_ending_punctuations
|
|
]
|
|
|
|
return sentences
|