andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Add truecasing

Browse source
This commit is contained in:
Andras Schmelczer 2022-06-25 11:05:21 +02:00
parent 88f43fb60e
commit e96aa553fe
1 changed files with 12 additions and 3 deletions
Download patch file Download diff file Expand all files Collapse all files

15
great_ai/src/great_ai/utilities/get_sentences.py
View file

@ -1,15 +1,19 @@
from typing import List from typing import List, cast
from segtok.segmenter import split_multi from segtok.segmenter import split_multi
from .data import sentence_ending_punctuations from .data import sentence_ending_punctuations
def get_sentences(text: str, ignore_partial: bool = False) -> List[str]: def get_sentences(
text: str, ignore_partial: bool = False, true_case: bool = False
) -> List[str]:
if text.strip() == "": if text.strip() == "":
return [] return []
possible_sentences = [s for s in split_multi(text) if s] possible_sentences = [
cast(str, s).strip() for s in split_multi(text) if cast(str, s).strip()
]
if ignore_partial: if ignore_partial:
possible_sentences = [ possible_sentences = [
@ -18,4 +22,9 @@ def get_sentences(text: str, ignore_partial: bool = False) -> List[str]:
if s[0].isupper() and s[-1] in sentence_ending_punctuations if s[0].isupper() and s[-1] in sentence_ending_punctuations
] ]
if true_case:
possible_sentences = [
s[0].lower() + s[1:] for s in possible_sentences # very crude method
]
return possible_sentences return possible_sentences