andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Update test style

Browse source
This commit is contained in:
Andras Schmelczer 2022-07-05 13:14:05 +02:00
parent f188724d4b
commit 2db2253578
18 changed files with 778 additions and 686 deletions
Download patch file Download diff file Expand all files Collapse all files

0
tests/__init__.py
View file

0
tests/great_ai/__init__.py
View file

62
tests/great_ai/test_async_starters.py Normal file
View file

@ -0,0 +1,62 @@
from asyncio import sleep
import pytest
from great_ai import (
ArgumentValidationError,
GreatAI,
WrongDecoratorOrderError,
parameter,
)
@pytest.mark.asyncio
async def test_create_trivial_cases() -> None:
@GreatAI.create
async def hello_world_1(name: str) -> str:
await sleep(0.5)
return f"Hello {name}!"
assert (await hello_world_1("andras").output) == "Hello andras!"
@GreatAI.create
async def hello_world_2(name: str) -> str:
await sleep(0.5)
return f"Hello {name}!"
assert (await hello_world_2("andras").output) == "Hello andras!"
@GreatAI.create()
async def hello_world_3(name: str) -> str:
await sleep(0.5)
return f"Hello {name}!"
assert (await hello_world_3("andras").output) == "Hello andras!"
@GreatAI.create()
async def hello_world_4(name):
await sleep(0.5)
return f"Hello {name}!"
assert (await hello_world_4("andras").output) == "Hello andras!"
@pytest.mark.asyncio
async def test_with_parameter() -> None:
@GreatAI.create
@parameter("name", validator=lambda v: len(v) > 5)
async def hello_world(name: str) -> str:
await sleep(0.5)
return f"Hello {name}!"
assert (await hello_world("andras").output) == "Hello andras!"
with pytest.raises(ArgumentValidationError):
await hello_world("short")
with pytest.raises(WrongDecoratorOrderError):
@parameter("name", validator=lambda v: len(v) > 5)
@GreatAI.create
def hello_world(name: str) -> str:
return f"Hello {name}!"

86
tests/great_ai/test_basic_starters.py
View file

@ -1,4 +1,3 @@
import unittest
from functools import lru_cache from functools import lru_cache
import pytest import pytest
@ -11,55 +10,62 @@ from great_ai import (
) )
class TestHumanReadableToByte(unittest.TestCase): def test_create_trivial_cases() -> None:
def test_create_simple_cases(self) -> None: @GreatAI.create
@GreatAI.create def hello_world_1(name: str) -> str:
def hello_world(name: str) -> str: return f"Hello {name}!"
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!" assert hello_world_1("andras").output == "Hello andras!"
@GreatAI.create @GreatAI.create
def hello_world(name): def hello_world_2(name):
return f"Hello {name}!" return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!" assert hello_world_2("andras").output == "Hello andras!"
@GreatAI.create() @GreatAI.create()
def hello_world(name: str) -> str: def hello_world_3(name: str) -> str:
return f"Hello {name}!" return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!" assert hello_world_3("andras").output == "Hello andras!"
def test_create_with_other_decorator(self) -> None: @GreatAI.create()
@GreatAI.create def hello_world_4(name):
@lru_cache return f"Hello {name}!"
def hello_world(name: str) -> str:
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!" assert hello_world_4("andras").output == "Hello andras!"
@lru_cache
@GreatAI.create()
def hello_world(name: str) -> str:
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!" def test_create_with_other_decorator() -> None:
@GreatAI.create
@lru_cache
def hello_world(name: str) -> str:
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!"
@lru_cache
@GreatAI.create()
def hello_world(name: str) -> str:
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!"
def test_with_parameter() -> None:
@GreatAI.create
@parameter("name", validator=lambda v: len(v) > 5)
def hello_world(name: str) -> str:
return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!"
with pytest.raises(ArgumentValidationError):
hello_world("short")
with pytest.raises(WrongDecoratorOrderError):
def test_with_parameter(self) -> None:
@GreatAI.create
@parameter("name", validator=lambda v: len(v) > 5) @parameter("name", validator=lambda v: len(v) > 5)
@GreatAI.create
def hello_world(name: str) -> str: def hello_world(name: str) -> str:
return f"Hello {name}!" return f"Hello {name}!"
assert hello_world("andras").output == "Hello andras!"
with pytest.raises(ArgumentValidationError):
hello_world("short")
with pytest.raises(WrongDecoratorOrderError):
@parameter("name", validator=lambda v: len(v) > 5)
@GreatAI.create
def hello_world(name: str) -> str:
return f"Hello {name}!"

9
tests/great_ai/test_great_ai.py Normal file
View file

@ -0,0 +1,9 @@
from great_ai import GreatAI
def test_process_batch() -> None:
@GreatAI.create(return_raw_result=True)
def f(x):
return x + 2
assert f.process_batch([3, 9, 34]) == [5, 11, 36]

0
tests/large_file/__init__.py
View file

40
tests/large_file/test_human_readable_to_byte.py
View file

@ -1,26 +1,26 @@
import unittest
from great_ai.large_file.helper import human_readable_to_byte from great_ai.large_file.helper import human_readable_to_byte
class TestHumanReadableToByte(unittest.TestCase): def test_simple_cases() -> None:
def test_simple_cases(self) -> None: assert human_readable_to_byte("1KB") == 1024
assert human_readable_to_byte("1KB") == 1024 assert human_readable_to_byte("2KB") == 2048
assert human_readable_to_byte("2KB") == 2048
def test_fractions(self) -> None:
assert human_readable_to_byte("0.5KB") == 512
assert human_readable_to_byte("20.5KB") == 1024 * 20 + 512
def test_formatting(self) -> None: def test_fractions() -> None:
assert human_readable_to_byte(" 1MB") == 1024 * 1024 assert human_readable_to_byte("0.5KB") == 512
assert human_readable_to_byte(" 2 MB") == 1024 * 1024 * 2 assert human_readable_to_byte("20.5KB") == 1024 * 20 + 512
assert human_readable_to_byte(" 4 MB ") == 1024 * 1024 * 4
assert human_readable_to_byte("8MB ") == 1024 * 1024 * 8
assert human_readable_to_byte(" 1.5 MB ") == 1024 * 1024 * 1.5
def test_casing(self) -> None:
assert human_readable_to_byte("0.5GB") == 0.5 * 1024 * 1024 * 1024 def test_formatting() -> None:
assert human_readable_to_byte("0.5gB") == 0.5 * 1024 * 1024 * 1024 assert human_readable_to_byte(" 1MB") == 1024 * 1024
assert human_readable_to_byte("0.5Gb") == 0.5 * 1024 * 1024 * 1024 assert human_readable_to_byte(" 2 MB") == 1024 * 1024 * 2
assert human_readable_to_byte("0.5gb") == 0.5 * 1024 * 1024 * 1024 assert human_readable_to_byte(" 4 MB ") == 1024 * 1024 * 4
assert human_readable_to_byte("8MB ") == 1024 * 1024 * 8
assert human_readable_to_byte(" 1.5 MB ") == 1024 * 1024 * 1.5
def test_casing() -> None:
assert human_readable_to_byte("0.5GB") == 0.5 * 1024 * 1024 * 1024
assert human_readable_to_byte("0.5gB") == 0.5 * 1024 * 1024 * 1024
assert human_readable_to_byte("0.5Gb") == 0.5 * 1024 * 1024 * 1024
assert human_readable_to_byte("0.5gb") == 0.5 * 1024 * 1024 * 1024

188
tests/large_file/test_large_file.py
View file

@ -1,9 +1,9 @@
import unittest
from pathlib import Path from pathlib import Path
from typing import Any from typing import Any
from unittest.mock import Mock, patch from unittest.mock import Mock, patch
import boto3 import boto3
import pytest
PATH = Path(__file__).parent.resolve() PATH = Path(__file__).parent.resolve()
@ -19,104 +19,112 @@ credentials = {
} }
class TestLargeFileS3(unittest.TestCase): def test_uninitialized() -> None:
def test_uninitialized(self) -> None: with pytest.raises(ValueError):
self.assertRaises(ValueError, LargeFileS3, "test-file") LargeFileS3("test-file")
def test_bad_file_modes(self) -> None:
self.assertRaises(ValueError, LargeFileS3, "test-file", "w", version=3)
self.assertRaises(ValueError, LargeFileS3, "test-file", "wb", version=3)
self.assertRaises(ValueError, LargeFileS3, "test-file", "w+r")
self.assertRaises(ValueError, LargeFileS3, "test-file", "test")
@patch.object(boto3, "client") def test_bad_file_modes() -> None:
def test_initialized_with_dict(self, client: Any) -> None: with pytest.raises(ValueError):
s3 = Mock() LargeFileS3("test-file", "w", version=3)
s3.list_objects_v2 = Mock(
return_value={
"Contents": [
{
"Key": "test-file/0",
"Size": 30,
},
{
"Key": "test-file/1",
"Size": 300,
},
{
"Key": "test-file/2",
"Size": 187,
},
]
}
)
boto3.client = Mock(return_value=s3)
LargeFileS3.configure_credentials( with pytest.raises(ValueError):
aws_region_name=credentials["aws_region_name"], LargeFileS3("test-file", "wb", version=3)
aws_access_key_id=credentials["aws_access_key_id"],
aws_secret_access_key=credentials["aws_secret_access_key"],
large_files_bucket_name=credentials["large_files_bucket_name"],
aws_endpoint_url=credentials["aws_endpoint_url"],
)
lf = LargeFileS3("test-file")
boto3.client.assert_called_once_with( with pytest.raises(ValueError):
"s3", LargeFileS3("test-file", "w+r")
aws_access_key_id=credentials["aws_access_key_id"],
aws_secret_access_key=credentials["aws_secret_access_key"],
region_name=credentials["aws_region_name"],
endpoint_url=credentials["aws_endpoint_url"],
)
s3.list_objects_v2.assert_called_once_with( with pytest.raises(ValueError):
Bucket=credentials["large_files_bucket_name"], Prefix="test-file" LargeFileS3("test-file", "test")
)
assert lf._version == 2
assert lf._local_name == "test-file-2"
@patch.object(boto3, "client") @patch.object(boto3, "client")
def test_initialized_with_file(self, client: Any) -> None: def test_initialized_with_dict(client: Any) -> None:
s3 = Mock() s3 = Mock()
s3.list_objects_v2 = Mock( s3.list_objects_v2 = Mock(
return_value={ return_value={
"Contents": [ "Contents": [
{ {
"Key": "test-file/0", "Key": "test-file/0",
"Size": 30, "Size": 30,
}, },
{ {
"Key": "test-file/1", "Key": "test-file/1",
"Size": 300, "Size": 300,
}, },
{ {
"Key": "test-file/2", "Key": "test-file/2",
"Size": 187, "Size": 187,
}, },
] ]
} }
) )
boto3.client = Mock(return_value=s3)
boto3.client = Mock(return_value=s3) LargeFileS3.configure_credentials(
aws_region_name=credentials["aws_region_name"],
aws_access_key_id=credentials["aws_access_key_id"],
aws_secret_access_key=credentials["aws_secret_access_key"],
large_files_bucket_name=credentials["large_files_bucket_name"],
aws_endpoint_url=credentials["aws_endpoint_url"],
)
lf = LargeFileS3("test-file")
LargeFileS3.configure_credentials_from_file( boto3.client.assert_called_once_with(
PATH / "../../docs/example_secrets.ini" "s3",
) aws_access_key_id=credentials["aws_access_key_id"],
lf = LargeFileS3("test-file") aws_secret_access_key=credentials["aws_secret_access_key"],
region_name=credentials["aws_region_name"],
endpoint_url=credentials["aws_endpoint_url"],
)
boto3.client.assert_called_once_with( s3.list_objects_v2.assert_called_once_with(
"s3", Bucket=credentials["large_files_bucket_name"], Prefix="test-file"
aws_access_key_id=credentials["aws_access_key_id"], )
aws_secret_access_key=credentials["aws_secret_access_key"],
region_name=credentials["aws_region_name"],
endpoint_url=credentials["aws_endpoint_url"],
)
assert s3.list_objects_v2.called assert lf._version == 2
s3.list_objects_v2.assert_called_once_with( assert lf._local_name == "test-file-2"
Bucket=credentials["large_files_bucket_name"], Prefix="test-file"
)
assert lf._version == 2
assert lf._local_name == "test-file-2" @patch.object(boto3, "client")
def test_initialized_with_file(client: Any) -> None:
s3 = Mock()
s3.list_objects_v2 = Mock(
return_value={
"Contents": [
{
"Key": "test-file/0",
"Size": 30,
},
{
"Key": "test-file/1",
"Size": 300,
},
{
"Key": "test-file/2",
"Size": 187,
},
]
}
)
boto3.client = Mock(return_value=s3)
LargeFileS3.configure_credentials_from_file(PATH / "../../docs/example_secrets.ini")
lf = LargeFileS3("test-file")
boto3.client.assert_called_once_with(
"s3",
aws_access_key_id=credentials["aws_access_key_id"],
aws_secret_access_key=credentials["aws_secret_access_key"],
region_name=credentials["aws_region_name"],
endpoint_url=credentials["aws_endpoint_url"],
)
assert s3.list_objects_v2.called
s3.list_objects_v2.assert_called_once_with(
Bucket=credentials["large_files_bucket_name"], Prefix="test-file"
)
assert lf._version == 2
assert lf._local_name == "test-file-2"

0
tests/utilities/__init__.py
View file

43
tests/utilities/test_chunk.py
View file

@ -1,32 +1,31 @@
import unittest
import pytest import pytest
from great_ai.utilities import chunk from great_ai.utilities import chunk
class TestChunk(unittest.TestCase): def test_simple() -> None:
def test_simple(self) -> None: i = [1, 2, 3, 4]
i = [1, 2, 3, 4]
assert list(chunk(i, 1)) == [[1], [2], [3], [4]] assert list(chunk(i, 1)) == [[1], [2], [3], [4]]
assert list(chunk(i, 2)) == [[1, 2], [3, 4]] assert list(chunk(i, 2)) == [[1, 2], [3, 4]]
assert list(chunk(i, 3)) == [[1, 2, 3], [4]] assert list(chunk(i, 3)) == [[1, 2, 3], [4]]
assert list(chunk(i, 4)) == [[1, 2, 3, 4]] assert list(chunk(i, 4)) == [[1, 2, 3, 4]]
assert list(chunk(i, 5)) == [[1, 2, 3, 4]] assert list(chunk(i, 5)) == [[1, 2, 3, 4]]
assert list(chunk(i, 125)) == [[1, 2, 3, 4]] assert list(chunk(i, 125)) == [[1, 2, 3, 4]]
def test_bad_argument(self) -> None:
with pytest.raises(AssertionError):
list(chunk([], -10))
with pytest.raises(AssertionError): def test_bad_argument() -> None:
list(chunk([], 0)) with pytest.raises(AssertionError):
list(chunk([], -10))
def test_generator(self) -> None: with pytest.raises(AssertionError):
def my_generator(): list(chunk([], 0))
for i in range(1, 11):
yield i
assert list(chunk(range(1, 11), 3)) == [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]
assert list(chunk(my_generator(), 3)) == [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]] def test_generator() -> None:
def my_generator():
for i in range(1, 11):
yield i
assert list(chunk(range(1, 11), 3)) == [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]
assert list(chunk(my_generator(), 3)) == [[1, 2, 3], [4, 5, 6], [7, 8, 9], [10]]

182
tests/utilities/test_clean.py
View file

@ -1,110 +1,112 @@
import unittest
from great_ai.utilities import clean from great_ai.utilities import clean
class TestClean(unittest.TestCase): def test_xml_handling() -> None:
def test_xml_handling(self) -> None: xml = '<strong>Hi, </strong> my name<br/>is <span style="color: hotpink;"> András</span>! &lt;&#51; <> < ></><> &lt;&gt; <|'
xml = '<strong>Hi, </strong> my name<br/>is <span style="color: hotpink;"> András</span>! &lt;&#51; <> < ></><> &lt;&gt; <|' clean_xml = "Hi, my name is András! <3 <> <|"
clean_xml = "Hi, my name is András! <3 <> <|" clean_xml_ascii = "Hi, my name is Andras! <3 <> <|"
clean_xml_ascii = "Hi, my name is Andras! <3 <> <|"
assert clean(xml) == clean_xml assert clean(xml) == clean_xml
assert clean(xml, ignore_xml=True, ignore_latex=True) == xml assert clean(xml, ignore_xml=True, ignore_latex=True) == xml
assert clean(xml, convert_to_ascii=True) == clean_xml_ascii assert clean(xml, convert_to_ascii=True) == clean_xml_ascii
def test_simple_latex_handling(self) -> None:
latex = 'Bj\\"{o}rn is \\textit{happy} 🙂'
clean_latex = "Björn is happy 🙂"
clean_latex_ascii = "Bjorn is happy"
assert clean(latex) == clean_latex
assert clean(latex, ignore_latex=True) == latex
assert clean(latex, convert_to_ascii=True) == clean_latex_ascii
def test_cursed(self) -> None:
cursed = """
t̴̨̢̝̻͚͓͉̀̄̃́͜o̸͉̰̼͖͖̅̀̓̿̇̚͝ ̶̢͕̄͊̾͑̂̀̉͑ṱ̸̨̛͈̣̠̤͍̰̂̅͋̀́h̸̹̐͗̅̉͐͂͝e̶̜̳̞̍͘ ̴̟̗̻̤̤̮̒̂̋̓́͐͘͜m̵̺̫̥̙̣̥͐̌͜o̴̖͙̙̎̒͂o̷̬̤͑̆̂̉̊̂n̸̥͒̊
"""
cleaned = "to the moon"
assert clean(cursed) == cursed.strip()
assert clean(cursed, convert_to_ascii=True) == cleaned
def test_whitespace(self) -> None:
text = """
word1
word2 \n\n\n\t
wo\t\f rd3
""" # noqa: W293 def test_simple_latex_handling() -> None:
cleaned = "word1 word2 wo rd3" latex = 'Bj\\"{o}rn is \\textit{happy} 🙂'
clean_latex = "Björn is happy 🙂"
clean_latex_ascii = "Bjorn is happy"
assert clean(text, convert_to_ascii=True) == cleaned assert clean(latex) == clean_latex
assert clean(text, ignore_xml=True, ignore_latex=True) == cleaned assert clean(latex, ignore_latex=True) == latex
assert clean(text) == cleaned assert clean(latex, convert_to_ascii=True) == clean_latex_ascii
def test_hyphens(self) -> None:
text = """
break - word
break- word
break -word
break -word
break-\tword
"""
cleaned = "break - word break-word break-word break-word break-word"
assert clean(text) == cleaned def test_cursed() -> None:
cursed = """
t̴̨̢̝̻͚͓͉̀̄̃́͜o̸͉̰̼͖͖̅̀̓̿̇̚͝ ̶̢͕̄͊̾͑̂̀̉͑ṱ̸̨̛͈̣̠̤͍̰̂̅͋̀́h̸̹̐͗̅̉͐͂͝e̶̜̳̞̍͘ ̴̟̗̻̤̤̮̒̂̋̓́͐͘͜m̵̺̫̥̙̣̥͐̌͜o̴̖͙̙̎̒͂o̷̬̤͑̆̂̉̊̂n̸̥͒̊
"""
cleaned = "to the moon"
def test_T_I_T_L_E_case(self) -> None: assert clean(cursed) == cursed.strip()
text = "an a r t i c l e is to F I N D the purpose of a tree" assert clean(cursed, convert_to_ascii=True) == cleaned
cleaned = "an article is to FIND the purpose of a tree"
assert clean(text) == cleaned def test_whitespace() -> None:
text = """
def test_bracket_removal(self) -> None: word1
text = "something [0], and [frefe, ferf]"
cleaned = "something, and"
assert clean(text, remove_brackets=True) == cleaned word2 \n\n\n\t
self.assertEqual( wo\t\f rd3
clean(text, ignore_xml=True, ignore_latex=True, remove_brackets=True),
cleaned,
)
self.assertEqual(
clean(text, convert_to_ascii=True, remove_brackets=True), cleaned
)
assert clean(text) == text
def test_latex_math_handling(self) -> None:
latex = "An increase of 3% was achieved with $q_1 = \\frac{3}{5}$."
bad_latex = "An increase of 3% was achieved with $q_1 = \\frac{3}/5$."
clean_latex = "An increase of 3% was achieved with q_1 = 3/5."
clean_bad_latex = "An increase of 3% was achieved with q_1 = 3//5."
assert clean(latex) == clean_latex """ # noqa: W293
assert clean(bad_latex) == clean_bad_latex cleaned = "word1 word2 wo rd3"
assert clean(latex, ignore_latex=True) == latex
assert clean(bad_latex, ignore_latex=True) == bad_latex
def test_everything(self) -> None: assert clean(text, convert_to_ascii=True) == cleaned
text = """ assert clean(text, ignore_xml=True, ignore_latex=True) == cleaned
Hi % 3 &gt;2 <h2 color="red">my paper</h2> there! cost- effective cost - effective cost -effective \\frac{1/2} hi \\frac{1}{2} \\textit{italic} \\`acc\\^ented text 北亰 fi æ IJ ij ff András öô 2132 rgrv \n\\ &nbsp; fd [32] [Bei et al., 2003]\n 🤡 🙄 😶‍🌫️ 🇲🇲 assert clean(text) == cleaned
"""
cleaned = "Hi% 3 >2 my paper there! cost-effective cost - effective cost-effective 1/2/hi 1/2 italic accented text Bei Jing fi ae IJ ij ff Andras oo 2132 rgrv fd"
self.assertEqual(
clean(text, convert_to_ascii=True, remove_brackets=True), cleaned
)
def test_empty(self) -> None: def test_hyphens() -> None:
assert clean("", convert_to_ascii=True) == "" text = """
break - word
break- word
break -word
break -word
break-\tword
"""
cleaned = "break - word break-word break-word break-word break-word"
def test_punctuation_fixing(self) -> None: assert clean(text) == cleaned
text = " Dear reader ( or listener ) , welcome ! "
fixed = "Dear reader (or listener), welcome!"
assert clean(text) == fixed def test_T_I_T_L_E_case() -> None:
text = "an a r t i c l e is to F I N D the purpose of a tree"
cleaned = "an article is to FIND the purpose of a tree"
assert clean(text) == cleaned
def test_bracket_removal() -> None:
text = "something [0], and [frefe, ferf]"
cleaned = "something, and"
assert clean(text, remove_brackets=True) == cleaned
assert (
clean(text, ignore_xml=True, ignore_latex=True, remove_brackets=True) == cleaned
)
assert clean(text, convert_to_ascii=True, remove_brackets=True) == cleaned
assert clean(text) == text
def test_latex_math_handling() -> None:
latex = "An increase of 3% was achieved with $q_1 = \\frac{3}{5}$."
bad_latex = "An increase of 3% was achieved with $q_1 = \\frac{3}/5$."
clean_latex = "An increase of 3% was achieved with q_1 = 3/5."
clean_bad_latex = "An increase of 3% was achieved with q_1 = 3//5."
assert clean(latex) == clean_latex
assert clean(bad_latex) == clean_bad_latex
assert clean(latex, ignore_latex=True) == latex
assert clean(bad_latex, ignore_latex=True) == bad_latex
def test_everything() -> None:
text = """
Hi % 3 &gt;2 <h2 color="red">my paper</h2> there! cost- effective cost - effective cost -effective \\frac{1/2} hi \\frac{1}{2} \\textit{italic} \\`acc\\^ented text 北亰 fi æ IJ ij ff András öô 2132 rgrv \n\\ &nbsp; fd [32] [Bei et al., 2003]\n 🤡 🙄 😶‍🌫️ 🇲🇲
"""
cleaned = "Hi% 3 >2 my paper there! cost-effective cost - effective cost-effective 1/2/hi 1/2 italic accented text Bei Jing fi ae IJ ij ff Andras oo 2132 rgrv fd"
clean(text, convert_to_ascii=True, remove_brackets=True) == cleaned
def test_empty() -> None:
assert clean("", convert_to_ascii=True) == ""
def test_punctuation_fixing() -> None:
text = " Dear reader ( or listener ) , welcome ! "
fixed = "Dear reader (or listener), welcome!"
assert clean(text) == fixed

127
tests/utilities/test_config_file.py
View file

@ -1,5 +1,4 @@
import os import os
import unittest
from pathlib import Path from pathlib import Path
import pytest import pytest
@ -9,83 +8,89 @@ from great_ai.utilities import ConfigFile
DATA_PATH = Path(__file__).parent.resolve() / "data" DATA_PATH = Path(__file__).parent.resolve() / "data"
class TestConfigFile(unittest.TestCase): def test_simple() -> None:
def test_simple(self) -> None: c = ConfigFile(DATA_PATH / "good.conf")
c = ConfigFile(DATA_PATH / "good.conf") assert c.zeroth_key == "test"
assert c.zeroth_key == "test" assert c.first_key == "András"
assert c.first_key == "András" assert c.second_key == "test 2"
assert c.second_key == "test 2" assert c.third_key == "test= 2=="
assert c.third_key == "test= 2==" assert (
assert ( c.fourth_key
c.fourth_key == """
== """
this# this#
is is
multiline multiline
""" """
) )
assert c.whitespace == "hardly matters" assert c.whitespace == "hardly matters"
with pytest.raises(KeyError): with pytest.raises(KeyError):
c.this c.this
def test_simple_dict(self) -> None:
c = ConfigFile(DATA_PATH / "good.conf") def test_simple_dict() -> None:
assert c["zeroth_key"] == "test" c = ConfigFile(DATA_PATH / "good.conf")
assert c["first_key"] == "András" assert c["zeroth_key"] == "test"
assert c["second_key"] == "test 2" assert c["first_key"] == "András"
assert c["third_key"] == "test= 2==" assert c["second_key"] == "test 2"
assert ( assert c["third_key"] == "test= 2=="
c["fourth_key"] assert (
== """ c["fourth_key"]
== """
this# this#
is is
multiline multiline
""" """
) )
assert c["whitespace"] == "hardly matters" assert c["whitespace"] == "hardly matters"
with pytest.raises(KeyError): with pytest.raises(KeyError):
c["#this"] c["#this"]
def test_string_path(self) -> None:
c = ConfigFile(str(DATA_PATH / "good.conf")) def test_string_path() -> None:
assert c.zeroth_key == "test" c = ConfigFile(str(DATA_PATH / "good.conf"))
assert c.first_key == "András" assert c.zeroth_key == "test"
assert c.second_key == "test 2" assert c.first_key == "András"
assert c.third_key == "test= 2==" assert c.second_key == "test 2"
assert ( assert c.third_key == "test= 2=="
c.fourth_key assert (
== """ c.fourth_key
== """
this# this#
is is
multiline multiline
""" """
) )
assert c.whitespace == "hardly matters" assert c.whitespace == "hardly matters"
def test_env(self) -> None:
os.environ["alma"] = "12"
c = ConfigFile(DATA_PATH / "env.conf")
del os.environ["alma"]
assert c.first_key == "test"
assert c.second_key == "12"
def test_env_not_exists(self) -> None: def test_env() -> None:
with pytest.raises(KeyError): os.environ["alma"] = "12"
ConfigFile(DATA_PATH / "env-bad.conf") c = ConfigFile(DATA_PATH / "env.conf")
del os.environ["alma"]
assert c.first_key == "test"
assert c.second_key == "12"
def test_env_not_exists_fallback(self) -> None:
os.environ["alma"] = "12"
c = ConfigFile(DATA_PATH / "env.conf")
assert c.fourth_key == "this is a default value"
def test_env_exists_ignore_fallback(self) -> None: def test_env_not_exists() -> None:
os.environ["alma"] = "12" with pytest.raises(KeyError):
os.environ["SOMETHING"] = "hi" ConfigFile(DATA_PATH / "env-bad.conf")
c = ConfigFile(DATA_PATH / "env.conf")
del os.environ["SOMETHING"]
assert c.fourth_key == "hi"
def test_duplicate_key(self) -> None: def test_env_not_exists_fallback() -> None:
with pytest.raises(KeyError): os.environ["alma"] = "12"
ConfigFile(DATA_PATH / "bad.conf") c = ConfigFile(DATA_PATH / "env.conf")
assert c.fourth_key == "this is a default value"
def test_env_exists_ignore_fallback() -> None:
os.environ["alma"] = "12"
os.environ["SOMETHING"] = "hi"
c = ConfigFile(DATA_PATH / "env.conf")
del os.environ["SOMETHING"]
assert c.fourth_key == "hi"
def test_duplicate_key() -> None:
with pytest.raises(KeyError):
ConfigFile(DATA_PATH / "bad.conf")

117
tests/utilities/test_evaluate_ranking.py
View file

@ -1,97 +1,94 @@
import unittest
from pathlib import Path from pathlib import Path
import matplotlib import matplotlib
import pytest
matplotlib.use("Agg") # don't show a window for each test matplotlib.use("Agg") # don't show a window for each test
from great_ai.utilities import evaluate_ranking from great_ai.utilities import evaluate_ranking
class TestEvaluateRanking(unittest.TestCase): def test_default() -> None:
def test_default(self) -> None: results = evaluate_ranking(
results = evaluate_ranking( ["a", "a", "b", "b", "c", "d", "d", "d"],
["a", "a", "b", "b", "c", "d", "d", "d"], [10, 7, 11, 6, 8, 2, 7, 1],
[10, 7, 11, 6, 8, 2, 7, 1], target_recall=0.6,
target_recall=0.6, reverse_order=True,
reverse_order=True, )
)
self.assertEqual( assert results == {"d": 1.0, "c": 0.6666666666666666, "b": 0.4}
results,
{"d": 1.0, "c": 0.6666666666666666, "b": 0.4},
)
results = evaluate_ranking( results = evaluate_ranking(
["a", "a", "b", "b", "c", "d", "d", "d"], ["a", "a", "b", "b", "c", "d", "d", "d"],
[10, 7, 11, 6, 8, 2, 7, 1], [10, 7, 11, 6, 8, 2, 7, 1],
target_recall=0.6, target_recall=0.6,
reverse_order=False, reverse_order=False,
disable_interpolation=True, disable_interpolation=True,
) )
self.assertEqual( assert results == {
results, "a": 0.6666666666666666,
{"a": 0.6666666666666666, "b": 0.3333333333333333, "c": 0.2857142857142857}, "b": 0.3333333333333333,
) "c": 0.2857142857142857,
}
def test_mismatching_lengths(self) -> None:
self.assertRaises( def test_mismatching_lengths() -> None:
ValueError, with pytest.raises(ValueError):
evaluate_ranking, evaluate_ranking(
["a", "a", "b", "b", "c"], ["a", "a", "b", "b", "c"],
[10, 7, 11, 6, 8, 2, 7, 1], [10, 7, 11, 6, 8, 2, 7, 1],
target_recall=0.6, target_recall=0.6,
) )
def test_invalid_recalls(self) -> None:
self.assertRaises( def test_invalid_recalls() -> None:
AssertionError, with pytest.raises(AssertionError):
evaluate_ranking, evaluate_ranking(
["a", "a", "b", "b"], ["a", "a", "b", "b"],
[10, 7, 11, 6], [10, 7, 11, 6],
target_recall=10.6, target_recall=10.6,
) )
self.assertRaises( with pytest.raises(AssertionError):
AssertionError, evaluate_ranking(
evaluate_ranking,
["a", "a", "b", "b"], ["a", "a", "b", "b"],
[10, 7, 11, 6], [10, 7, 11, 6],
target_recall=-0.0001, target_recall=-0.0001,
) )
self.assertRaises( with pytest.raises(AssertionError):
AssertionError, evaluate_ranking(
evaluate_ranking,
["a", "a", "b", "b"], ["a", "a", "b", "b"],
[10, 7, 11, 6], [10, 7, 11, 6],
target_recall=1.00001, target_recall=1.00001,
) )
def test_empty(self) -> None:
evaluate_ranking(
[],
[],
target_recall=0.6,
)
evaluate_ranking( def test_empty() -> None:
["a"], evaluate_ranking(
[1], [],
target_recall=0.6, [],
) target_recall=0.6,
)
def test_save(self) -> None: evaluate_ranking(
path = Path("test.svg") ["a"],
path.unlink(missing_ok=True) [1],
target_recall=0.6,
)
evaluate_ranking(
["a", "a", "b", "b", "c", "d", "d", "d"],
[10, 7, 11, 6, 8, 2, 7, 1],
target_recall=0.1,
output_svg=path,
)
self.assertTrue(path.exists()) def test_save() -> None:
path.unlink() path = Path("test.svg")
path.unlink(missing_ok=True)
evaluate_ranking(
["a", "a", "b", "b", "c", "d", "d", "d"],
[10, 7, 11, 6, 8, 2, 7, 1],
target_recall=0.1,
output_svg=path,
)
assert path.exists()
path.unlink()

87
tests/utilities/test_get_sentences.py
View file

@ -1,58 +1,59 @@
import unittest
from great_ai.utilities import get_sentences from great_ai.utilities import get_sentences
class TestGetSentences(unittest.TestCase): def test_default() -> None:
def test_default(self) -> None: text = "This is a complete sentence. So is this. However this is n" # ot.
text = "This is a complete sentence. So is this. However this is n" # ot. expected = ["This is a complete sentence.", "So is this.", "However this is n"]
expected = ["This is a complete sentence.", "So is this.", "However this is n"]
assert get_sentences(text) == expected assert get_sentences(text) == expected
assert get_sentences(text, ignore_partial=True) == expected[0:2] assert get_sentences(text, ignore_partial=True) == expected[0:2]
def test_complex(self) -> None:
text = """
This is a complete sentence. So is this.
End of paragraph.
Negation contractions (like don't or ain't) are resolved. def test_complex() -> None:
text = """
This is a complete sentence. So is this.
End of paragraph.
However this is not a sent
"""
expected = [ Negation contractions (like don't or ain't) are resolved.
"This is a complete sentence.",
"So is this.",
"End of paragraph.",
"Negation contractions (like don't or ain't) are resolved.",
"However this is not a sent",
]
print(get_sentences(text, ignore_partial=True)) However this is not a sent
"""
assert get_sentences(text) == expected expected = [
assert get_sentences(text, ignore_partial=True) == expected[:-1] "This is a complete sentence.",
"So is this.",
"End of paragraph.",
"Negation contractions (like don't or ain't) are resolved.",
"However this is not a sent",
]
def test_true_casing(self) -> None: print(get_sentences(text, ignore_partial=True))
text = "This is also referred to as a Convolutional Neural Network (CNN)."
expected = ["this is also referred to as a Convolutional Neural Network (CNN)."]
assert get_sentences(text, true_case=True) == expected assert get_sentences(text) == expected
assert get_sentences(text, ignore_partial=True) == expected[:-1]
def test_remove_punctuation(self) -> None:
text = "Also, we --- the authors --- have to find less intrusive, and higher potential procedures. "
expected = [
"Also we the authors have to find less intrusive and higher potential procedures"
]
assert get_sentences(text, remove_punctuation=True) == expected def test_true_casing() -> None:
text = "This is also referred to as a Convolutional Neural Network (CNN)."
expected = ["this is also referred to as a Convolutional Neural Network (CNN)."]
def test_empty(self) -> None: assert get_sentences(text, true_case=True) == expected
assert get_sentences("") == []
assert get_sentences(" ") == []
assert get_sentences(" \n ") == [] def test_remove_punctuation() -> None:
assert get_sentences("", ignore_partial=True) == [] text = "Also, we --- the authors --- have to find less intrusive, and higher potential procedures. "
assert get_sentences("", true_case=True) == [] expected = [
assert get_sentences("", remove_punctuation=True) == [] "Also we the authors have to find less intrusive and higher potential procedures"
]
assert get_sentences(text, remove_punctuation=True) == expected
def test_empty() -> None:
assert get_sentences("") == []
assert get_sentences(" ") == []
assert get_sentences(" \n ") == []
assert get_sentences("", ignore_partial=True) == []
assert get_sentences("", true_case=True) == []
assert get_sentences("", remove_punctuation=True) == []

51
tests/utilities/test_language.py
View file

@ -1,31 +1,30 @@
import unittest
from great_ai.utilities import english_name_of_language, is_english, predict_language from great_ai.utilities import english_name_of_language, is_english, predict_language
class TestLanguage(unittest.TestCase): def test_predict_language() -> None:
def test_predict_language(self) -> None: assert predict_language("This is an English text.") == "en"
assert predict_language("This is an English text.") == "en" assert predict_language("Ez egy magyar szöveg.") == "hu"
assert predict_language("Ez egy magyar szöveg.") == "hu" assert predict_language("32") == "und"
assert predict_language("32") == "und" assert predict_language("") == "und"
assert predict_language("") == "und"
def test_is_english(self) -> None:
self.assertTrue(is_english("en"))
self.assertTrue(is_english("en-US"))
self.assertFalse(is_english("hu"))
self.assertFalse(is_english("de"))
self.assertFalse(is_english("zh"))
self.assertFalse(is_english("zh-TW"))
self.assertFalse(is_english("und"))
self.assertFalse(is_english(""))
self.assertFalse(is_english(None))
def english_name_of_language(self) -> None: def test_is_english() -> None:
assert english_name_of_language("en") == "English" assert is_english("en")
assert english_name_of_language("hu") == "Hungarian" assert is_english("en-US")
assert english_name_of_language("zh") == "Chinese" assert not is_english("hu")
assert english_name_of_language("zh-TW") == "Chinese" assert not is_english("de")
assert english_name_of_language("und") == "Unknown language" assert not is_english("zh")
assert english_name_of_language("") == "Unknown language" assert not is_english("zh-TW")
assert english_name_of_language(None) == "Unknown language" assert not is_english("und")
assert not is_english("")
assert not is_english(None)
def english_name_of_language() -> None:
assert english_name_of_language("en") == "English"
assert english_name_of_language("hu") == "Hungarian"
assert english_name_of_language("zh") == "Chinese"
assert english_name_of_language("zh-TW") == "Chinese"
assert english_name_of_language("und") == "Unknown language"
assert english_name_of_language("") == "Unknown language"
assert english_name_of_language(None) == "Unknown language"

202
tests/utilities/test_parallel_map.py
View file

@ -1,5 +1,3 @@
import unittest
import pytest import pytest
from great_ai.utilities import WorkerException, parallel_map from great_ai.utilities import WorkerException, parallel_map
@ -7,110 +5,112 @@ from great_ai.utilities import WorkerException, parallel_map
COUNT = int(1e5) + 3 COUNT = int(1e5) + 3
class TestParallelMap(unittest.TestCase): def test_simple_case_with_progress_bar() -> None:
def test_simple_case_with_progress_bar(self) -> None: assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=4)) == [
assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=4)) == [ v**2 for v in range(COUNT)
v**2 for v in range(COUNT) ]
]
def test_with_iterable(self) -> None:
from time import sleep
def my_generator(): def test_with_iterable() -> None:
for i in range(10): from time import sleep
yield i
sleep(0.1)
expected = [v**3 for v in range(10)] def my_generator():
for i in range(10):
yield i
sleep(0.1)
assert ( expected = [v**3 for v in range(10)]
list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1))
== expected assert (
list(parallel_map(lambda x: x**3, my_generator(), chunk_size=1)) == expected
)
def test_simple_case_without_progress_bar() -> None:
assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=2)) == [
v**2 for v in range(COUNT)
]
def test_simple_case_invalid_values() -> None:
with pytest.raises(AssertionError):
list(parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
with pytest.raises(AssertionError):
list(parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
def test_this_process_exception() -> None:
def my_generator():
yield 1
yield 2
yield 3
assert False
with pytest.raises(AssertionError):
list(
parallel_map(lambda v: v**2, my_generator(), concurrency=2, chunk_size=2)
) )
def test_simple_case_without_progress_bar(self) -> None: with pytest.raises(AssertionError):
assert list(parallel_map(lambda v: v**2, range(COUNT), concurrency=2)) == [ list(
v**2 for v in range(COUNT) parallel_map(lambda v: v**2, my_generator(), concurrency=1, chunk_size=2)
]
def test_simple_case_invalid_values(self) -> None:
with pytest.raises(AssertionError):
list(parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
with pytest.raises(AssertionError):
list(parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
def test_this_process_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
assert False
with pytest.raises(AssertionError):
list(
parallel_map(
lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
)
)
with pytest.raises(AssertionError):
list(
parallel_map(
lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
)
)
def test_ignore_this_process_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
yield 1 / 0
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4]
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4, 9]
def test_worker_process_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
with pytest.raises(WorkerException):
list(parallel_map(oh_no, range(COUNT), concurrency=2))
with pytest.raises(WorkerException):
list(parallel_map(oh_no, range(COUNT), concurrency=1))
def test_ignore_worker_process_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
assert (
list(parallel_map(oh_no, range(3), concurrency=2, ignore_exceptions=True))
== [None] * 3
)
assert (
list(parallel_map(oh_no, range(3), concurrency=1, ignore_exceptions=True))
== [None] * 3
) )
def test_no_op(self) -> None:
assert list(parallel_map(lambda v: v**2, [])) == [] def test_ignore_this_process_exception() -> None:
assert list(parallel_map(lambda v: v**2, [], chunk_size=100)) == [] def my_generator():
assert list(parallel_map(lambda v: v**2, [], concurrency=100)) == [] yield 1
yield 2
yield 3
yield 1 / 0
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4]
assert list(
parallel_map(
lambda v: v**2,
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4, 9]
def test_worker_process_exception() -> None:
def oh_no(_):
raise ValueError("hi")
with pytest.raises(WorkerException):
list(parallel_map(oh_no, range(COUNT), concurrency=2))
with pytest.raises(WorkerException):
list(parallel_map(oh_no, range(COUNT), concurrency=1))
def test_ignore_worker_process_exception() -> None:
def oh_no(_):
raise ValueError("hi")
assert (
list(parallel_map(oh_no, range(3), concurrency=2, ignore_exceptions=True))
== [None] * 3
)
assert (
list(parallel_map(oh_no, range(3), concurrency=1, ignore_exceptions=True))
== [None] * 3
)
def test_no_op() -> None:
assert list(parallel_map(lambda v: v**2, [])) == []
assert list(parallel_map(lambda v: v**2, [], chunk_size=100)) == []
assert list(parallel_map(lambda v: v**2, [], concurrency=100)) == []

203
tests/utilities/test_threaded_parallel_map.py
View file

@ -1,5 +1,3 @@
import unittest
import pytest import pytest
from great_ai.utilities import WorkerException, threaded_parallel_map from great_ai.utilities import WorkerException, threaded_parallel_map
@ -7,118 +5,125 @@ from great_ai.utilities import WorkerException, threaded_parallel_map
COUNT = int(1e5) + 3 COUNT = int(1e5) + 3
class TestParallelMap(unittest.TestCase): def test_simple_case_with_progress_bar() -> None:
def test_simple_case_with_progress_bar(self) -> None: assert list(
assert list( threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4)
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=4) ) == [v**2 for v in range(COUNT)]
) == [v**2 for v in range(COUNT)]
def test_with_iterable(self) -> None:
from time import sleep
def my_generator(): def test_with_iterable() -> None:
for i in range(10): from time import sleep
yield i
sleep(0.1)
expected = [v**3 for v in range(10)] def my_generator():
for i in range(10):
yield i
sleep(0.1)
assert ( expected = [v**3 for v in range(10)]
list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1))
== expected
)
def test_simple_case_without_progress_bar(self) -> None: assert (
assert list( list(threaded_parallel_map(lambda x: x**3, my_generator(), chunk_size=1))
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2) == expected
) == [v**2 for v in range(COUNT)] )
def test_simple_case_invalid_values(self) -> None:
with pytest.raises(AssertionError):
list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
with pytest.raises(AssertionError): def test_simple_case_without_progress_bar() -> None:
list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0)) assert list(
threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=2)
) == [v**2 for v in range(COUNT)]
def test_this_worker_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
assert False
with pytest.raises(AssertionError): def test_simple_case_invalid_values() -> None:
list( with pytest.raises(AssertionError):
threaded_parallel_map( list(threaded_parallel_map(lambda v: v**2, range(COUNT), concurrency=0))
lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
)
)
with pytest.raises(AssertionError): with pytest.raises(AssertionError):
list( list(threaded_parallel_map(lambda v: v**2, range(COUNT), chunk_size=0))
threaded_parallel_map(
lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
)
)
def test_ignore_this_worker_exception(self) -> None:
def my_generator():
yield 1
yield 2
yield 3
yield 1 / 0
assert list( def test_this_worker_exception() -> None:
def my_generator():
yield 1
yield 2
yield 3
assert False
with pytest.raises(AssertionError):
list(
threaded_parallel_map( threaded_parallel_map(
lambda v: v**2, lambda v: v**2, my_generator(), concurrency=2, chunk_size=2
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
) )
) == [1, 4] )
assert list(
with pytest.raises(AssertionError):
list(
threaded_parallel_map( threaded_parallel_map(
lambda v: v**2, lambda v: v**2, my_generator(), concurrency=1, chunk_size=2
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
) )
) == [1, 4, 9]
def test_worker_worker_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
with pytest.raises(WorkerException):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2))
with pytest.raises(WorkerException):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1))
def test_ignore_worker_worker_exception(self) -> None:
def oh_no(_):
raise ValueError("hi")
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=2, ignore_exceptions=True
)
)
== [None] * 3
)
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=1, ignore_exceptions=True
)
)
== [None] * 3
) )
def test_no_op(self) -> None:
assert list(threaded_parallel_map(lambda v: v**2, [])) == [] def test_ignore_this_worker_exception() -> None:
assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == [] def my_generator():
assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == [] yield 1
yield 2
yield 3
yield 1 / 0
assert list(
threaded_parallel_map(
lambda v: v**2,
my_generator(),
concurrency=2,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4]
assert list(
threaded_parallel_map(
lambda v: v**2,
my_generator(),
concurrency=1,
chunk_size=2,
ignore_exceptions=True,
)
) == [1, 4, 9]
def test_worker_worker_exception() -> None:
def oh_no(_):
raise ValueError("hi")
with pytest.raises(WorkerException):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=2))
with pytest.raises(WorkerException):
list(threaded_parallel_map(oh_no, range(COUNT), concurrency=1))
def test_ignore_worker_worker_exception() -> None:
def oh_no(_):
raise ValueError("hi")
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=2, ignore_exceptions=True
)
)
== [None] * 3
)
assert (
list(
threaded_parallel_map(
oh_no, range(3), concurrency=1, ignore_exceptions=True
)
)
== [None] * 3
)
def test_no_op() -> None:
assert list(threaded_parallel_map(lambda v: v**2, [])) == []
assert list(threaded_parallel_map(lambda v: v**2, [], chunk_size=100)) == []
assert list(threaded_parallel_map(lambda v: v**2, [], concurrency=100)) == []

67
tests/utilities/test_unique.py
View file

@ -1,5 +1,3 @@
import unittest
from great_ai.utilities import unique from great_ai.utilities import unique
original = [ original = [
@ -12,43 +10,44 @@ original = [
] ]
class TestUnique(unittest.TestCase): def test_default() -> None:
def test_default(self) -> None: values = [
values = [ *original,
*original, ("a", 2),
("a", 2), ("a", 1),
("a", 1), ("a", -3),
("a", -3), ("b", 5),
("b", 5), ("c", 5),
("c", 5), ("d", 2),
("d", 2), ]
]
expected = original expected = original
assert unique(values) == expected assert unique(values) == expected
assert unique(values, key=lambda v: v) == expected assert unique(values, key=lambda v: v) == expected
def test_with_key_0(self) -> None:
values = original
expected = [ def test_with_key_0() -> None:
("a", 1), values = original
("b", 5),
("c", 5),
("d", 2),
]
assert unique(values, key=lambda v: v[0]) == expected expected = [
("a", 1),
("b", 5),
("c", 5),
("d", 2),
]
def test_with_key_1(self) -> None: assert unique(values, key=lambda v: v[0]) == expected
values = original
expected = [
("a", 1),
("b", 5),
("a", -3),
("d", 2),
]
assert unique(values, key=lambda v: v[1]) == expected def test_with_key_1() -> None:
values = original
expected = [
("a", 1),
("b", 5),
("a", -3),
("d", 2),
]
assert unique(values, key=lambda v: v[1]) == expected