|
|
|
|
@ -24,10 +24,10 @@
|
|
|
|
|
"from sklearn.feature_extraction.text import TfidfVectorizer\n",
|
|
|
|
|
"from sklearn.model_selection import GridSearchCV\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"from good_ai.utilities.clean import clean\n",
|
|
|
|
|
"from good_ai.utilities.parallel_map import parallel_map\n",
|
|
|
|
|
"from good_ai.utilities.language import is_english, predict_language\n",
|
|
|
|
|
"from good_ai import save_model, set_default_config, LargeFile\n",
|
|
|
|
|
"from great_ai.utilities.clean import clean\n",
|
|
|
|
|
"from great_ai.utilities.parallel_map import parallel_map\n",
|
|
|
|
|
"from great_ai.utilities.language import is_english, predict_language\n",
|
|
|
|
|
"from great_ai import save_model, set_default_config, LargeFile\n",
|
|
|
|
|
"\n",
|
|
|
|
|
"from preprocess import preprocess"
|
|
|
|
|
]
|
|
|
|
|
@ -97,56 +97,56 @@
|
|
|
|
|
"name": "stderr",
|
|
|
|
|
"output_type": "stream",
|
|
|
|
|
"text": [
|
|
|
|
|
" 0%| | 0/5 [00:00<?, ?it/s]INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7093)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,811)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,196)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,561)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1370)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,330)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,603)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing environment: 'document' @(26,0)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,533)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6244)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6549)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,997)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,549)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\)' @(1,674)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,748)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,70)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,112)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,175)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,321)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,700)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1180)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,754)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,487)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,390)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,942)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1471)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2235)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,27)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1176)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2502)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2545)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2856)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,257)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,391)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,648)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,346)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,958)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,964)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1017)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1287)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,104)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,119)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1139)\n",
|
|
|
|
|
"INFO:good_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,366)\n",
|
|
|
|
|
" 0%| | 0/5 [00:00<?, ?it/s]INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,7093)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,811)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,196)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,561)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1370)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,215)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,330)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,603)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing environment: 'document' @(26,0)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,533)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,713)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6244)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,6549)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,997)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,549)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\)' @(1,674)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,748)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,70)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,128)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,112)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,175)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,321)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected closing math mode: '\\]' @(1,700)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1180)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,754)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,487)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,390)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,942)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1471)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2235)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,27)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1176)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2502)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2545)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,2856)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,84)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,257)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,391)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,648)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,346)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,716)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,782)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,958)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,964)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1017)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1287)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,104)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,119)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,1139)\n",
|
|
|
|
|
"INFO:great_ai.utilities.external.pylatexenc.latexwalker:Ignoring parse error (tolerant parsing mode): Unexpected mismatching closing brace: '}' @(1,366)\n",
|
|
|
|
|
"100%|██████████| 5/5 [34:00<00:00, 408.07s/it] \n"
|
|
|
|
|
]
|
|
|
|
|
}
|
|
|
|
|
|