902 lines
33 KiB
Text
902 lines
33 KiB
Text
{
|
||
"cells": [
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 48,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"executionInfo": {
|
||
"elapsed": 2529,
|
||
"status": "ok",
|
||
"timestamp": 1656596749103,
|
||
"user": {
|
||
"displayName": "Schmelczer András",
|
||
"userId": "08401926777942666437"
|
||
},
|
||
"user_tz": -120
|
||
},
|
||
"id": "j7l0nD9hDQbB",
|
||
"outputId": "88a9931b-396a-4cf1-c659-8a7b098b3cdd"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"evaluation-experiment-2-stage #1-2m6dmb.json\n",
|
||
"evaluation-experiment-2-stage #1-sa6a0y.json\n",
|
||
"Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n",
|
||
"Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.20.1)\n",
|
||
"Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.3.2)\n",
|
||
"\u001b[31mERROR: Could not find a version that satisfies the requirement great-ai==0.0.6 (from versions: none)\u001b[0m\n",
|
||
"\u001b[31mERROR: No matching distribution found for great-ai==0.0.6\u001b[0m\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from pathlib import Path\n",
|
||
"import json\n",
|
||
"\n",
|
||
"annotations = []\n",
|
||
"for p in Path(\".\").glob(\"*.json\"):\n",
|
||
" with open(p, encoding=\"utf-8\") as f:\n",
|
||
" print(p)\n",
|
||
" annotations.append(json.load(f))\n",
|
||
"\n",
|
||
"evaluations = {\n",
|
||
" sentence: [\n",
|
||
" annotation[sentence] for annotation in annotations if sentence in annotation\n",
|
||
" ]\n",
|
||
" for sentence in {\n",
|
||
" sentence for annotation in annotations for sentence in annotation.keys()\n",
|
||
" }\n",
|
||
"}\n",
|
||
"\n",
|
||
"X = [s for s in evaluations.keys()]\n",
|
||
"y = [int(sum(e) > 0) for e in evaluations.values()]\n",
|
||
"\n",
|
||
"# !pip install transformers datasets great-ai==0.0.6"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 22,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/",
|
||
"height": 1000,
|
||
"referenced_widgets": [
|
||
"9c57de70e68a41ecbde5093bd671715a",
|
||
"9185277b1e5945a9a6a3ad75811bc86b",
|
||
"4be5b3c59dc04aa2b92f51654e815589",
|
||
"8e0ede9d1dd84c149a7e282211c7071b",
|
||
"9bd5d2fb87bd428796155cc67d06b333",
|
||
"bf773a86ec0a4899bb3636035f7ab35e",
|
||
"22119b79eb514ad684cc3f00f519fb4a",
|
||
"cb7ec6240337466d8833c70083e1c3cb",
|
||
"34631de39509438aad98cbd3fc64c999",
|
||
"32adc54185894f0598c2d9ad438c76e2",
|
||
"981e11fb9d4f4a2ba28c011741a1eaba"
|
||
]
|
||
},
|
||
"executionInfo": {
|
||
"elapsed": 118131,
|
||
"status": "ok",
|
||
"timestamp": 1656593941974,
|
||
"user": {
|
||
"displayName": "Schmelczer András",
|
||
"userId": "08401926777942666437"
|
||
},
|
||
"user_tz": -120
|
||
},
|
||
"id": "AL3etUQ3LtKN",
|
||
"outputId": "fe00589f-64dd-4b70-e612-3873b504c00a"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Could not locate the tokenizer configuration file, will try to use the model config instead.\n",
|
||
"loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
|
||
"Model config BertConfig {\n",
|
||
" \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
|
||
" \"attention_probs_dropout_prob\": 0.1,\n",
|
||
" \"classifier_dropout\": null,\n",
|
||
" \"hidden_act\": \"gelu\",\n",
|
||
" \"hidden_dropout_prob\": 0.1,\n",
|
||
" \"hidden_size\": 768,\n",
|
||
" \"initializer_range\": 0.02,\n",
|
||
" \"intermediate_size\": 3072,\n",
|
||
" \"layer_norm_eps\": 1e-12,\n",
|
||
" \"max_position_embeddings\": 512,\n",
|
||
" \"model_type\": \"bert\",\n",
|
||
" \"num_attention_heads\": 12,\n",
|
||
" \"num_hidden_layers\": 12,\n",
|
||
" \"pad_token_id\": 0,\n",
|
||
" \"position_embedding_type\": \"absolute\",\n",
|
||
" \"transformers_version\": \"4.20.1\",\n",
|
||
" \"type_vocab_size\": 2,\n",
|
||
" \"use_cache\": true,\n",
|
||
" \"vocab_size\": 31090\n",
|
||
"}\n",
|
||
"\n",
|
||
"loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/33593020f507d72099bd84ea6cd2296feb424fecd62d4a8edcc2a02899af6e29.38339d84e6e392addd730fd85fae32652c4cc7c5423633d6fa73e5f7937bbc38\n",
|
||
"loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer.json from cache at None\n",
|
||
"loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/added_tokens.json from cache at None\n",
|
||
"loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/special_tokens_map.json from cache at None\n",
|
||
"loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer_config.json from cache at None\n",
|
||
"loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
|
||
"Model config BertConfig {\n",
|
||
" \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
|
||
" \"attention_probs_dropout_prob\": 0.1,\n",
|
||
" \"classifier_dropout\": null,\n",
|
||
" \"hidden_act\": \"gelu\",\n",
|
||
" \"hidden_dropout_prob\": 0.1,\n",
|
||
" \"hidden_size\": 768,\n",
|
||
" \"initializer_range\": 0.02,\n",
|
||
" \"intermediate_size\": 3072,\n",
|
||
" \"layer_norm_eps\": 1e-12,\n",
|
||
" \"max_position_embeddings\": 512,\n",
|
||
" \"model_type\": \"bert\",\n",
|
||
" \"num_attention_heads\": 12,\n",
|
||
" \"num_hidden_layers\": 12,\n",
|
||
" \"pad_token_id\": 0,\n",
|
||
" \"position_embedding_type\": \"absolute\",\n",
|
||
" \"transformers_version\": \"4.20.1\",\n",
|
||
" \"type_vocab_size\": 2,\n",
|
||
" \"use_cache\": true,\n",
|
||
" \"vocab_size\": 31090\n",
|
||
"}\n",
|
||
"\n",
|
||
"loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
|
||
"Model config BertConfig {\n",
|
||
" \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
|
||
" \"attention_probs_dropout_prob\": 0.1,\n",
|
||
" \"classifier_dropout\": null,\n",
|
||
" \"hidden_act\": \"gelu\",\n",
|
||
" \"hidden_dropout_prob\": 0.1,\n",
|
||
" \"hidden_size\": 768,\n",
|
||
" \"initializer_range\": 0.02,\n",
|
||
" \"intermediate_size\": 3072,\n",
|
||
" \"layer_norm_eps\": 1e-12,\n",
|
||
" \"max_position_embeddings\": 512,\n",
|
||
" \"model_type\": \"bert\",\n",
|
||
" \"num_attention_heads\": 12,\n",
|
||
" \"num_hidden_layers\": 12,\n",
|
||
" \"pad_token_id\": 0,\n",
|
||
" \"position_embedding_type\": \"absolute\",\n",
|
||
" \"transformers_version\": \"4.20.1\",\n",
|
||
" \"type_vocab_size\": 2,\n",
|
||
" \"use_cache\": true,\n",
|
||
" \"vocab_size\": 31090\n",
|
||
"}\n",
|
||
"\n",
|
||
"loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n",
|
||
"Model config BertConfig {\n",
|
||
" \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n",
|
||
" \"attention_probs_dropout_prob\": 0.1,\n",
|
||
" \"classifier_dropout\": null,\n",
|
||
" \"hidden_act\": \"gelu\",\n",
|
||
" \"hidden_dropout_prob\": 0.1,\n",
|
||
" \"hidden_size\": 768,\n",
|
||
" \"initializer_range\": 0.02,\n",
|
||
" \"intermediate_size\": 3072,\n",
|
||
" \"layer_norm_eps\": 1e-12,\n",
|
||
" \"max_position_embeddings\": 512,\n",
|
||
" \"model_type\": \"bert\",\n",
|
||
" \"num_attention_heads\": 12,\n",
|
||
" \"num_hidden_layers\": 12,\n",
|
||
" \"pad_token_id\": 0,\n",
|
||
" \"position_embedding_type\": \"absolute\",\n",
|
||
" \"transformers_version\": \"4.20.1\",\n",
|
||
" \"type_vocab_size\": 2,\n",
|
||
" \"use_cache\": true,\n",
|
||
" \"vocab_size\": 31090\n",
|
||
"}\n",
|
||
"\n",
|
||
"loading weights file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/de14937a851e8180a2bc5660c0041d385f8a0c62b1b2ccafa46df31043a2390c.74830bb01a0ffcdeaed8be9916312726d0c4cd364ac6fc15b375f789eaff4cbb\n",
|
||
"Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n",
|
||
"- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||
"- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||
"Some weights of BertForSequenceClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n",
|
||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"application/vnd.jupyter.widget-view+json": {
|
||
"model_id": "9c57de70e68a41ecbde5093bd671715a",
|
||
"version_major": 2,
|
||
"version_minor": 0
|
||
},
|
||
"text/plain": [
|
||
" 0%| | 0/1 [00:00<?, ?ba/s]"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n",
|
||
"PyTorch: setting up devices\n",
|
||
"The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n",
|
||
"/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n",
|
||
" FutureWarning,\n",
|
||
"***** Running training *****\n",
|
||
" Num examples = 400\n",
|
||
" Num Epochs = 50\n",
|
||
" Instantaneous batch size per device = 32\n",
|
||
" Total train batch size (w. parallel, distributed & accumulation) = 32\n",
|
||
" Gradient Accumulation steps = 1\n",
|
||
" Total optimization steps = 650\n"
|
||
]
|
||
},
|
||
{
|
||
"data": {
|
||
"text/html": [
|
||
"\n",
|
||
" <div>\n",
|
||
" \n",
|
||
" <progress value='130' max='650' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
|
||
" [130/650 01:43 < 07:01, 1.23 it/s, Epoch 10/50]\n",
|
||
" </div>\n",
|
||
" <table border=\"1\" class=\"dataframe\">\n",
|
||
" <thead>\n",
|
||
" <tr style=\"text-align: left;\">\n",
|
||
" <th>Epoch</th>\n",
|
||
" <th>Training Loss</th>\n",
|
||
" <th>Validation Loss</th>\n",
|
||
" <th>F1</th>\n",
|
||
" </tr>\n",
|
||
" </thead>\n",
|
||
" <tbody>\n",
|
||
" <tr>\n",
|
||
" <td>1</td>\n",
|
||
" <td>0.586800</td>\n",
|
||
" <td>0.512138</td>\n",
|
||
" <td>0.719101</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>2</td>\n",
|
||
" <td>0.411600</td>\n",
|
||
" <td>0.416675</td>\n",
|
||
" <td>0.849057</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>3</td>\n",
|
||
" <td>0.245600</td>\n",
|
||
" <td>0.417070</td>\n",
|
||
" <td>0.864000</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>4</td>\n",
|
||
" <td>0.147800</td>\n",
|
||
" <td>0.575878</td>\n",
|
||
" <td>0.852459</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>5</td>\n",
|
||
" <td>0.056800</td>\n",
|
||
" <td>0.474259</td>\n",
|
||
" <td>0.896552</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>6</td>\n",
|
||
" <td>0.022500</td>\n",
|
||
" <td>0.754236</td>\n",
|
||
" <td>0.843137</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>7</td>\n",
|
||
" <td>0.001000</td>\n",
|
||
" <td>0.857636</td>\n",
|
||
" <td>0.834783</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>8</td>\n",
|
||
" <td>0.000500</td>\n",
|
||
" <td>0.920232</td>\n",
|
||
" <td>0.869565</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>9</td>\n",
|
||
" <td>0.000300</td>\n",
|
||
" <td>0.970790</td>\n",
|
||
" <td>0.877193</td>\n",
|
||
" </tr>\n",
|
||
" <tr>\n",
|
||
" <td>10</td>\n",
|
||
" <td>0.000300</td>\n",
|
||
" <td>0.948689</td>\n",
|
||
" <td>0.862385</td>\n",
|
||
" </tr>\n",
|
||
" </tbody>\n",
|
||
"</table><p>"
|
||
],
|
||
"text/plain": [
|
||
"<IPython.core.display.HTML object>"
|
||
]
|
||
},
|
||
"metadata": {},
|
||
"output_type": "display_data"
|
||
},
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-13\n",
|
||
"Configuration saved in models/checkpoint-13/config.json\n",
|
||
"Model weights saved in models/checkpoint-13/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-91] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-26\n",
|
||
"Configuration saved in models/checkpoint-26/config.json\n",
|
||
"Model weights saved in models/checkpoint-26/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-117] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-39\n",
|
||
"Configuration saved in models/checkpoint-39/config.json\n",
|
||
"Model weights saved in models/checkpoint-39/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-130] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-52\n",
|
||
"Configuration saved in models/checkpoint-52/config.json\n",
|
||
"Model weights saved in models/checkpoint-52/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-143] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-65\n",
|
||
"Configuration saved in models/checkpoint-65/config.json\n",
|
||
"Model weights saved in models/checkpoint-65/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-156] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-78\n",
|
||
"Configuration saved in models/checkpoint-78/config.json\n",
|
||
"Model weights saved in models/checkpoint-78/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-13] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-91\n",
|
||
"Configuration saved in models/checkpoint-91/config.json\n",
|
||
"Model weights saved in models/checkpoint-91/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-26] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-104\n",
|
||
"Configuration saved in models/checkpoint-104/config.json\n",
|
||
"Model weights saved in models/checkpoint-104/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-39] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-117\n",
|
||
"Configuration saved in models/checkpoint-117/config.json\n",
|
||
"Model weights saved in models/checkpoint-117/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-52] due to args.save_total_limit\n",
|
||
"***** Running Evaluation *****\n",
|
||
" Num examples = 100\n",
|
||
" Batch size = 32\n",
|
||
"Saving model checkpoint to models/checkpoint-130\n",
|
||
"Configuration saved in models/checkpoint-130/config.json\n",
|
||
"Model weights saved in models/checkpoint-130/pytorch_model.bin\n",
|
||
"Deleting older checkpoint [models/checkpoint-78] due to args.save_total_limit\n",
|
||
"\n",
|
||
"\n",
|
||
"Training completed. Do not forget to share your model on huggingface.co/models =)\n",
|
||
"\n",
|
||
"\n",
|
||
"Loading best model from models/checkpoint-65 (score: 0.896551724137931).\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from transformers import (\n",
|
||
" AutoModelForSequenceClassification,\n",
|
||
" AutoTokenizer,\n",
|
||
" DataCollatorWithPadding,\n",
|
||
" Trainer,\n",
|
||
" TrainingArguments,\n",
|
||
" EarlyStoppingCallback,\n",
|
||
")\n",
|
||
"import numpy as np\n",
|
||
"from datasets import Dataset, load_metric\n",
|
||
"\n",
|
||
"MODEL = \"allenai/scibert_scivocab_uncased\"\n",
|
||
"BATCH_SIZE = 32\n",
|
||
"\n",
|
||
"tokenizer = AutoTokenizer.from_pretrained(MODEL)\n",
|
||
"model = AutoModelForSequenceClassification.from_pretrained(MODEL, num_labels=2)\n",
|
||
"data_collator = DataCollatorWithPadding(tokenizer=tokenizer)\n",
|
||
"\n",
|
||
"\n",
|
||
"def tokenize_function(v):\n",
|
||
" return tokenizer(v[\"text\"])\n",
|
||
"\n",
|
||
"\n",
|
||
"dataset = (\n",
|
||
" Dataset.from_dict({\"text\": X, \"label\": y})\n",
|
||
" .map(lambda v: tokenizer(v[\"text\"], truncation=True), batched=True)\n",
|
||
" .remove_columns(\"text\")\n",
|
||
" .train_test_split(test_size=0.2, shuffle=True)\n",
|
||
")\n",
|
||
"\n",
|
||
"f1_score = load_metric(\"f1\")\n",
|
||
"\n",
|
||
"\n",
|
||
"def compute_metrics(p):\n",
|
||
" pred, labels = p\n",
|
||
" pred = np.argmax(pred, axis=1)\n",
|
||
" return f1_score.compute(predictions=pred, references=labels)\n",
|
||
"\n",
|
||
"\n",
|
||
"training_args = TrainingArguments(\n",
|
||
" output_dir=Path(\"models\"),\n",
|
||
" per_device_train_batch_size=BATCH_SIZE,\n",
|
||
" per_device_eval_batch_size=BATCH_SIZE,\n",
|
||
" save_total_limit=5,\n",
|
||
" num_train_epochs=50,\n",
|
||
" save_strategy=\"epoch\",\n",
|
||
" evaluation_strategy=\"epoch\",\n",
|
||
" logging_strategy=\"epoch\",\n",
|
||
" weight_decay=0.01,\n",
|
||
" metric_for_best_model=\"f1\",\n",
|
||
" load_best_model_at_end=True,\n",
|
||
")\n",
|
||
"\n",
|
||
"result = Trainer(\n",
|
||
" model=model,\n",
|
||
" args=training_args,\n",
|
||
" train_dataset=dataset[\"train\"],\n",
|
||
" eval_dataset=dataset[\"test\"],\n",
|
||
" data_collator=data_collator,\n",
|
||
" compute_metrics=compute_metrics,\n",
|
||
" callbacks=[EarlyStoppingCallback(early_stopping_patience=5)],\n",
|
||
").train()"
|
||
]
|
||
},
|
||
{
|
||
"cell_type": "code",
|
||
"execution_count": 44,
|
||
"metadata": {
|
||
"colab": {
|
||
"base_uri": "https://localhost:8080/"
|
||
},
|
||
"executionInfo": {
|
||
"elapsed": 25368,
|
||
"status": "ok",
|
||
"timestamp": 1656594537509,
|
||
"user": {
|
||
"displayName": "Schmelczer András",
|
||
"userId": "08401926777942666437"
|
||
},
|
||
"user_tz": -120
|
||
},
|
||
"id": "fyNKltdquZSP",
|
||
"outputId": "e8c2cbb1-78e1-41a3-b7cf-b0cd573bc45d"
|
||
},
|
||
"outputs": [
|
||
{
|
||
"name": "stderr",
|
||
"output_type": "stream",
|
||
"text": [
|
||
"Configuration saved in pretrained/config.json\n",
|
||
"Model weights saved in pretrained/pytorch_model.bin\n"
|
||
]
|
||
},
|
||
{
|
||
"name": "stdout",
|
||
"output_type": "stream",
|
||
"text": [
|
||
" adding: pretrained/ (stored 0%)\n",
|
||
" adding: pretrained/config.json (deflated 49%)\n",
|
||
" adding: pretrained/pytorch_model.bin (deflated 7%)\n"
|
||
]
|
||
}
|
||
],
|
||
"source": [
|
||
"from great_ai.large_file import LargeFileS3\n",
|
||
"\n",
|
||
"LargeFileS3.configure_credentials_from_file(\"config.ini\")\n",
|
||
"\n",
|
||
"model.save_pretrained(\"pretrained\")\n",
|
||
"LargeFileS3(\"scibert-highlights\").push(\"pretrained\")"
|
||
]
|
||
}
|
||
],
|
||
"metadata": {
|
||
"accelerator": "GPU",
|
||
"colab": {
|
||
"collapsed_sections": [],
|
||
"name": "Copy of Forms snippets",
|
||
"provenance": [
|
||
{
|
||
"file_id": "/v2/external/notebooks/snippets/forms.ipynb",
|
||
"timestamp": 1656585404621
|
||
}
|
||
]
|
||
},
|
||
"gpuClass": "standard",
|
||
"kernelspec": {
|
||
"display_name": "Python 3.10.4 ('.env': venv)",
|
||
"language": "python",
|
||
"name": "python3"
|
||
},
|
||
"language_info": {
|
||
"name": "python",
|
||
"version": "3.10.4"
|
||
},
|
||
"vscode": {
|
||
"interpreter": {
|
||
"hash": "02dd6d3afbfa9fbbe1037d64ad9014965528a1ccad21929d6e72f466389a68ad"
|
||
}
|
||
},
|
||
"widgets": {
|
||
"application/vnd.jupyter.widget-state+json": {
|
||
"22119b79eb514ad684cc3f00f519fb4a": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "DescriptionStyleModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "DescriptionStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"description_width": ""
|
||
}
|
||
},
|
||
"32adc54185894f0598c2d9ad438c76e2": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_module_version": "1.2.0",
|
||
"model_name": "LayoutModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"34631de39509438aad98cbd3fc64c999": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "ProgressStyleModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "ProgressStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"bar_color": null,
|
||
"description_width": ""
|
||
}
|
||
},
|
||
"4be5b3c59dc04aa2b92f51654e815589": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "FloatProgressModel",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "FloatProgressModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "ProgressView",
|
||
"bar_style": "success",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_cb7ec6240337466d8833c70083e1c3cb",
|
||
"max": 1,
|
||
"min": 0,
|
||
"orientation": "horizontal",
|
||
"style": "IPY_MODEL_34631de39509438aad98cbd3fc64c999",
|
||
"value": 1
|
||
}
|
||
},
|
||
"8e0ede9d1dd84c149a7e282211c7071b": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "HTMLModel",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HTMLModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HTMLView",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_32adc54185894f0598c2d9ad438c76e2",
|
||
"placeholder": "",
|
||
"style": "IPY_MODEL_981e11fb9d4f4a2ba28c011741a1eaba",
|
||
"value": " 1/1 [00:00<00:00, 5.38ba/s]"
|
||
}
|
||
},
|
||
"9185277b1e5945a9a6a3ad75811bc86b": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "HTMLModel",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HTMLModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HTMLView",
|
||
"description": "",
|
||
"description_tooltip": null,
|
||
"layout": "IPY_MODEL_bf773a86ec0a4899bb3636035f7ab35e",
|
||
"placeholder": "",
|
||
"style": "IPY_MODEL_22119b79eb514ad684cc3f00f519fb4a",
|
||
"value": "100%"
|
||
}
|
||
},
|
||
"981e11fb9d4f4a2ba28c011741a1eaba": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "DescriptionStyleModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "DescriptionStyleModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "StyleView",
|
||
"description_width": ""
|
||
}
|
||
},
|
||
"9bd5d2fb87bd428796155cc67d06b333": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_module_version": "1.2.0",
|
||
"model_name": "LayoutModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"9c57de70e68a41ecbde5093bd671715a": {
|
||
"model_module": "@jupyter-widgets/controls",
|
||
"model_module_version": "1.5.0",
|
||
"model_name": "HBoxModel",
|
||
"state": {
|
||
"_dom_classes": [],
|
||
"_model_module": "@jupyter-widgets/controls",
|
||
"_model_module_version": "1.5.0",
|
||
"_model_name": "HBoxModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/controls",
|
||
"_view_module_version": "1.5.0",
|
||
"_view_name": "HBoxView",
|
||
"box_style": "",
|
||
"children": [
|
||
"IPY_MODEL_9185277b1e5945a9a6a3ad75811bc86b",
|
||
"IPY_MODEL_4be5b3c59dc04aa2b92f51654e815589",
|
||
"IPY_MODEL_8e0ede9d1dd84c149a7e282211c7071b"
|
||
],
|
||
"layout": "IPY_MODEL_9bd5d2fb87bd428796155cc67d06b333"
|
||
}
|
||
},
|
||
"bf773a86ec0a4899bb3636035f7ab35e": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_module_version": "1.2.0",
|
||
"model_name": "LayoutModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
},
|
||
"cb7ec6240337466d8833c70083e1c3cb": {
|
||
"model_module": "@jupyter-widgets/base",
|
||
"model_module_version": "1.2.0",
|
||
"model_name": "LayoutModel",
|
||
"state": {
|
||
"_model_module": "@jupyter-widgets/base",
|
||
"_model_module_version": "1.2.0",
|
||
"_model_name": "LayoutModel",
|
||
"_view_count": null,
|
||
"_view_module": "@jupyter-widgets/base",
|
||
"_view_module_version": "1.2.0",
|
||
"_view_name": "LayoutView",
|
||
"align_content": null,
|
||
"align_items": null,
|
||
"align_self": null,
|
||
"border": null,
|
||
"bottom": null,
|
||
"display": null,
|
||
"flex": null,
|
||
"flex_flow": null,
|
||
"grid_area": null,
|
||
"grid_auto_columns": null,
|
||
"grid_auto_flow": null,
|
||
"grid_auto_rows": null,
|
||
"grid_column": null,
|
||
"grid_gap": null,
|
||
"grid_row": null,
|
||
"grid_template_areas": null,
|
||
"grid_template_columns": null,
|
||
"grid_template_rows": null,
|
||
"height": null,
|
||
"justify_content": null,
|
||
"justify_items": null,
|
||
"left": null,
|
||
"margin": null,
|
||
"max_height": null,
|
||
"max_width": null,
|
||
"min_height": null,
|
||
"min_width": null,
|
||
"object_fit": null,
|
||
"object_position": null,
|
||
"order": null,
|
||
"overflow": null,
|
||
"overflow_x": null,
|
||
"overflow_y": null,
|
||
"padding": null,
|
||
"right": null,
|
||
"top": null,
|
||
"visibility": null,
|
||
"width": null
|
||
}
|
||
}
|
||
}
|
||
}
|
||
},
|
||
"nbformat": 4,
|
||
"nbformat_minor": 0
|
||
}
|