{ "cells": [ { "cell_type": "code", "execution_count": 48, "metadata": { "colab": { "base_uri": "https://localhost:8080/" }, "executionInfo": { "elapsed": 2529, "status": "ok", "timestamp": 1656596749103, "user": { "displayName": "Schmelczer András", "userId": "08401926777942666437" }, "user_tz": -120 }, "id": "j7l0nD9hDQbB", "outputId": "88a9931b-396a-4cf1-c659-8a7b098b3cdd" }, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "evaluation-experiment-2-stage #1-2m6dmb.json\n", "evaluation-experiment-2-stage #1-sa6a0y.json\n", "Looking in indexes: https://pypi.org/simple, https://us-python.pkg.dev/colab-wheels/public/simple/\n", "Requirement already satisfied: transformers in /usr/local/lib/python3.7/dist-packages (4.20.1)\n", "Requirement already satisfied: datasets in /usr/local/lib/python3.7/dist-packages (2.3.2)\n", "\u001b[31mERROR: Could not find a version that satisfies the requirement great-ai==0.0.6 (from versions: none)\u001b[0m\n", "\u001b[31mERROR: No matching distribution found for great-ai==0.0.6\u001b[0m\n" ] } ], "source": [ "from pathlib import Path\n", "import json\n", "\n", "annotations = []\n", "for p in Path(\".\").glob(\"*.json\"):\n", " with open(p, encoding=\"utf-8\") as f:\n", " print(p)\n", " annotations.append(json.load(f))\n", "\n", "evaluations = {\n", " sentence: [\n", " annotation[sentence] for annotation in annotations if sentence in annotation\n", " ]\n", " for sentence in {\n", " sentence for annotation in annotations for sentence in annotation.keys()\n", " }\n", "}\n", "\n", "X = [s for s in evaluations.keys()]\n", "y = [int(sum(e) > 0) for e in evaluations.values()]\n", "\n", "# !pip install transformers datasets great-ai==0.0.6" ] }, { "cell_type": "code", "execution_count": 22, "metadata": { "colab": { "base_uri": "https://localhost:8080/", "height": 1000, "referenced_widgets": [ "9c57de70e68a41ecbde5093bd671715a", "9185277b1e5945a9a6a3ad75811bc86b", "4be5b3c59dc04aa2b92f51654e815589", "8e0ede9d1dd84c149a7e282211c7071b", "9bd5d2fb87bd428796155cc67d06b333", "bf773a86ec0a4899bb3636035f7ab35e", "22119b79eb514ad684cc3f00f519fb4a", "cb7ec6240337466d8833c70083e1c3cb", "34631de39509438aad98cbd3fc64c999", "32adc54185894f0598c2d9ad438c76e2", "981e11fb9d4f4a2ba28c011741a1eaba" ] }, "executionInfo": { "elapsed": 118131, "status": "ok", "timestamp": 1656593941974, "user": { "displayName": "Schmelczer András", "userId": "08401926777942666437" }, "user_tz": -120 }, "id": "AL3etUQ3LtKN", "outputId": "fe00589f-64dd-4b70-e612-3873b504c00a" }, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ "Could not locate the tokenizer configuration file, will try to use the model config instead.\n", "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", "Model config BertConfig {\n", " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"classifier_dropout\": null,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"layer_norm_eps\": 1e-12,\n", " \"max_position_embeddings\": 512,\n", " \"model_type\": \"bert\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 0,\n", " \"position_embedding_type\": \"absolute\",\n", " \"transformers_version\": \"4.20.1\",\n", " \"type_vocab_size\": 2,\n", " \"use_cache\": true,\n", " \"vocab_size\": 31090\n", "}\n", "\n", "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/vocab.txt from cache at /root/.cache/huggingface/transformers/33593020f507d72099bd84ea6cd2296feb424fecd62d4a8edcc2a02899af6e29.38339d84e6e392addd730fd85fae32652c4cc7c5423633d6fa73e5f7937bbc38\n", "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer.json from cache at None\n", "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/added_tokens.json from cache at None\n", "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/special_tokens_map.json from cache at None\n", "loading file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/tokenizer_config.json from cache at None\n", "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", "Model config BertConfig {\n", " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"classifier_dropout\": null,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"layer_norm_eps\": 1e-12,\n", " \"max_position_embeddings\": 512,\n", " \"model_type\": \"bert\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 0,\n", " \"position_embedding_type\": \"absolute\",\n", " \"transformers_version\": \"4.20.1\",\n", " \"type_vocab_size\": 2,\n", " \"use_cache\": true,\n", " \"vocab_size\": 31090\n", "}\n", "\n", "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", "Model config BertConfig {\n", " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"classifier_dropout\": null,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"layer_norm_eps\": 1e-12,\n", " \"max_position_embeddings\": 512,\n", " \"model_type\": \"bert\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 0,\n", " \"position_embedding_type\": \"absolute\",\n", " \"transformers_version\": \"4.20.1\",\n", " \"type_vocab_size\": 2,\n", " \"use_cache\": true,\n", " \"vocab_size\": 31090\n", "}\n", "\n", "loading configuration file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/config.json from cache at /root/.cache/huggingface/transformers/858852fd2471ce39075378592ddc87f5a6551e64c6825d1b92c8dab9318e0fc3.03ff9e9f998b9a9d40647a2148a202e3fb3d568dc0f170dda9dda194bab4d5dd\n", "Model config BertConfig {\n", " \"_name_or_path\": \"allenai/scibert_scivocab_uncased\",\n", " \"attention_probs_dropout_prob\": 0.1,\n", " \"classifier_dropout\": null,\n", " \"hidden_act\": \"gelu\",\n", " \"hidden_dropout_prob\": 0.1,\n", " \"hidden_size\": 768,\n", " \"initializer_range\": 0.02,\n", " \"intermediate_size\": 3072,\n", " \"layer_norm_eps\": 1e-12,\n", " \"max_position_embeddings\": 512,\n", " \"model_type\": \"bert\",\n", " \"num_attention_heads\": 12,\n", " \"num_hidden_layers\": 12,\n", " \"pad_token_id\": 0,\n", " \"position_embedding_type\": \"absolute\",\n", " \"transformers_version\": \"4.20.1\",\n", " \"type_vocab_size\": 2,\n", " \"use_cache\": true,\n", " \"vocab_size\": 31090\n", "}\n", "\n", "loading weights file https://huggingface.co/allenai/scibert_scivocab_uncased/resolve/main/pytorch_model.bin from cache at /root/.cache/huggingface/transformers/de14937a851e8180a2bc5660c0041d385f8a0c62b1b2ccafa46df31043a2390c.74830bb01a0ffcdeaed8be9916312726d0c4cd364ac6fc15b375f789eaff4cbb\n", "Some weights of the model checkpoint at allenai/scibert_scivocab_uncased were not used when initializing BertForSequenceClassification: ['cls.seq_relationship.bias', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.bias', 'cls.seq_relationship.weight', 'cls.predictions.decoder.bias', 'cls.predictions.decoder.weight', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight']\n", "- This IS expected if you are initializing BertForSequenceClassification from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n", "- This IS NOT expected if you are initializing BertForSequenceClassification from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n", "Some weights of BertForSequenceClassification were not initialized from the model checkpoint at allenai/scibert_scivocab_uncased and are newly initialized: ['classifier.bias', 'classifier.weight']\n", "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" ] }, { "data": { "application/vnd.jupyter.widget-view+json": { "model_id": "9c57de70e68a41ecbde5093bd671715a", "version_major": 2, "version_minor": 0 }, "text/plain": [ " 0%| | 0/1 [00:00, ?ba/s]" ] }, "metadata": {}, "output_type": "display_data" }, { "name": "stderr", "output_type": "stream", "text": [ "Asking to truncate to max_length but no maximum length is provided and the model has no predefined maximum length. Default to no truncation.\n", "PyTorch: setting up devices\n", "The default value for the training argument `--report_to` will change in v5 (from all installed integrations to none). In v5, you will need to use `--report_to all` to get the same behavior as now. You should start updating your code and make this info disappear :-).\n", "/usr/local/lib/python3.7/dist-packages/transformers/optimization.py:310: FutureWarning: This implementation of AdamW is deprecated and will be removed in a future version. Use the PyTorch implementation torch.optim.AdamW instead, or set `no_deprecation_warning=True` to disable this warning\n", " FutureWarning,\n", "***** Running training *****\n", " Num examples = 400\n", " Num Epochs = 50\n", " Instantaneous batch size per device = 32\n", " Total train batch size (w. parallel, distributed & accumulation) = 32\n", " Gradient Accumulation steps = 1\n", " Total optimization steps = 650\n" ] }, { "data": { "text/html": [ "\n", "
| Epoch | \n", "Training Loss | \n", "Validation Loss | \n", "F1 | \n", "
|---|---|---|---|
| 1 | \n", "0.586800 | \n", "0.512138 | \n", "0.719101 | \n", "
| 2 | \n", "0.411600 | \n", "0.416675 | \n", "0.849057 | \n", "
| 3 | \n", "0.245600 | \n", "0.417070 | \n", "0.864000 | \n", "
| 4 | \n", "0.147800 | \n", "0.575878 | \n", "0.852459 | \n", "
| 5 | \n", "0.056800 | \n", "0.474259 | \n", "0.896552 | \n", "
| 6 | \n", "0.022500 | \n", "0.754236 | \n", "0.843137 | \n", "
| 7 | \n", "0.001000 | \n", "0.857636 | \n", "0.834783 | \n", "
| 8 | \n", "0.000500 | \n", "0.920232 | \n", "0.869565 | \n", "
| 9 | \n", "0.000300 | \n", "0.970790 | \n", "0.877193 | \n", "
| 10 | \n", "0.000300 | \n", "0.948689 | \n", "0.862385 | \n", "
"
],
"text/plain": [
"