{ "cells": [ { "cell_type": "code", "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ "data/evaluation-experiment-2-stage #1-sa6a0y.json\n", "data/evaluation-experiment-2-stage #1-2m6dmb.json\n" ] } ], "source": [ "from pathlib import Path\n", "import json\n", "\n", "annotations = []\n", "for p in Path(\"data\").glob(\"*.json\"):\n", " with open(p, encoding=\"utf-8\") as f:\n", " print(p)\n", " annotations.append(json.load(f))\n", "\n", "evaluations = {\n", " sentence: [\n", " annotation[sentence] for annotation in annotations if sentence in annotation\n", " ]\n", " for sentence in {\n", " sentence for annotation in annotations for sentence in annotation.keys()\n", " }\n", "}\n", "\n", "X = [s for s in evaluations.keys()]\n", "y = [int(sum(e) > 0) for e in evaluations.values()]" ] }, { "cell_type": "code", "execution_count": 7, "metadata": {}, "outputs": [], "source": [ "y1 = [e[0] for e in evaluations.values() if len(e) == 2]\n", "y2 = [e[1] for e in evaluations.values() if len(e) == 2]" ] }, { "cell_type": "code", "execution_count": 8, "metadata": {}, "outputs": [ { "data": { "text/plain": [ "0.3546448712421808" ] }, "execution_count": 8, "metadata": {}, "output_type": "execute_result" } ], "source": [ "import sklearn.metrics\n", "\n", "sklearn.metrics.cohen_kappa_score(y1, y2)" ] }, { "cell_type": "code", "execution_count": 9, "metadata": {}, "outputs": [], "source": [ "X = [s for s in evaluations.keys()]\n", "y = [int(sum(e) > 0) for e in evaluations.values()]" ] }, { "cell_type": "code", "execution_count": 10, "metadata": {}, "outputs": [ { "data": { "text/html": [ "
Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n",
" ('classifier', LinearSVC())])In a Jupyter environment, please rerun this cell to show the HTML representation or trust the notebook. Pipeline(steps=[('vectorizer',\n",
" TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)),\n",
" ('classifier', LinearSVC())])TfidfVectorizer(max_df=0.3, min_df=3, sublinear_tf=True)
LinearSVC()