diff --git a/examples/simple/data/data.ipynb b/examples/simple/data/data.ipynb
index e54811b..feef14d 100644
--- a/examples/simple/data/data.ipynb
+++ b/examples/simple/data/data.ipynb
@@ -51,7 +51,8 @@
"\n",
"manifest = (\n",
" urllib.request.urlopen(\n",
- " \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
+ " \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
+ " \"open-corpus/2022-02-01/manifest.txt\"\n",
" )\n",
" .read()\n",
" .decode()\n",
@@ -61,7 +62,9 @@
"shuffle(lines)\n",
"chunks = lines[:MAX_CHUNK_COUNT]\n",
"\n",
- "f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
+ "f\"\"\"Processing {len(chunks)} out of the {\n",
+ " len(manifest.split())\n",
+ "} available chunks\"\"\""
]
},
{
@@ -104,7 +107,8 @@
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
" # Extract\n",
" response = urllib.request.urlopen(\n",
- " f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
+ " f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
+ " \"open-corpus/2022-02-01/{chunk_key}\"\n",
" ) # a gzipped JSON Lines file\n",
"\n",
" decompressed = gzip.decompress(response.read())\n",
@@ -115,13 +119,14 @@
" return [\n",
" (\n",
" clean(\n",
- " f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
+ " f'{c[\"title\"]} {c[\"paperAbstract\"]} '\n",
+ " f'{c[\"journalName\"]} {c[\"venue\"]}',\n",
" convert_to_ascii=True,\n",
- " ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
+ " ), # The text is cleaned to remove common artifacts\n",
" c[\"fieldsOfStudy\"],\n",
" ) # Create pairs of `(text, [...domains])`\n",
" for c in chunk\n",
- " if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
+ " if (c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"])))\n",
" ]\n",
"\n",
"\n",
@@ -145,7 +150,7 @@
"source": [
"### Load\n",
"\n",
- "Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
+ "Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each data-point with a split label according to the ratios we set. Additional tags can be also given.\n",
"\n",
"#### Production-ready backend\n",
"\n",
diff --git a/examples/simple/data/index.html b/examples/simple/data/index.html
index 5a11182..293e340 100644
--- a/examples/simple/data/index.html
+++ b/examples/simple/data/index.html
@@ -408,7 +408,7 @@
@@ -1968,7 +1968,8 @@ Licensed under the Apache License, Version 2.0.
manifest=(urllib.request.urlopen(
- "https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt"
+ "https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
+ "open-corpus/2022-02-01/manifest.txt").read().decode()
@@ -1978,14 +1979,17 @@ Licensed under the Apache License, Version 2.0.
shuffle(lines)chunks=lines[:MAX_CHUNK_COUNT]
-f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"
+f"""Processing {len(chunks)} out of the {
+ len(manifest.split())
+} available chunks"""
import urllib.request
from random import shuffle
manifest = (
urllib.request.urlopen(
- "https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt"
+ "https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
+ "open-corpus/2022-02-01/manifest.txt"
)
.read()
.decode()
@@ -1995,7 +1999,9 @@ lines = manifest.split()
shuffle(lines)
chunks = lines[:MAX_CHUNK_COUNT]
-f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"
+f"""Processing {len(chunks)} out of the {
+ len(manifest.split())
+} available chunks"""
@@ -2081,7 +2087,8 @@ f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"<
defpreprocess_chunk(chunk_key:str)->List[Tuple[str,List[str]]]:# Extractresponse=urllib.request.urlopen(
- f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}"
+ f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
+ "open-corpus/2022-02-01/{chunk_key}")# a gzipped JSON Lines filedecompressed=gzip.decompress(response.read())
@@ -2092,13 +2099,14 @@ f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"<
return[(clean(
- f'{c["title"]}{c["paperAbstract"]}{c["journalName"]}{c["venue"]}',
+ f'{c["title"]}{c["paperAbstract"]} '
+ f'{c["journalName"]}{c["venue"]}',convert_to_ascii=True,
- ),# The text is cleaned to remove PDF extraction, web scraping, and other common artifacts
+ ),# The text is cleaned to remove common artifactsc["fieldsOfStudy"],)# Create pairs of `(text, [...domains])`forcinchunk
- ifc["fieldsOfStudy"]andis_english(predict_language(c["paperAbstract"]))
+ if(c["fieldsOfStudy"]andis_english(predict_language(c["paperAbstract"])))]
@@ -2121,7 +2129,8 @@ from great_ai.utilities import (
def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:
# Extract
response = urllib.request.urlopen(
- f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}"
+ f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
+ "open-corpus/2022-02-01/{chunk_key}"
) # a gzipped JSON Lines file
decompressed = gzip.decompress(response.read())
@@ -2132,13 +2141,14 @@ def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:
return [
(
clean(
- f'{c["title"]} {c["paperAbstract"]} {c["journalName"]} {c["venue"]}',
+ f'{c["title"]} {c["paperAbstract"]} '
+ f'{c["journalName"]} {c["venue"]}',
convert_to_ascii=True,
- ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts
+ ), # The text is cleaned to remove common artifacts
c["fieldsOfStudy"],
) # Create pairs of `(text, [...domains])`
for c in chunk
- if c["fieldsOfStudy"] and is_english(predict_language(c["paperAbstract"]))
+ if (c["fieldsOfStudy"] and is_english(predict_language(c["paperAbstract"])))
]
@@ -2215,7 +2225,7 @@ preprocessed_data = unchunk(
Upload the dataset (or a part of it) to a central repository using great_ai.add_ground_truth. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.
Upload the dataset (or a part of it) to a central repository using great_ai.add_ground_truth. This step automatically tags each data-point with a split label according to the ratios we set. Additional tags can be also given.
diff --git a/explanation/explanation.md b/explanation/explanation.md
index a110ace..e0966aa 100644
--- a/explanation/explanation.md
+++ b/explanation/explanation.md
@@ -1,7 +1,9 @@
# Explanation
-A lot more details and discussion about the problem context and approaches of GreatAI can be found in my thesis.
+A lot more details and discussion about the problem context and approaches of GreatAI along with its evaluation can be found in my thesis.
+
+