Deployed 8cd3449 with MkDocs version: 1.3.0
This commit is contained in:
parent
7d09422380
commit
2d907ccfaf
13 changed files with 2597 additions and 2299 deletions
|
|
@ -87,7 +87,7 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████| 4/4 [04:42<00:00, 70.62s/it] \n"
|
||||
"100%|██████████| 4/4 [04:22<00:00, 65.51s/it] \n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -103,10 +103,11 @@
|
|||
" unchunk,\n",
|
||||
")\n",
|
||||
"\n",
|
||||
"\n",
|
||||
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
|
||||
" response = urllib.request.urlopen(\n",
|
||||
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
|
||||
" \"open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||
" f\"open-corpus/2022-02-01/{chunk_key}\"\n",
|
||||
" ) # a gzipped JSON Lines file\n",
|
||||
"\n",
|
||||
" decompressed = gzip.decompress(response.read())\n",
|
||||
|
|
@ -165,9 +166,29 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\u001b[38;5;226mEnvironment variable ENVIRONMENT is not set, defaulting to development mode ‼️\u001b[0m\n",
|
||||
"\u001b[38;5;226mCannot find credentials files, defaulting to using ParallelTinyDbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;226mThe selected tracing database (ParallelTinyDbDriver) is not recommended for production\u001b[0m\n",
|
||||
"\u001b[38;5;226mCannot find credentials files, defaulting to using LargeFileLocal\u001b[0m\n",
|
||||
"\u001b[38;5;39mGreatAI (v0.1.6): configured ✅\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 tracing_database: ParallelTinyDbDriver\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 large_file_implementation: LargeFileLocal\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 is_production: False\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 should_log_exception_stack: True\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 prediction_cache_size: 512\u001b[0m\n",
|
||||
"\u001b[38;5;39m 🔩 dashboard_table_size: 50\u001b[0m\n",
|
||||
"\u001b[38;5;226mYou still need to check whether you follow all best practices before trusting your deployment.\u001b[0m\n",
|
||||
"\u001b[38;5;226m> Find out more at https://se-ml.github.io/practices\u001b[0m\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from great_ai import add_ground_truth\n",
|
||||
"\n",
|
||||
|
|
|
|||
|
|
@ -2083,10 +2083,11 @@ f"""Processing {len(chunks)} out of the {
|
|||
<span class="n">unchunk</span><span class="p">,</span>
|
||||
<span class="p">)</span>
|
||||
|
||||
|
||||
<span class="k">def</span> <span class="nf">preprocess_chunk</span><span class="p">(</span><span class="n">chunk_key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]:</span>
|
||||
<span class="n">response</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span>
|
||||
<span class="sa">f</span><span class="s2">"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"</span>
|
||||
<span class="s2">"open-corpus/2022-02-01/</span><span class="si">{chunk_key}</span><span class="s2">"</span>
|
||||
<span class="sa">f</span><span class="s2">"open-corpus/2022-02-01/</span><span class="si">{</span><span class="n">chunk_key</span><span class="si">}</span><span class="s2">"</span>
|
||||
<span class="p">)</span> <span class="c1"># a gzipped JSON Lines file</span>
|
||||
|
||||
<span class="n">decompressed</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
|
||||
|
|
@ -2123,10 +2124,11 @@ from great_ai.utilities import (
|
|||
unchunk,
|
||||
)
|
||||
|
||||
|
||||
def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:
|
||||
response = urllib.request.urlopen(
|
||||
f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
|
||||
"open-corpus/2022-02-01/{chunk_key}"
|
||||
f"open-corpus/2022-02-01/{chunk_key}"
|
||||
) # a gzipped JSON Lines file
|
||||
|
||||
decompressed = gzip.decompress(response.read())
|
||||
|
|
@ -2172,7 +2174,7 @@ preprocessed_data = unchunk(
|
|||
|
||||
|
||||
<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr">
|
||||
<pre>100%|██████████| 4/4 [04:42<00:00, 70.62s/it]
|
||||
<pre>100%|██████████| 4/4 [04:22<00:00, 65.51s/it]
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
|
@ -2233,13 +2235,13 @@ preprocessed_data = unchunk(
|
|||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs ">
|
||||
<div class="jp-Cell jp-CodeCell jp-Notebook-cell jp-mod-noOutputs ">
|
||||
</div><div class="jp-Cell jp-CodeCell jp-Notebook-cell ">
|
||||
<div class="jp-Cell jp-CodeCell jp-Notebook-cell ">
|
||||
<div class="jp-Cell-inputWrapper">
|
||||
<div class="jp-Collapser jp-InputCollapser jp-Cell-inputCollapser">
|
||||
</div>
|
||||
<div class="jp-InputArea jp-Cell-inputArea">
|
||||
<div class="jp-InputPrompt jp-InputArea-prompt">In [8]:</div><div class="jp-CodeMirrorEditor jp-Editor jp-InputArea-editor" data-type="inline">
|
||||
<div class="jp-InputPrompt jp-InputArea-prompt">In [5]:</div><div class="jp-CodeMirrorEditor jp-Editor jp-InputArea-editor" data-type="inline">
|
||||
|
||||
|
||||
<div class="CodeMirror cm-s-jupyter">
|
||||
|
|
@ -2268,6 +2270,41 @@ add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)</div>
|
|||
</div>
|
||||
</div>
|
||||
|
||||
<div class="jp-Cell-outputWrapper">
|
||||
<div class="jp-Collapser jp-OutputCollapser jp-Cell-outputCollapser">
|
||||
</div>
|
||||
|
||||
|
||||
<div class="jp-OutputArea jp-Cell-outputArea">
|
||||
|
||||
<div class="jp-OutputArea-child">
|
||||
|
||||
|
||||
<div class="jp-OutputPrompt jp-OutputArea-prompt"></div>
|
||||
|
||||
|
||||
<div class="jp-RenderedText jp-OutputArea-output" data-mime-type="application/vnd.jupyter.stderr">
|
||||
<pre><span style="color: rgb(255,255,0)">Environment variable ENVIRONMENT is not set, defaulting to development mode ‼️</span>
|
||||
<span style="color: rgb(255,255,0)">Cannot find credentials files, defaulting to using ParallelTinyDbDriver</span>
|
||||
<span style="color: rgb(255,255,0)">The selected tracing database (ParallelTinyDbDriver) is not recommended for production</span>
|
||||
<span style="color: rgb(255,255,0)">Cannot find credentials files, defaulting to using LargeFileLocal</span>
|
||||
<span style="color: rgb(0,175,255)">GreatAI (v0.1.6): configured ✅</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 tracing_database: ParallelTinyDbDriver</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 large_file_implementation: LargeFileLocal</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 is_production: False</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 should_log_exception_stack: True</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 prediction_cache_size: 512</span>
|
||||
<span style="color: rgb(0,175,255)"> 🔩 dashboard_table_size: 50</span>
|
||||
<span style="color: rgb(255,255,0)">You still need to check whether you follow all best practices before trusting your deployment.</span>
|
||||
<span style="color: rgb(255,255,0)">> Find out more at https://se-ml.github.io/practices</span>
|
||||
</pre>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</div>
|
||||
<div class="jp-Cell jp-MarkdownCell jp-Notebook-cell">
|
||||
|
|
@ -2290,7 +2327,7 @@ add_ground_truth(X, y, train_split_ratio=0.8, test_split_ratio=0.2)</div>
|
|||
<small>
|
||||
|
||||
Last update:
|
||||
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 15, 2022</span>
|
||||
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 16, 2022</span>
|
||||
|
||||
|
||||
</small>
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue