andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Deployed 0585a4e with MkDocs version: 1.3.0

Browse source
This commit is contained in:
2022-07-12 19:53:07 +00:00
parent 7305d2ead3
commit 8cda980b81
33 changed files with 782 additions and 464 deletions
Download patch file Download diff file Expand all files Collapse all files

10
404.html
View file

@ -403,7 +403,7 @@
<li class="md-nav__item">
<a href="/how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -417,7 +417,7 @@
<li class="md-nav__item">
<a href="/how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -431,7 +431,7 @@
<li class="md-nav__item">
<a href="/how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -444,8 +444,8 @@
<li class="md-nav__item">
<a href="/how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="/how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

10
examples/simple-mag/train/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

19
examples/simple/data/data.ipynb
View file

@ -51,7 +51,8 @@
"\n",
"manifest = (\n",
" urllib.request.urlopen(\n",
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt\"\n",
" \"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
" \"open-corpus/2022-02-01/manifest.txt\"\n",
" )\n",
" .read()\n",
" .decode()\n",
@ -61,7 +62,9 @@
"shuffle(lines)\n",
"chunks = lines[:MAX_CHUNK_COUNT]\n",
"\n",
"f\"Processing {len(chunks)} out of the {len(manifest.split())} available chunks\""
"f\"\"\"Processing {len(chunks)} out of the {\n",
" len(manifest.split())\n",
"} available chunks\"\"\""
]
},
{
@ -104,7 +107,8 @@
"def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:\n",
" # Extract\n",
" response = urllib.request.urlopen(\n",
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}\"\n",
" f\"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/\"\n",
" \"open-corpus/2022-02-01/{chunk_key}\"\n",
" ) # a gzipped JSON Lines file\n",
"\n",
" decompressed = gzip.decompress(response.read())\n",
@ -115,13 +119,14 @@
" return [\n",
" (\n",
" clean(\n",
" f'{c[\"title\"]} {c[\"paperAbstract\"]} {c[\"journalName\"]} {c[\"venue\"]}',\n",
" f'{c[\"title\"]} {c[\"paperAbstract\"]} '\n",
" f'{c[\"journalName\"]} {c[\"venue\"]}',\n",
" convert_to_ascii=True,\n",
" ), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts\n",
" ), # The text is cleaned to remove common artifacts\n",
" c[\"fieldsOfStudy\"],\n",
" ) # Create pairs of `(text, [...domains])`\n",
" for c in chunk\n",
" if c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"]))\n",
" if (c[\"fieldsOfStudy\"] and is_english(predict_language(c[\"paperAbstract\"])))\n",
" ]\n",
"\n",
"\n",
@ -145,7 +150,7 @@
"source": [
"### Load\n",
"\n",
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.\n",
"Upload the dataset (or a part of it) to a central repository using `great_ai.add_ground_truth`. This step automatically tags each data-point with a split label according to the ratios we set. Additional tags can be also given.\n",
"\n",
"#### Production-ready backend\n",
"\n",

46
examples/simple/data/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -1968,7 +1968,8 @@ Licensed under the Apache License, Version 2.0.
<span class="n">manifest</span> <span class="o">=</span> <span class="p">(</span>
<span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span>
<span class="s2">&quot;https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt&quot;</span>
<span class="s2">&quot;https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/&quot;</span>
<span class="s2">&quot;open-corpus/2022-02-01/manifest.txt&quot;</span>
<span class="p">)</span>
<span class="o">.</span><span class="n">read</span><span class="p">()</span>
<span class="o">.</span><span class="n">decode</span><span class="p">()</span>
@ -1978,14 +1979,17 @@ Licensed under the Apache License, Version 2.0.
<span class="n">shuffle</span><span class="p">(</span><span class="n">lines</span><span class="p">)</span>
<span class="n">chunks</span> <span class="o">=</span> <span class="n">lines</span><span class="p">[:</span><span class="n">MAX_CHUNK_COUNT</span><span class="p">]</span>
<span class="sa">f</span><span class="s2">&quot;Processing </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">chunks</span><span class="p">)</span><span class="si">}</span><span class="s2"> out of the </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">manifest</span><span class="o">.</span><span class="n">split</span><span class="p">())</span><span class="si">}</span><span class="s2"> available chunks&quot;</span>
<span class="sa">f</span><span class="s2">&quot;&quot;&quot;Processing </span><span class="si">{</span><span class="nb">len</span><span class="p">(</span><span class="n">chunks</span><span class="p">)</span><span class="si">}</span><span class="s2"> out of the </span><span class="si">{</span>
<span class="nb">len</span><span class="p">(</span><span class="n">manifest</span><span class="o">.</span><span class="n">split</span><span class="p">())</span>
<span class="si">}</span><span class="s2"> available chunks&quot;&quot;&quot;</span>
</pre></div>
<div id="cell-2" class="clipboard-copy-txt">import urllib.request
from random import shuffle
manifest = (
urllib.request.urlopen(
"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/manifest.txt"
"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
"open-corpus/2022-02-01/manifest.txt"
)
.read()
.decode()
@ -1995,7 +1999,9 @@ lines = manifest.split()
shuffle(lines)
chunks = lines[:MAX_CHUNK_COUNT]
f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"</div>
f"""Processing {len(chunks)} out of the {
len(manifest.split())
} available chunks"""</div>
</div>
</div>
@ -2081,7 +2087,8 @@ f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"<
<span class="k">def</span> <span class="nf">preprocess_chunk</span><span class="p">(</span><span class="n">chunk_key</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Tuple</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]]]:</span>
<span class="c1"># Extract</span>
<span class="n">response</span> <span class="o">=</span> <span class="n">urllib</span><span class="o">.</span><span class="n">request</span><span class="o">.</span><span class="n">urlopen</span><span class="p">(</span>
<span class="sa">f</span><span class="s2">&quot;https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/</span><span class="si">{</span><span class="n">chunk_key</span><span class="si">}</span><span class="s2">&quot;</span>
<span class="sa">f</span><span class="s2">&quot;https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/&quot;</span>
<span class="s2">&quot;open-corpus/2022-02-01/</span><span class="si">{chunk_key}</span><span class="s2">&quot;</span>
<span class="p">)</span> <span class="c1"># a gzipped JSON Lines file</span>
<span class="n">decompressed</span> <span class="o">=</span> <span class="n">gzip</span><span class="o">.</span><span class="n">decompress</span><span class="p">(</span><span class="n">response</span><span class="o">.</span><span class="n">read</span><span class="p">())</span>
@ -2092,13 +2099,14 @@ f"Processing {len(chunks)} out of the {len(manifest.split())} available chunks"<
<span class="k">return</span> <span class="p">[</span>
<span class="p">(</span>
<span class="n">clean</span><span class="p">(</span>
<span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;paperAbstract&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;journalName&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;venue&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;title&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;paperAbstract&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> &#39;</span>
<span class="sa">f</span><span class="s1">&#39;</span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;journalName&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1"> </span><span class="si">{</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;venue&quot;</span><span class="p">]</span><span class="si">}</span><span class="s1">&#39;</span><span class="p">,</span>
<span class="n">convert_to_ascii</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<span class="p">),</span> <span class="c1"># The text is cleaned to remove PDF extraction, web scraping, and other common artifacts</span>
<span class="p">),</span> <span class="c1"># The text is cleaned to remove common artifacts</span>
<span class="n">c</span><span class="p">[</span><span class="s2">&quot;fieldsOfStudy&quot;</span><span class="p">],</span>
<span class="p">)</span> <span class="c1"># Create pairs of `(text, [...domains])`</span>
<span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">chunk</span>
<span class="k">if</span> <span class="n">c</span><span class="p">[</span><span class="s2">&quot;fieldsOfStudy&quot;</span><span class="p">]</span> <span class="ow">and</span> <span class="n">is_english</span><span class="p">(</span><span class="n">predict_language</span><span class="p">(</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;paperAbstract&quot;</span><span class="p">]))</span>
<span class="k">if</span> <span class="p">(</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;fieldsOfStudy&quot;</span><span class="p">]</span> <span class="ow">and</span> <span class="n">is_english</span><span class="p">(</span><span class="n">predict_language</span><span class="p">(</span><span class="n">c</span><span class="p">[</span><span class="s2">&quot;paperAbstract&quot;</span><span class="p">])))</span>
<span class="p">]</span>
@ -2121,7 +2129,8 @@ from great_ai.utilities import (
def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:
# Extract
response = urllib.request.urlopen(
f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/open-corpus/2022-02-01/{chunk_key}"
f"https://s3-us-west-2.amazonaws.com/ai2-s2-research-public/"
"open-corpus/2022-02-01/{chunk_key}"
) # a gzipped JSON Lines file
decompressed = gzip.decompress(response.read())
@ -2132,13 +2141,14 @@ def preprocess_chunk(chunk_key: str) -> List[Tuple[str, List[str]]]:
return [
(
clean(
f'{c["title"]} {c["paperAbstract"]} {c["journalName"]} {c["venue"]}',
f'{c["title"]} {c["paperAbstract"]} '
f'{c["journalName"]} {c["venue"]}',
convert_to_ascii=True,
), # The text is cleaned to remove PDF extraction, web scraping, and other common artifacts
), # The text is cleaned to remove common artifacts
c["fieldsOfStudy"],
) # Create pairs of `(text, [...domains])`
for c in chunk
if c["fieldsOfStudy"] and is_english(predict_language(c["paperAbstract"]))
if (c["fieldsOfStudy"] and is_english(predict_language(c["paperAbstract"])))
]
@ -2215,7 +2225,7 @@ preprocessed_data = unchunk(
</div>
<div class="jp-InputArea jp-Cell-inputArea"><div class="jp-InputPrompt jp-InputArea-prompt">
</div><div class="jp-RenderedHTMLCommon jp-RenderedMarkdown jp-MarkdownOutput " data-mime-type="text/markdown">
<h3 id="load">Load<a class="anchor-link" href="#load">&#182;</a></h3><p>Upload the dataset (or a part of it) to a central repository using <code>great_ai.add_ground_truth</code>. This step automatically tags each datapoint with a split label according to the ratios we set. Additional tags can be also given.</p>
<h3 id="load">Load<a class="anchor-link" href="#load">&#182;</a></h3><p>Upload the dataset (or a part of it) to a central repository using <code>great_ai.add_ground_truth</code>. This step automatically tags each data-point with a split label according to the ratios we set. Additional tags can be also given.</p>
<h4 id="production-ready-backend">Production-ready backend<a class="anchor-link" href="#production-ready-backend">&#182;</a></h4><p>The MongoDB driver is automatically configured if <code>mongo.ini</code> exists with the following scheme:</p>
<div class="highlight"><pre><span></span><span class="na">mongo_connection_string</span><span class="o">=</span><span class="s">mongodb://localhost:27017/</span><span class="w"></span>
<span class="na">mongo_database</span><span class="o">=</span><span class="s">my_great_ai_db</span><span class="w"></span>

1
examples/simple/deploy/deploy.ipynb
View file

@ -202,7 +202,6 @@
"\n",
" # Configure matplotlib to have nice, high-resolution charts\n",
" %matplotlib inline\n",
"\n",
" plt.rcParams[\"figure.figsize\"] = (30, 15)\n",
" plt.rcParams[\"figure.facecolor\"] = \"white\"\n",
" plt.rcParams[\"font.size\"] = 12\n",

12
examples/simple/deploy/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -2114,7 +2114,6 @@ def get_label(
<span class="c1"># Configure matplotlib to have nice, high-resolution charts</span>
<span class="o">%</span><span class="n">matplotlib</span> <span class="n">inline</span>
<span class="n">plt</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;figure.figsize&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span><span class="mi">30</span><span class="p">,</span> <span class="mi">15</span><span class="p">)</span>
<span class="n">plt</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;figure.facecolor&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="s2">&quot;white&quot;</span>
<span class="n">plt</span><span class="o">.</span><span class="n">rcParams</span><span class="p">[</span><span class="s2">&quot;font.size&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="mi">12</span>
@ -2148,7 +2147,6 @@ def get_label(
# Configure matplotlib to have nice, high-resolution charts
%matplotlib inline
plt.rcParams["figure.figsize"] = (30, 15)
plt.rcParams["figure.facecolor"] = "white"
plt.rcParams["font.size"] = 12

10
examples/simple/train/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

6
explanation/explanation.md
View file

@ -1,7 +1,9 @@
# Explanation
A lot more details and discussion about the problem context and approaches of GreatAI can be found in my thesis.
A lot more details and discussion about the problem context and approaches of GreatAI along with its evaluation can be found in my thesis.
![front page](/media/thesis-frontpage.png)
<div style="display: flex; justify-content: space-evenly;" markdown>
[::fontawesome-solid-graduation-cap: Download](thesis/main.pdf){ .md-button .md-button--primary }
[::fontawesome-solid-graduation-cap: Download (available soon)](thesis/main.pdf){ .md-button .md-button--primary }
</div>

17
explanation/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -742,9 +742,10 @@
<h1 id="explanation">Explanation<a class="headerlink" href="#explanation" title="Permanent link">#</a></h1>
<p>A lot more details and discussion about the problem context and approaches of GreatAI can be found in my thesis.</p>
<p>A lot more details and discussion about the problem context and approaches of GreatAI along with its evaluation can be found in my thesis.</p>
<p><img alt="front page" src="/media/thesis-frontpage.png" /></p>
<div style="display: flex; justify-content: space-evenly;">
<p><a class="md-button md-button--primary" href="thesis/main.pdf">:<span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M623.1 136.9 340.4 35.7a62.951 62.951 0 0 0-42.43 0L16.05 136.9C6.438 140.4 0 149.6 0 160s6.438 19.65 16.05 23.09l60.02 21.51c-11.89 15.8-20.26 34.16-24.55 53.95C40.05 263.4 32 274.8 32 288c0 9.953 4.814 18.49 11.94 24.36l-24.83 149C17.48 471.1 25 480 34.89 480h58.22c9.887 0 17.41-8.879 15.78-18.63l-24.83-149C91.19 306.5 96 297.1 96 288c0-10.29-5.174-19.03-12.72-24.89a106.301 106.301 0 0 1 24.94-47.03l190.6 68.23a62.951 62.951 0 0 0 42.43 0l282.7-101.2C633.6 179.6 640 170.4 640 160s-6.4-19.6-16.9-23.1zm-272 177.5c-9.4 3.7-20.2 5.6-31.1 5.6a94.767 94.767 0 0 1-32-5.555L142.8 262.5 128 405.3c0 41.3 85.1 74.7 192 74.7 105.1 0 192-33.4 192-74.67l-14.78-142.9L351.1 314.4z"/></svg></span> Download</a></p>
<p><a class="md-button md-button--primary" href="thesis/main.pdf">:<span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 640 512"><!--! Font Awesome Free 6.1.1 by @fontawesome - https://fontawesome.com License - https://fontawesome.com/license/free (Icons: CC BY 4.0, Fonts: SIL OFL 1.1, Code: MIT License) Copyright 2022 Fonticons, Inc.--><path d="M623.1 136.9 340.4 35.7a62.951 62.951 0 0 0-42.43 0L16.05 136.9C6.438 140.4 0 149.6 0 160s6.438 19.65 16.05 23.09l60.02 21.51c-11.89 15.8-20.26 34.16-24.55 53.95C40.05 263.4 32 274.8 32 288c0 9.953 4.814 18.49 11.94 24.36l-24.83 149C17.48 471.1 25 480 34.89 480h58.22c9.887 0 17.41-8.879 15.78-18.63l-24.83-149C91.19 306.5 96 297.1 96 288c0-10.29-5.174-19.03-12.72-24.89a106.301 106.301 0 0 1 24.94-47.03l190.6 68.23a62.951 62.951 0 0 0 42.43 0l282.7-101.2C633.6 179.6 640 170.4 640 160s-6.4-19.6-16.9-23.1zm-272 177.5c-9.4 3.7-20.2 5.6-31.1 5.6a94.767 94.767 0 0 1-32-5.555L142.8 262.5 128 405.3c0 41.3 85.1 74.7 192 74.7 105.1 0 192-33.4 192-74.67l-14.78-142.9L351.1 314.4z"/></svg></span> Download (available soon)</a></p>
</div>
<hr>
@ -752,7 +753,7 @@
<small>
Last update:
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 11, 2022</span>
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 12, 2022</span>
</small>

64
how-to-guides/call-remote/index.html
View file

@ -339,10 +339,12 @@
<li class="md-nav__item md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" >
<li class="md-nav__item md-nav__item--active md-nav__item--nested">
<input class="md-nav__toggle md-toggle" data-md-toggle="__nav_3" type="checkbox" id="__nav_3" checked>
@ -408,7 +410,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +424,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +438,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -448,10 +450,20 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<li class="md-nav__item md-nav__item--active">
<input class="md-nav__toggle md-toggle" data-md-toggle="toc" type="checkbox" id="__toc">
<a href="./" class="md-nav__link md-nav__link--active">
Call remote GreatAI instances
</a>
</li>
@ -755,6 +767,42 @@
<footer class="md-footer">
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../large_file/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to use LargeFile-s" rel="prev">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</div>
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Previous
</span>
How to use LargeFile-s
</div>
</div>
</a>
<a href="../scraping/" class="md-footer__link md-footer__link--next" aria-label="Next: Scraping" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
Scraping
</div>
</div>
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M4 11v2h12l-5.5 5.5 1.42 1.42L19.84 12l-7.92-7.92L10.5 5.5 16 11H4Z"/></svg>
</div>
</a>
</nav>
<div class="md-footer-meta md-typeset">
<div class="md-footer-meta__inner md-grid">
<div class="md-copyright">

14
how-to-guides/configure-service/index.html
View file

@ -498,7 +498,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -512,7 +512,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -526,7 +526,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -539,8 +539,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -1009,13 +1009,13 @@
<a href="../use-service/" class="md-footer__link md-footer__link--next" aria-label="Next: How to use a GreatAI service" rel="next">
<a href="../use-service/" class="md-footer__link md-footer__link--next" aria-label="Next: How to deploy a GreatAI service" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
How to use a GreatAI service
How to deploy a GreatAI service
</div>
</div>
<div class="md-footer__button md-icon">

10
how-to-guides/create-service/index.html
View file

@ -492,7 +492,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -506,7 +506,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -520,7 +520,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -533,8 +533,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

67
how-to-guides/handle-training-data/handle-training-data.md
View file

@ -1,6 +1,69 @@
# How to handle training data
# How to manage training data
In order to simplify your training data management, `great-ai` provide two complementing approaches for inputting new data-points.
## Upload data
## Use feedback
At the start of your experiments' first iteration, after you've gathered suitable samples for training, you can call [great_ai.add_ground_truth][]. This automatically stores a timestamp and also allows you to assign tags to the data. Using these attributes, [great_ai.query_ground_truth][] can be called to get a filtered view of the training data.
!!! important "Train-test-validation splits"
It is a best-practice to lock-away a test split of your data that is only used for the final quality assessment. This prevents you from accidentally training on it, or inadvertently tuning the model to have the highest accuracy metrics on the test split. This, of course, may lead to dubious results, hence, care must be taken to avoid it.
With [great_ai.add_ground_truth][], there is an option to tag the samples with `train`, `test`, and `validation` randomly, following a predefined distribution. This happens as soon as they're written in the database. Later on, these can be queried by providing the name of the appropriate tags.
The nice thing about this is that the 'input-expected output' pairs are stored as traces. Thus, they behave exactly like regular prediction traces.
```python
from great_ai import add_ground_truth
add_ground_truth(
[1, 2],
['odd', 'even'],
tags='my_tag',
train_split_ratio=1, #(1)
test_split_ratio=1
)
```
1. Note that the ratios don't have to add up to 1. They are just weights. There is also a `validation_split_ratio` which is 0 by default.
```python
>>> from great_ai import query_ground_truth
>>> query_ground_truth('my_tag')
[Trace[str]({'created': '2022-07-12T18:36:12.825706',
'exception': None,
'feedback': 'odd', #(1)
'logged_values': {'input': 1}, #(2)
'models': [],
'original_execution_time_ms': 0.0,
'output': 'odd',
'tags': ['ground_truth', 'test', 'my_tag'], #(3)
'trace_id': '4fcf2ce6-a172-469d-94b2-874577655814'}),
Trace[str]({'created': '2022-07-12T18:36:12.825706',
'exception': None,
'feedback': 'even',
'logged_values': {'input': 2},
'models': [],
'original_execution_time_ms': 0.0,
'output': 'even',
'tags': ['ground_truth', 'train', 'my_tag'],
'trace_id': 'abee0671-beb9-4284-8c3b-c65e5836ce38'})]
```
1. Expected output. This can be also accessed through the `.output` property.
2. The input value is stored here.
3. Notice how `ground_truth` always included as a tag when using [great_ai.add_ground_truth][].
## Get feedback
After the initial data gathering, end-to-end feedback can be also integrated into the dataset.
The scaffolded REST API contains endpoints for managing traces and their feedbacks.
![screenshot of swagger](/media/feedback.png){ loading=lazy }
When [great_ai.query_ground_truth][] is executed, it implicitly filters for traces that have feedback. Therefore, both the `ground_truth` and the `online` traces that have received feedback are returned. No matter the origin of the data, it can be accessed using the same API.
## Remove clutter
Traces can be deleted either through the REST API or by calling [great_ai.delete_ground_truth][]. The latter provides the same interface as [great_ai.query_ground_truth][] except it deletes the matched points.

107
how-to-guides/handle-training-data/index.html
View file

@ -17,7 +17,7 @@
<title>How to handle training data - GreatAI documentation</title>
<title>How to manage training data - GreatAI documentation</title>
@ -83,7 +83,7 @@
<div data-md-component="skip">
<a href="#how-to-handle-training-data" class="md-skip">
<a href="#how-to-manage-training-data" class="md-skip">
Skip to content
</a>
@ -116,7 +116,7 @@
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
How to handle training data
How to manage training data
</span>
</div>
@ -410,7 +410,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -433,12 +433,12 @@
<label class="md-nav__link md-nav__link--active" for="__toc">
How to handle training data
How to manage training data
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
How to handle training data
How to manage training data
</a>
@ -464,8 +464,15 @@
</li>
<li class="md-nav__item">
<a href="#use-feedback" class="md-nav__link">
Use feedback
<a href="#get-feedback" class="md-nav__link">
Get feedback
</a>
</li>
<li class="md-nav__item">
<a href="#remove-clutter" class="md-nav__link">
Remove clutter
</a>
</li>
@ -486,7 +493,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -499,8 +506,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -775,8 +782,15 @@
</li>
<li class="md-nav__item">
<a href="#use-feedback" class="md-nav__link">
Use feedback
<a href="#get-feedback" class="md-nav__link">
Get feedback
</a>
</li>
<li class="md-nav__item">
<a href="#remove-clutter" class="md-nav__link">
Remove clutter
</a>
</li>
@ -803,16 +817,69 @@
<h1 id="how-to-handle-training-data">How to handle training data<a class="headerlink" href="#how-to-handle-training-data" title="Permanent link">#</a></h1>
<h1 id="how-to-manage-training-data">How to manage training data<a class="headerlink" href="#how-to-manage-training-data" title="Permanent link">#</a></h1>
<p>In order to simplify your training data management, <code>great-ai</code> provide two complementing approaches for inputting new data-points.</p>
<h2 id="upload-data">Upload data<a class="headerlink" href="#upload-data" title="Permanent link">#</a></h2>
<h2 id="use-feedback">Use feedback<a class="headerlink" href="#use-feedback" title="Permanent link">#</a></h2>
<p>At the start of your experiments' first iteration, after you've gathered suitable samples for training, you can call <a class="autorefs autorefs-internal" href="../../reference/#great_ai.add_ground_truth">great_ai.add_ground_truth</a>. This automatically stores a timestamp and also allows you to assign tags to the data. Using these attributes, <a class="autorefs autorefs-internal" href="../../reference/#great_ai.query_ground_truth">great_ai.query_ground_truth</a> can be called to get a filtered view of the training data.</p>
<div class="admonition important">
<p class="admonition-title">Train-test-validation splits</p>
<p>It is a best-practice to lock-away a test split of your data that is only used for the final quality assessment. This prevents you from accidentally training on it, or inadvertently tuning the model to have the highest accuracy metrics on the test split. This, of course, may lead to dubious results, hence, care must be taken to avoid it.</p>
<p>With <a class="autorefs autorefs-internal" href="../../reference/#great_ai.add_ground_truth">great_ai.add_ground_truth</a>, there is an option to tag the samples with <code>train</code>, <code>test</code>, and <code>validation</code> randomly, following a predefined distribution. This happens as soon as they're written in the database. Later on, these can be queried by providing the name of the appropriate tags.</p>
</div>
<p>The nice thing about this is that the 'input-expected output' pairs are stored as traces. Thus, they behave exactly like regular prediction traces.</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">from</span> <span class="nn">great_ai</span> <span class="kn">import</span> <span class="n">add_ground_truth</span>
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="n">add_ground_truth</span><span class="p">(</span>
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span>
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="p">[</span><span class="s1">&#39;odd&#39;</span><span class="p">,</span> <span class="s1">&#39;even&#39;</span><span class="p">],</span>
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> <span class="n">tags</span><span class="o">=</span><span class="s1">&#39;my_tag&#39;</span><span class="p">,</span>
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a> <span class="n">train_split_ratio</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span> <span class="c1">#(1)</span>
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a> <span class="n">test_split_ratio</span><span class="o">=</span><span class="mi">1</span>
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="p">)</span>
</code></pre></div>
<ol>
<li>Note that the ratios don't have to add up to 1. They are just weights. There is also a <code>validation_split_ratio</code> which is 0 by default.</li>
</ol>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="o">&gt;&gt;&gt;</span> <span class="kn">from</span> <span class="nn">great_ai</span> <span class="kn">import</span> <span class="n">query_ground_truth</span>
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="o">&gt;&gt;&gt;</span> <span class="n">query_ground_truth</span><span class="p">(</span><span class="s1">&#39;my_tag&#39;</span><span class="p">)</span>
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="p">[</span><span class="n">Trace</span><span class="p">[</span><span class="nb">str</span><span class="p">]({</span><span class="s1">&#39;created&#39;</span><span class="p">:</span> <span class="s1">&#39;2022-07-12T18:36:12.825706&#39;</span><span class="p">,</span>
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a> <span class="s1">&#39;exception&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a> <span class="s1">&#39;feedback&#39;</span><span class="p">:</span> <span class="s1">&#39;odd&#39;</span><span class="p">,</span> <span class="c1">#(1)</span>
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a> <span class="s1">&#39;logged_values&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;input&#39;</span><span class="p">:</span> <span class="mi">1</span><span class="p">},</span> <span class="c1">#(2)</span>
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a> <span class="s1">&#39;models&#39;</span><span class="p">:</span> <span class="p">[],</span>
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a> <span class="s1">&#39;original_execution_time_ms&#39;</span><span class="p">:</span> <span class="mf">0.0</span><span class="p">,</span>
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a> <span class="s1">&#39;output&#39;</span><span class="p">:</span> <span class="s1">&#39;odd&#39;</span><span class="p">,</span>
<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a> <span class="s1">&#39;tags&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;ground_truth&#39;</span><span class="p">,</span> <span class="s1">&#39;test&#39;</span><span class="p">,</span> <span class="s1">&#39;my_tag&#39;</span><span class="p">],</span> <span class="c1">#(3) </span>
<a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a> <span class="s1">&#39;trace_id&#39;</span><span class="p">:</span> <span class="s1">&#39;4fcf2ce6-a172-469d-94b2-874577655814&#39;</span><span class="p">}),</span>
<a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a> <span class="n">Trace</span><span class="p">[</span><span class="nb">str</span><span class="p">]({</span><span class="s1">&#39;created&#39;</span><span class="p">:</span> <span class="s1">&#39;2022-07-12T18:36:12.825706&#39;</span><span class="p">,</span>
<a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a> <span class="s1">&#39;exception&#39;</span><span class="p">:</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a> <span class="s1">&#39;feedback&#39;</span><span class="p">:</span> <span class="s1">&#39;even&#39;</span><span class="p">,</span>
<a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a> <span class="s1">&#39;logged_values&#39;</span><span class="p">:</span> <span class="p">{</span><span class="s1">&#39;input&#39;</span><span class="p">:</span> <span class="mi">2</span><span class="p">},</span>
<a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a> <span class="s1">&#39;models&#39;</span><span class="p">:</span> <span class="p">[],</span>
<a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a> <span class="s1">&#39;original_execution_time_ms&#39;</span><span class="p">:</span> <span class="mf">0.0</span><span class="p">,</span>
<a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a> <span class="s1">&#39;output&#39;</span><span class="p">:</span> <span class="s1">&#39;even&#39;</span><span class="p">,</span>
<a id="__codelineno-1-19" name="__codelineno-1-19" href="#__codelineno-1-19"></a> <span class="s1">&#39;tags&#39;</span><span class="p">:</span> <span class="p">[</span><span class="s1">&#39;ground_truth&#39;</span><span class="p">,</span> <span class="s1">&#39;train&#39;</span><span class="p">,</span> <span class="s1">&#39;my_tag&#39;</span><span class="p">],</span>
<a id="__codelineno-1-20" name="__codelineno-1-20" href="#__codelineno-1-20"></a> <span class="s1">&#39;trace_id&#39;</span><span class="p">:</span> <span class="s1">&#39;abee0671-beb9-4284-8c3b-c65e5836ce38&#39;</span><span class="p">})]</span>
</code></pre></div>
<ol>
<li>Expected output. This can be also accessed through the <code>.output</code> property.</li>
<li>The input value is stored here.</li>
<li>Notice how <code>ground_truth</code> always included as a tag when using <a class="autorefs autorefs-internal" href="../../reference/#great_ai.add_ground_truth">great_ai.add_ground_truth</a>. </li>
</ol>
<h2 id="get-feedback">Get feedback<a class="headerlink" href="#get-feedback" title="Permanent link">#</a></h2>
<p>After the initial data gathering, end-to-end feedback can be also integrated into the dataset. </p>
<p>The scaffolded REST API contains endpoints for managing traces and their feedbacks.</p>
<p><img alt="screenshot of swagger" loading="lazy" src="/media/feedback.png" /></p>
<p>When <a class="autorefs autorefs-internal" href="../../reference/#great_ai.query_ground_truth">great_ai.query_ground_truth</a> is executed, it implicitly filters for traces that have feedback. Therefore, both the <code>ground_truth</code> and the <code>online</code> traces that have received feedback are returned. No matter the origin of the data, it can be accessed using the same API.</p>
<h2 id="remove-clutter">Remove clutter<a class="headerlink" href="#remove-clutter" title="Permanent link">#</a></h2>
<p>Traces can be deleted either through the REST API or by calling <a class="autorefs autorefs-internal" href="../../reference/#great_ai.delete_ground_truth">great_ai.delete_ground_truth</a>. The latter provides the same interface as <a class="autorefs autorefs-internal" href="../../reference/#great_ai.query_ground_truth">great_ai.query_ground_truth</a> except it deletes the matched points.</p>
<hr>
<div class="md-source-file">
<small>
Last update:
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 11, 2022</span>
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 12, 2022</span>
</small>
@ -833,7 +900,7 @@
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../use-service/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to use a GreatAI service" rel="prev">
<a href="../use-service/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to deploy a GreatAI service" rel="prev">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</div>
@ -842,20 +909,20 @@
<span class="md-footer__direction">
Previous
</span>
How to use a GreatAI service
How to deploy a GreatAI service
</div>
</div>
</a>
<a href="../large_file/" class="md-footer__link md-footer__link--next" aria-label="Next: How to use LargeFile" rel="next">
<a href="../large_file/" class="md-footer__link md-footer__link--next" aria-label="Next: How to use LargeFile-s" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
How to use LargeFile
How to use LargeFile-s
</div>
</div>
<div class="md-footer__button md-icon">

10
how-to-guides/install/index.html
View file

@ -451,7 +451,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -465,7 +465,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -479,7 +479,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -492,8 +492,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

206
how-to-guides/large_file/index.html
View file

@ -17,7 +17,7 @@
<title>How to use LargeFile - GreatAI documentation</title>
<title>How to use LargeFile-s - GreatAI documentation</title>
@ -83,7 +83,7 @@
<div data-md-component="skip">
<a href="#how-to-use-largefile" class="md-skip">
<a href="#how-to-use-largefile-s" class="md-skip">
Skip to content
</a>
@ -116,7 +116,7 @@
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
How to use LargeFile
How to use LargeFile-s
</span>
</div>
@ -410,7 +410,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -424,7 +424,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -447,12 +447,12 @@
<label class="md-nav__link md-nav__link--active" for="__toc">
How to use LargeFile
How to use LargeFile-s
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
How to use LargeFile
How to use LargeFile-s
</a>
@ -477,20 +477,13 @@
</li>
<li class="md-nav__item">
<a href="#solution" class="md-nav__link">
Solution
</a>
<nav class="md-nav" aria-label="Solution">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#simple-example" class="md-nav__link">
Simple example
</a>
</li>
<nav class="md-nav" aria-label="Simple example">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#more-details" class="md-nav__link">
@ -510,14 +503,19 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#command-line-example" class="md-nav__link">
Command-line example
<a href="#from-the-command-line" class="md-nav__link">
From the command-line
</a>
<nav class="md-nav" aria-label="Command-line example">
<nav class="md-nav" aria-label="From the command-line">
<ul class="md-nav__list">
<li class="md-nav__item">
@ -525,13 +523,6 @@
Setup
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-expected-options" class="md-nav__link">
Print the expected options
</a>
</li>
<li class="md-nav__item">
@ -558,11 +549,6 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
@ -580,8 +566,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -855,20 +841,13 @@
</li>
<li class="md-nav__item">
<a href="#solution" class="md-nav__link">
Solution
</a>
<nav class="md-nav" aria-label="Solution">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#simple-example" class="md-nav__link">
Simple example
</a>
</li>
<nav class="md-nav" aria-label="Simple example">
<ul class="md-nav__list">
<li class="md-nav__item">
<a href="#more-details" class="md-nav__link">
@ -888,14 +867,19 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
<li class="md-nav__item">
<a href="#command-line-example" class="md-nav__link">
Command-line example
<a href="#from-the-command-line" class="md-nav__link">
From the command-line
</a>
<nav class="md-nav" aria-label="Command-line example">
<nav class="md-nav" aria-label="From the command-line">
<ul class="md-nav__list">
<li class="md-nav__item">
@ -903,13 +887,6 @@
Setup
</a>
</li>
<li class="md-nav__item">
<a href="#print-the-expected-options" class="md-nav__link">
Print the expected options
</a>
</li>
<li class="md-nav__item">
@ -936,11 +913,6 @@
</ul>
</nav>
</li>
</ul>
</nav>
</li>
</ul>
@ -965,88 +937,88 @@
<h1 id="how-to-use-largefile">How to use LargeFile<a class="headerlink" href="#how-to-use-largefile" title="Permanent link">#</a></h1>
<p>Storing, versioning, and downloading files from S3 made as easy as using <code>open()</code> in Python. Caching included.</p>
<h1 id="how-to-use-largefile-s">How to use LargeFile-s<a class="headerlink" href="#how-to-use-largefile-s" title="Permanent link">#</a></h1>
<p>The functions <a class="autorefs autorefs-internal" href="../../reference/#great_ai.use_model">save_model</a> and <a class="autorefs autorefs-internal" href="../../reference/#great_ai.use_model">@use_model</a> wrap LargeFile instances. Hence, besides configuring LargeFile, users have few reasons to use this.</p>
<h2 id="motivation">Motivation<a class="headerlink" href="#motivation" title="Permanent link">#</a></h2>
<p>Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation <a href="https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository">doesn't support the deletion</a> of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.</p>
<h2 id="solution">Solution<a class="headerlink" href="#solution" title="Permanent link">#</a></h2>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a>pip install open-large
<p>Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation <a href="https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository">doesn't support deleting</a> large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.</p>
<details class="note">
<summary>Using LargeFile-s directly (usually not needed)</summary>
<h2 id="simple-example">Simple example<a class="headerlink" href="#simple-example" title="Permanent link">#</a></h2>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">from</span> <span class="nn">great_ai.large_file</span> <span class="kn">import</span> <span class="n">LargeFileS3</span>
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a>
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="n">LargeFileS3</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">({</span>
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a> <span class="s2">&quot;aws_region_name&quot;</span><span class="p">:</span> <span class="s2">&quot;your_region_like_eu-west-2&quot;</span><span class="p">,</span>
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a> <span class="s2">&quot;aws_access_key_id&quot;</span><span class="p">:</span> <span class="s2">&quot;YOUR_ACCESS_KEY_ID&quot;</span><span class="p">,</span>
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a> <span class="s2">&quot;aws_secret_access_key&quot;</span><span class="p">:</span> <span class="s2">&quot;YOUR_VERY_SECRET_ACCESS_KEY&quot;</span><span class="p">,</span>
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a> <span class="s2">&quot;large_files_bucket_name&quot;</span><span class="p">:</span> <span class="s2">&quot;create_a_bucket_and_put_its_name_here&quot;</span><span class="p">,</span>
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="p">})</span>
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a>
<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a><span class="c1"># Creates a new version and deletes the older version </span>
<a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="c1"># leaving the 3 most recently used intact</span>
<a id="__codelineno-0-12" name="__codelineno-0-12" href="#__codelineno-0-12"></a><span class="k">with</span> <span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">,</span> <span class="n">keep_last_n</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<a id="__codelineno-0-13" name="__codelineno-0-13" href="#__codelineno-0-13"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100000</span><span class="p">):</span>
<a id="__codelineno-0-14" name="__codelineno-0-14" href="#__codelineno-0-14"></a> <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;test</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
<a id="__codelineno-0-15" name="__codelineno-0-15" href="#__codelineno-0-15"></a>
<a id="__codelineno-0-16" name="__codelineno-0-16" href="#__codelineno-0-16"></a><span class="c1"># By default the latest version is returned</span>
<a id="__codelineno-0-17" name="__codelineno-0-17" href="#__codelineno-0-17"></a><span class="c1"># but an optional `version` keyword argument can be provided as well</span>
<a id="__codelineno-0-18" name="__codelineno-0-18" href="#__codelineno-0-18"></a><span class="k">with</span> <span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span> <span class="c1">#(1)</span>
<a id="__codelineno-0-19" name="__codelineno-0-19" href="#__codelineno-0-19"></a> <span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span>
</code></pre></div>
<h3 id="simple-example">Simple example<a class="headerlink" href="#simple-example" title="Permanent link">#</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="kn">from</span> <span class="nn">great_ai.large_file</span> <span class="kn">import</span> <span class="n">LargeFileS3</span>
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a>
<a id="__codelineno-1-3" name="__codelineno-1-3" href="#__codelineno-1-3"></a><span class="n">LargeFileS3</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">({</span>
<a id="__codelineno-1-4" name="__codelineno-1-4" href="#__codelineno-1-4"></a> <span class="s2">&quot;aws_region_name&quot;</span><span class="p">:</span> <span class="s2">&quot;your_region_like_eu-west-2&quot;</span><span class="p">,</span>
<a id="__codelineno-1-5" name="__codelineno-1-5" href="#__codelineno-1-5"></a> <span class="s2">&quot;aws_access_key_id&quot;</span><span class="p">:</span> <span class="s2">&quot;YOUR_ACCESS_KEY_ID&quot;</span><span class="p">,</span>
<a id="__codelineno-1-6" name="__codelineno-1-6" href="#__codelineno-1-6"></a> <span class="s2">&quot;aws_secret_access_key&quot;</span><span class="p">:</span> <span class="s2">&quot;YOUR_VERY_SECRET_ACCESS_KEY&quot;</span><span class="p">,</span>
<a id="__codelineno-1-7" name="__codelineno-1-7" href="#__codelineno-1-7"></a> <span class="s2">&quot;large_files_bucket_name&quot;</span><span class="p">:</span> <span class="s2">&quot;create_a_bucket_and_put_its_name_here&quot;</span><span class="p">,</span>
<a id="__codelineno-1-8" name="__codelineno-1-8" href="#__codelineno-1-8"></a><span class="p">})</span>
<a id="__codelineno-1-9" name="__codelineno-1-9" href="#__codelineno-1-9"></a>
<a id="__codelineno-1-10" name="__codelineno-1-10" href="#__codelineno-1-10"></a><span class="c1"># Creates a new version and deletes the older version leaving the 3 most recently used intact</span>
<a id="__codelineno-1-11" name="__codelineno-1-11" href="#__codelineno-1-11"></a><span class="k">with</span> <span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;w&quot;</span><span class="p">,</span> <span class="n">keep_last_n</span><span class="o">=</span><span class="mi">3</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<a id="__codelineno-1-12" name="__codelineno-1-12" href="#__codelineno-1-12"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="mi">100000</span><span class="p">):</span>
<a id="__codelineno-1-13" name="__codelineno-1-13" href="#__codelineno-1-13"></a> <span class="n">f</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="s1">&#39;test</span><span class="se">\n</span><span class="s1">&#39;</span><span class="p">)</span>
<a id="__codelineno-1-14" name="__codelineno-1-14" href="#__codelineno-1-14"></a>
<a id="__codelineno-1-15" name="__codelineno-1-15" href="#__codelineno-1-15"></a><span class="c1"># By default the latest version is returned</span>
<a id="__codelineno-1-16" name="__codelineno-1-16" href="#__codelineno-1-16"></a><span class="c1"># but an optional `version` keyword argument can be provided as well</span>
<a id="__codelineno-1-17" name="__codelineno-1-17" href="#__codelineno-1-17"></a><span class="k">with</span> <span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;test.txt&quot;</span><span class="p">,</span> <span class="s2">&quot;r&quot;</span><span class="p">)</span> <span class="k">as</span> <span class="n">f</span><span class="p">:</span>
<a id="__codelineno-1-18" name="__codelineno-1-18" href="#__codelineno-1-18"></a> <span class="nb">print</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">readlines</span><span class="p">()[</span><span class="mi">0</span><span class="p">])</span>
</code></pre></div>
<blockquote>
<p>Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
In this case, the latest version is already in the local cache, no download is required.</p>
</blockquote>
<ol>
<li>In this case, the latest version is already in the local cache, no download is required.</li>
</ol>
<h3 id="more-details">More details<a class="headerlink" href="#more-details" title="Permanent link">#</a></h3>
<p><code>LargeFile</code> behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the <a href="https://docs.python.org/3/library/functions.html#open">different keywords</a> <code>open()</code> accepts.</p>
<p><code>LargeFile</code> behaves like an opened file (in the background it is a temp file after all). Binary reads and writes are supported along with the <a href="https://docs.python.org/3/library/functions.html#open">different keywords <code>open()</code> accepts</a>.</p>
<p>The local cache can be configured with these properties:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="n">LargeFile</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s1">&#39;.cache&#39;</span><span class="p">)</span>
<a id="__codelineno-2-2" name="__codelineno-2-2" href="#__codelineno-2-2"></a><span class="n">LargeFile</span><span class="o">.</span><span class="n">max_cache_size</span> <span class="o">=</span> <span class="s2">&quot;30 GB&quot;</span>
<div class="highlight"><pre><span></span><code><a id="__codelineno-1-1" name="__codelineno-1-1" href="#__codelineno-1-1"></a><span class="n">LargeFileS3</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s1">&#39;.cache&#39;</span><span class="p">)</span>
<a id="__codelineno-1-2" name="__codelineno-1-2" href="#__codelineno-1-2"></a><span class="n">LargeFileS3</span><span class="o">.</span><span class="n">max_cache_size</span> <span class="o">=</span> <span class="s2">&quot;30 GB&quot;</span>
</code></pre></div>
<h4 id="i-only-need-a-path">I only need a path<a class="headerlink" href="#i-only-need-a-path" title="Permanent link">#</a></h4>
<p>In case you only need a path to the "remote" file, this pattern can be applied:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="n">path_to_model</span> <span class="o">=</span> <span class="n">LargeFile</span><span class="p">(</span><span class="s2">&quot;folder-of-my-bert-model&quot;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">31</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
<div class="highlight"><pre><span></span><code><a id="__codelineno-2-1" name="__codelineno-2-1" href="#__codelineno-2-1"></a><span class="n">path_to_model</span> <span class="o">=</span> <span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;folder-of-my-bert-model&quot;</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="mi">31</span><span class="p">)</span><span class="o">.</span><span class="n">get</span><span class="p">()</span>
</code></pre></div>
<blockquote>
<p>This will first download the file/folder into your local cache folder. Then, it returns a <code>Path</code> object to the local version. Which can be turned into a string with <code>str(path_to_model)</code>.</p>
</blockquote>
<p>The same approach works for uploads:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="n">LargeFile</span><span class="p">(</span><span class="s2">&quot;folder-of-my-bert-model&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">push</span><span class="p">(</span><span class="s1">&#39;path_to_local/folder_or_file&#39;</span><span class="p">)</span>
<div class="highlight"><pre><span></span><code><a id="__codelineno-3-1" name="__codelineno-3-1" href="#__codelineno-3-1"></a><span class="n">LargeFileS3</span><span class="p">(</span><span class="s2">&quot;folder-of-my-bert-model&quot;</span><span class="p">)</span><span class="o">.</span><span class="n">push</span><span class="p">(</span><span class="s1">&#39;path_to_local/folder_or_file&#39;</span><span class="p">)</span>
</code></pre></div>
<blockquote>
<p>This way, both regular files and folders can be handled. The uploaded file is called <strong>folder-of-my-bert-model</strong>, the local name is ignored.</p>
</blockquote>
<p>Lastly, all version of the remote object can be deleted by calling <code>LargeFile("my-file").delete()</code>. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.</p>
<h3 id="command-line-example">Command-line example<a class="headerlink" href="#command-line-example" title="Permanent link">#</a></h3>
<p>The package can be used as a module from the command-line to give you more flexibility.</p>
<h4 id="setup">Setup<a class="headerlink" href="#setup" title="Permanent link">#</a></h4>
<p>Lastly, all version of the remote object can be deleted by calling <code>LargeFileS3("my-file").delete()</code>. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.</p>
</details>
<h2 id="from-the-command-line">From the command-line<a class="headerlink" href="#from-the-command-line" title="Permanent link">#</a></h2>
<p>The main reason for using the <code>large-file</code> or <code>python3 -m great_ai.large_file</code> commands is to upload or download models from the terminal. For example, when building a docker image, it is best-practice to cache the referred models.</p>
<h3 id="setup">Setup<a class="headerlink" href="#setup" title="Permanent link">#</a></h3>
<p>Create an .ini file (or use <em>~/.aws/credentials</em>). It may look like this:</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a><span class="na">aws_region_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">your_region_like_eu-west-2</span><span class="w"></span>
<a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a><span class="na">aws_access_key_id</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">YOUR_ACCESS_KEY_ID</span><span class="w"></span>
<a id="__codelineno-5-3" name="__codelineno-5-3" href="#__codelineno-5-3"></a><span class="na">aws_secret_access_key</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">YOUR_VERY_SECRET_ACCESS_KEY</span><span class="w"></span>
<a id="__codelineno-5-4" name="__codelineno-5-4" href="#__codelineno-5-4"></a><span class="na">large_files_bucket_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">my_large_files</span><span class="w"></span>
<a id="__codelineno-5-5" name="__codelineno-5-5" href="#__codelineno-5-5"></a><span class="na">endpoint_url</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com</span><span class="w"></span>
<div class="highlight"><pre><span></span><code><a id="__codelineno-4-1" name="__codelineno-4-1" href="#__codelineno-4-1"></a><span class="na">aws_region_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">your_region_like_eu-west-2</span><span class="w"></span>
<a id="__codelineno-4-2" name="__codelineno-4-2" href="#__codelineno-4-2"></a><span class="na">aws_access_key_id</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">YOUR_ACCESS_KEY_ID</span><span class="w"></span>
<a id="__codelineno-4-3" name="__codelineno-4-3" href="#__codelineno-4-3"></a><span class="na">aws_secret_access_key</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">YOUR_VERY_SECRET_ACCESS_KEY</span><span class="w"></span>
<a id="__codelineno-4-4" name="__codelineno-4-4" href="#__codelineno-4-4"></a><span class="na">large_files_bucket_name</span><span class="w"> </span><span class="o">=</span><span class="w"> </span><span class="s">my_large_files</span><span class="w"></span>
</code></pre></div>
<blockquote>
<p>Just like in <a href="example_secrets.ini">example secrets</a>.</p>
</blockquote>
<h4 id="print-the-expected-options">Print the expected options<a class="headerlink" href="#print-the-expected-options" title="Permanent link">#</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>python3 -m large_file --help
</code></pre></div>
<h4 id="upload-some-files">Upload some files<a class="headerlink" href="#upload-some-files" title="Permanent link">#</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
<h3 id="upload-some-files">Upload some files<a class="headerlink" href="#upload-some-files" title="Permanent link">#</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-5-1" name="__codelineno-5-1" href="#__codelineno-5-1"></a>large-file --backend s3 --secrets secrets.ini <span class="se">\</span>
<a id="__codelineno-5-2" name="__codelineno-5-2" href="#__codelineno-5-2"></a> --push my_first_file.json folder/my_second_file my_folder
</code></pre></div>
<blockquote>
<p>Only the filename is used as the S3 name, the rest of the path is ignored.</p>
</blockquote>
<h4 id="download-some-files-to-the-local-cache">Download some files to the local cache<a class="headerlink" href="#download-some-files-to-the-local-cache" title="Permanent link">#</a></h4>
<div class="admonition important">
<p class="admonition-title">Using MongoDB</p>
<p>The possible values for <code>--backend</code> are <code>s3</code>, <code>mongo</code>, and <code>local</code>. The latter doesn't need credentials, it only versions and stores your files in a local folder. MongoDB on the other hand requires a <code>mongo_connection_string</code> and a <code>mongo_database</code> to be specified. For storing large files, it uses the GridFS specification.</p>
</div>
<h3 id="download-some-files-to-the-local-cache">Download some files to the local cache<a class="headerlink" href="#download-some-files-to-the-local-cache" title="Permanent link">#</a></h3>
<p>This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.</p>
<div class="highlight"><pre><span></span><code><a id="__codelineno-8-1" name="__codelineno-8-1" href="#__codelineno-8-1"></a>python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
<div class="highlight"><pre><span></span><code><a id="__codelineno-6-1" name="__codelineno-6-1" href="#__codelineno-6-1"></a>large-file --backend s3 --secrets ~/.aws/credentials <span class="se">\</span>
<a id="__codelineno-6-2" name="__codelineno-6-2" href="#__codelineno-6-2"></a> --cache my_first_file.json:3 my_second_file my_folder:0
</code></pre></div>
<blockquote>
<p>Versions may be specified by using <code>:</code>-s.</p>
</blockquote>
<h4 id="delete-remote-files">Delete remote files<a class="headerlink" href="#delete-remote-files" title="Permanent link">#</a></h4>
<div class="highlight"><pre><span></span><code><a id="__codelineno-9-1" name="__codelineno-9-1" href="#__codelineno-9-1"></a>python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
<h3 id="delete-remote-files">Delete remote files<a class="headerlink" href="#delete-remote-files" title="Permanent link">#</a></h3>
<div class="highlight"><pre><span></span><code><a id="__codelineno-7-1" name="__codelineno-7-1" href="#__codelineno-7-1"></a>large-file --backend s3 --secrets ~/.aws/credentials <span class="se">\</span>
<a id="__codelineno-7-2" name="__codelineno-7-2" href="#__codelineno-7-2"></a> --delete my_first_file.json
</code></pre></div>
<hr>
@ -1054,7 +1026,7 @@ In this case, the latest version is already in the local cache, no download is r
<small>
Last update:
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 11, 2022</span>
<span class="git-revision-date-localized-plugin git-revision-date-localized-plugin-date">July 12, 2022</span>
</small>
@ -1075,7 +1047,7 @@ In this case, the latest version is already in the local cache, no download is r
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../handle-training-data/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to handle training data" rel="prev">
<a href="../handle-training-data/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to manage training data" rel="prev">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</div>
@ -1084,20 +1056,20 @@ In this case, the latest version is already in the local cache, no download is r
<span class="md-footer__direction">
Previous
</span>
How to handle training data
How to manage training data
</div>
</div>
</a>
<a href="../scraping/" class="md-footer__link md-footer__link--next" aria-label="Next: Scraping" rel="next">
<a href="../call-remote/" class="md-footer__link md-footer__link--next" aria-label="Next: Call remote GreatAI instances" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
Scraping
Call remote GreatAI instances
</div>
</div>
<div class="md-footer__button md-icon">

69
how-to-guides/large_file/large_file.md
View file

@ -1,18 +1,14 @@
# How to use LargeFile
# How to use LargeFile-s
Storing, versioning, and downloading files from S3 made as easy as using `open()` in Python. Caching included.
The functions [save_model][great_ai.use_model] and [@use_model][great_ai.use_model] wrap LargeFile instances. Hence, besides configuring LargeFile, users have few reasons to use this.
## Motivation
Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support the deletion](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) of large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
Oftentimes, especially when working with data-heavy applications, large files can proliferate in a repository. Version controlling them is an obvious next step, however, GitHub's git LFS implementation [doesn't support deleting](https://docs.github.com/en/repositories/working-with-files/managing-large-files/removing-files-from-git-large-file-storage#git-lfs-objects-in-your-repository) large files, making it easy for them to eat-up the LFS quota and explode the size of your repos.
## Solution
```
pip install open-large
```
### Simple example
??? note "Using LargeFile-s directly (usually not needed)"
## Simple example
```python
from great_ai.large_file import LargeFileS3
@ -24,29 +20,29 @@ LargeFileS3.configure_credentials({
"large_files_bucket_name": "create_a_bucket_and_put_its_name_here",
})
# Creates a new version and deletes the older version leaving the 3 most recently used intact
# Creates a new version and deletes the older version
# leaving the 3 most recently used intact
with LargeFileS3("test.txt", "w", keep_last_n=3) as f:
for i in range(100000):
f.write('test\n')
# By default the latest version is returned
# but an optional `version` keyword argument can be provided as well
with LargeFileS3("test.txt", "r") as f:
with LargeFileS3("test.txt", "r") as f: #(1)
print(f.readlines()[0])
```
> Automatically creates a file, writes to it, uploads it to S3, and then queries the most recent version of it.
> In this case, the latest version is already in the local cache, no download is required.
1. In this case, the latest version is already in the local cache, no download is required.
### More details
`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reading and writing is supported along with the [different keywords](https://docs.python.org/3/library/functions.html#open) `open()` accepts.
`LargeFile` behaves like an opened file (in the background it is a temp file after all). Binary reads and writes are supported along with the [different keywords `open()` accepts](https://docs.python.org/3/library/functions.html#open).
The local cache can be configured with these properties:
```python
LargeFile.cache_path = Path('.cache')
LargeFile.max_cache_size = "30 GB"
LargeFileS3.cache_path = Path('.cache')
LargeFileS3.max_cache_size = "30 GB"
```
#### I only need a path
@ -54,7 +50,7 @@ LargeFile.max_cache_size = "30 GB"
In case you only need a path to the "remote" file, this pattern can be applied:
```python
path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
path_to_model = LargeFileS3("folder-of-my-bert-model", version=31).get()
```
> This will first download the file/folder into your local cache folder. Then, it returns a `Path` object to the local version. Which can be turned into a string with `str(path_to_model)`.
@ -62,57 +58,54 @@ path_to_model = LargeFile("folder-of-my-bert-model", version=31).get()
The same approach works for uploads:
```python
LargeFile("folder-of-my-bert-model").push('path_to_local/folder_or_file')
LargeFileS3("folder-of-my-bert-model").push('path_to_local/folder_or_file')
```
> This way, both regular files and folders can be handled. The uploaded file is called **folder-of-my-bert-model**, the local name is ignored.
Lastly, all version of the remote object can be deleted by calling `LargeFile("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
Lastly, all version of the remote object can be deleted by calling `LargeFileS3("my-file").delete()`. It will still reside in your local cache afterwards, its deletion will happen next time your local cache has to be pruned.
### Command-line example
## From the command-line
The package can be used as a module from the command-line to give you more flexibility.
The main reason for using the `large-file` or `python3 -m great_ai.large_file` commands is to upload or download models from the terminal. For example, when building a docker image, it is best-practice to cache the referred models.
#### Setup
### Setup
Create an .ini file (or use _~/.aws/credentials_). It may look like this:
Create an .ini file (or use *~/.aws/credentials*). It may look like this:
```ini
aws_region_name = your_region_like_eu-west-2
aws_access_key_id = YOUR_ACCESS_KEY_ID
aws_secret_access_key = YOUR_VERY_SECRET_ACCESS_KEY
large_files_bucket_name = my_large_files
endpoint_url = this is optional, for backblaze, use this: https://s3.us-west-002.backblazeb2.com
```
> Just like in [example secrets](example_secrets.ini).
#### Print the expected options
### Upload some files
```sh
python3 -m large_file --help
```
#### Upload some files
```sh
python3 -m large_file --backend s3 --secrets secrets.ini --push my_first_file.json folder/my_second_file my_folder
large-file --backend s3 --secrets secrets.ini \
--push my_first_file.json folder/my_second_file my_folder
```
> Only the filename is used as the S3 name, the rest of the path is ignored.
#### Download some files to the local cache
!!! important "Using MongoDB"
The possible values for `--backend` are `s3`, `mongo`, and `local`. The latter doesn't need credentials, it only versions and stores your files in a local folder. MongoDB on the other hand requires a `mongo_connection_string` and a `mongo_database` to be specified. For storing large files, it uses the GridFS specification.
### Download some files to the local cache
This can be useful when building a Docker image for example. This way, the files can already reside inside the container and need not be downloaded later.
```sh
python3 -m large_file --backend s3 -secrets ~/.aws/credentials --cache my_first_file.json:3 my_second_file my_folder:0
large-file --backend s3 --secrets ~/.aws/credentials \
--cache my_first_file.json:3 my_second_file my_folder:0
```
> Versions may be specified by using `:`-s.
#### Delete remote files
### Delete remote files
```sh
python3 -m large_file --backend s3 --secrets ~/.aws/credentials --delete my_first_file.json
large-file --backend s3 --secrets ~/.aws/credentials \
--delete my_first_file.json
```

14
how-to-guides/scraping/index.html
View file

@ -405,7 +405,7 @@
<li class="md-nav__item">
<a href="../use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -419,7 +419,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -433,7 +433,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -446,8 +446,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -1955,7 +1955,7 @@ greeter("Bob")</div>
<nav class="md-footer__inner md-grid" aria-label="Footer" >
<a href="../large_file/" class="md-footer__link md-footer__link--prev" aria-label="Previous: How to use LargeFile" rel="prev">
<a href="../call-remote/" class="md-footer__link md-footer__link--prev" aria-label="Previous: Call remote GreatAI instances" rel="prev">
<div class="md-footer__button md-icon">
<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M20 11v2H8l5.5 5.5-1.42 1.42L4.16 12l7.92-7.92L13.5 5.5 8 11h12Z"/></svg>
</div>
@ -1964,7 +1964,7 @@ greeter("Bob")</div>
<span class="md-footer__direction">
Previous
</span>
How to use LargeFile
Call remote GreatAI instances
</div>
</div>
</a>

24
how-to-guides/use-service/index.html
View file

@ -17,7 +17,7 @@
<title>How to use a GreatAI service - GreatAI documentation</title>
<title>How to deploy a GreatAI service - GreatAI documentation</title>
@ -83,7 +83,7 @@
<div data-md-component="skip">
<a href="#how-to-use-a-greatai-service" class="md-skip">
<a href="#how-to-deploy-a-greatai-service" class="md-skip">
Skip to content
</a>
@ -116,7 +116,7 @@
<div class="md-header__topic" data-md-component="header-topic">
<span class="md-ellipsis">
How to use a GreatAI service
How to deploy a GreatAI service
</span>
</div>
@ -419,12 +419,12 @@
<label class="md-nav__link md-nav__link--active" for="__toc">
How to use a GreatAI service
How to deploy a GreatAI service
<span class="md-nav__icon md-icon"></span>
</label>
<a href="./" class="md-nav__link md-nav__link--active">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
@ -526,7 +526,7 @@
<li class="md-nav__item">
<a href="../handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -540,7 +540,7 @@
<li class="md-nav__item">
<a href="../large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -553,8 +553,8 @@
<li class="md-nav__item">
<a href="../call_remote.md" class="md-nav__link">
None
<a href="../call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -911,7 +911,7 @@
<h1 id="how-to-use-a-greatai-service">How to use a GreatAI service<a class="headerlink" href="#how-to-use-a-greatai-service" title="Permanent link">#</a></h1>
<h1 id="how-to-deploy-a-greatai-service">How to deploy a GreatAI service<a class="headerlink" href="#how-to-deploy-a-greatai-service" title="Permanent link">#</a></h1>
<p>After <a href="/how-to-guides/create-service">creating a GreatAI service</a> by wrapping your prediction function, and optionally <a href="/how-to-guides/configure-service">configuring it</a>, it's time to do some prediction.</p>
<p>Let's take the following example:</p>
<div class="highlight"><span class="filename">greeter.py</span><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="kn">from</span> <span class="nn">great_ai</span> <span class="kn">import</span> <span class="n">GreatAI</span>
@ -1054,13 +1054,13 @@
<a href="../handle-training-data/" class="md-footer__link md-footer__link--next" aria-label="Next: How to handle training data" rel="next">
<a href="../handle-training-data/" class="md-footer__link md-footer__link--next" aria-label="Next: How to manage training data" rel="next">
<div class="md-footer__title">
<div class="md-ellipsis">
<span class="md-footer__direction">
Next
</span>
How to handle training data
How to manage training data
</div>
</div>
<div class="md-footer__button md-icon">

2
how-to-guides/use-service/use-service.md
View file

@ -1,4 +1,4 @@
# How to use a GreatAI service
# How to deploy a GreatAI service
After [creating a GreatAI service](/how-to-guides/create-service) by wrapping your prediction function, and optionally [configuring it](/how-to-guides/configure-service), it's time to do some prediction.

10
index.html
View file

@ -480,7 +480,7 @@
<li class="md-nav__item">
<a href="how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -494,7 +494,7 @@
<li class="md-nav__item">
<a href="how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -508,7 +508,7 @@
<li class="md-nav__item">
<a href="how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -521,8 +521,8 @@
<li class="md-nav__item">
<a href="how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

BIN
media/feedback.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 46 KiB

BIN
media/thesis-frontpage.png Normal file
View file

Binary file not shown.
Side by side

After

Width:  |  Height:  |  Size: 138 KiB

392
reference/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -3127,6 +3127,10 @@ Returns:
<div class="doc doc-contents first">
<p>Communicate with a GreatAI object through an HTTP request.</p>
<p>Wrapper over <code>call_remote_great_ai_async</code> making it synchronous. For more info, see
<code>call_remote_great_ai_async</code>.</p>
<details class="quote">
<summary>Source code in <code>great_ai/remote/call_remote_great_ai.py</code></summary>
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-16">16</a></span>
@ -3152,30 +3156,40 @@ Returns:
<span class="normal"><a href="#__codelineno-0-36">36</a></span>
<span class="normal"><a href="#__codelineno-0-37">37</a></span>
<span class="normal"><a href="#__codelineno-0-38">38</a></span>
<span class="normal"><a href="#__codelineno-0-39">39</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="k">def</span> <span class="nf">call_remote_great_ai</span><span class="p">(</span>
<span class="normal"><a href="#__codelineno-0-39">39</a></span>
<span class="normal"><a href="#__codelineno-0-40">40</a></span>
<span class="normal"><a href="#__codelineno-0-41">41</a></span>
<span class="normal"><a href="#__codelineno-0-42">42</a></span>
<span class="normal"><a href="#__codelineno-0-43">43</a></span>
<span class="normal"><a href="#__codelineno-0-44">44</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="k">def</span> <span class="nf">call_remote_great_ai</span><span class="p">(</span>
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="n">base_uri</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<a id="__codelineno-0-18" name="__codelineno-0-18"></a> <span class="n">data</span><span class="p">:</span> <span class="n">Mapping</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
<a id="__codelineno-0-19" name="__codelineno-0-19"></a> <span class="n">retry_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="n">timeout_in_seconds</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">300</span><span class="p">,</span>
<a id="__codelineno-0-21" name="__codelineno-0-21"></a> <span class="n">model_class</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="n">T</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Trace</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="n">asyncio</span><span class="o">.</span><span class="n">get_running_loop</span><span class="p">()</span>
<a id="__codelineno-0-25" name="__codelineno-0-25"></a> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="sa">f</span><span class="s2">&quot;Already running in an event loop, you have to call `</span><span class="si">{</span><span class="n">call_remote_great_ai_async</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">`&quot;</span>
<a id="__codelineno-0-27" name="__codelineno-0-27"></a> <span class="p">)</span>
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="k">except</span> <span class="ne">RuntimeError</span><span class="p">:</span>
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="k">pass</span>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a>
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="n">future</span> <span class="o">=</span> <span class="n">call_remote_great_ai_async</span><span class="p">(</span>
<a id="__codelineno-0-32" name="__codelineno-0-32"></a> <span class="n">base_uri</span><span class="o">=</span><span class="n">base_uri</span><span class="p">,</span>
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
<a id="__codelineno-0-34" name="__codelineno-0-34"></a> <span class="n">retry_count</span><span class="o">=</span><span class="n">retry_count</span><span class="p">,</span>
<a id="__codelineno-0-35" name="__codelineno-0-35"></a> <span class="n">timeout_in_seconds</span><span class="o">=</span><span class="n">timeout_in_seconds</span><span class="p">,</span>
<a id="__codelineno-0-36" name="__codelineno-0-36"></a> <span class="n">model_class</span><span class="o">=</span><span class="n">model_class</span><span class="p">,</span>
<a id="__codelineno-0-37" name="__codelineno-0-37"></a> <span class="p">)</span>
<a id="__codelineno-0-38" name="__codelineno-0-38"></a>
<a id="__codelineno-0-39" name="__codelineno-0-39"></a> <span class="k">return</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">future</span><span class="p">)</span>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="sd">&quot;&quot;&quot;Communicate with a GreatAI object through an HTTP request.</span>
<a id="__codelineno-0-24" name="__codelineno-0-24"></a>
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> Wrapper over `call_remote_great_ai_async` making it synchronous. For more info, see</span>
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> `call_remote_great_ai_async`.</span>
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="n">asyncio</span><span class="o">.</span><span class="n">get_running_loop</span><span class="p">()</span>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="k">raise</span> <span class="ne">Exception</span><span class="p">(</span>
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="sa">f</span><span class="s2">&quot;Already running in an event loop, you have to call `</span><span class="si">{</span><span class="n">call_remote_great_ai_async</span><span class="o">.</span><span class="vm">__name__</span><span class="si">}</span><span class="s2">`&quot;</span>
<a id="__codelineno-0-32" name="__codelineno-0-32"></a> <span class="p">)</span>
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="k">except</span> <span class="ne">RuntimeError</span><span class="p">:</span>
<a id="__codelineno-0-34" name="__codelineno-0-34"></a> <span class="k">pass</span>
<a id="__codelineno-0-35" name="__codelineno-0-35"></a>
<a id="__codelineno-0-36" name="__codelineno-0-36"></a> <span class="n">future</span> <span class="o">=</span> <span class="n">call_remote_great_ai_async</span><span class="p">(</span>
<a id="__codelineno-0-37" name="__codelineno-0-37"></a> <span class="n">base_uri</span><span class="o">=</span><span class="n">base_uri</span><span class="p">,</span>
<a id="__codelineno-0-38" name="__codelineno-0-38"></a> <span class="n">data</span><span class="o">=</span><span class="n">data</span><span class="p">,</span>
<a id="__codelineno-0-39" name="__codelineno-0-39"></a> <span class="n">retry_count</span><span class="o">=</span><span class="n">retry_count</span><span class="p">,</span>
<a id="__codelineno-0-40" name="__codelineno-0-40"></a> <span class="n">timeout_in_seconds</span><span class="o">=</span><span class="n">timeout_in_seconds</span><span class="p">,</span>
<a id="__codelineno-0-41" name="__codelineno-0-41"></a> <span class="n">model_class</span><span class="o">=</span><span class="n">model_class</span><span class="p">,</span>
<a id="__codelineno-0-42" name="__codelineno-0-42"></a> <span class="p">)</span>
<a id="__codelineno-0-43" name="__codelineno-0-43"></a>
<a id="__codelineno-0-44" name="__codelineno-0-44"></a> <span class="k">return</span> <span class="n">asyncio</span><span class="o">.</span><span class="n">run</span><span class="p">(</span><span class="n">future</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
</details>
</div>
@ -3198,6 +3212,78 @@ Returns:
<div class="doc doc-contents first">
<p>Communicate with a GreatAI object through an HTTP request.</p>
<p>Send a POST request using <a href="https://www.python-httpx.org/">httpx</a> to implement a
remote call. Error-handling and retries are provided by httpx.</p>
<p>The return value is inflated into a Trace. If <code>model_class</code> is specified, the
original output is deserialised.</p>
<p><strong>Parameters:</strong></p>
<table>
<thead>
<tr>
<th>Name</th>
<th>Type</th>
<th>Description</th>
<th>Default</th>
</tr>
</thead>
<tbody>
<tr>
<td><code>base_uri</code></td>
<td>
<code>str</code>
</td>
<td><p>Address of the remote instance, example: 'http://localhost:6060'</p></td>
<td>
<em>required</em>
</td>
</tr>
<tr>
<td><code>data</code></td>
<td>
<code><span title="typing.Mapping">Mapping</span>[str, <span title="typing.Any">Any</span>]</code>
</td>
<td><p>The input sent as a json to the remote instance.</p></td>
<td>
<em>required</em>
</td>
</tr>
<tr>
<td><code>retry_count</code></td>
<td>
<code>int</code>
</td>
<td><p>Retry on any HTTP communication failure.</p></td>
<td>
<code>4</code>
</td>
</tr>
<tr>
<td><code>timeout_in_seconds</code></td>
<td>
<code><span title="typing.Optional">Optional</span>[int]</code>
</td>
<td><p>Overall permissable max length of the request. <code>None</code> means
no timeout.</p></td>
<td>
<code>300</code>
</td>
</tr>
<tr>
<td><code>model_class</code></td>
<td>
<code><span title="typing.Optional">Optional</span>[<span title="typing.Type">Type</span>[<span title="great_ai.remote.call_remote_great_ai_async.T">T</span>]]</code>
</td>
<td><p>A subtype of BaseModel to be used for deserialising the <code>.output</code>
of the trace.</p></td>
<td>
<code>None</code>
</td>
</tr>
</tbody>
</table>
<details class="quote">
<summary>Source code in <code>great_ai/remote/call_remote_great_ai_async.py</code></summary>
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-12">12</a></span>
@ -3239,46 +3325,80 @@ Returns:
<span class="normal"><a href="#__codelineno-0-48">48</a></span>
<span class="normal"><a href="#__codelineno-0-49">49</a></span>
<span class="normal"><a href="#__codelineno-0-50">50</a></span>
<span class="normal"><a href="#__codelineno-0-51">51</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="k">async</span> <span class="k">def</span> <span class="nf">call_remote_great_ai_async</span><span class="p">(</span>
<span class="normal"><a href="#__codelineno-0-51">51</a></span>
<span class="normal"><a href="#__codelineno-0-52">52</a></span>
<span class="normal"><a href="#__codelineno-0-53">53</a></span>
<span class="normal"><a href="#__codelineno-0-54">54</a></span>
<span class="normal"><a href="#__codelineno-0-55">55</a></span>
<span class="normal"><a href="#__codelineno-0-56">56</a></span>
<span class="normal"><a href="#__codelineno-0-57">57</a></span>
<span class="normal"><a href="#__codelineno-0-58">58</a></span>
<span class="normal"><a href="#__codelineno-0-59">59</a></span>
<span class="normal"><a href="#__codelineno-0-60">60</a></span>
<span class="normal"><a href="#__codelineno-0-61">61</a></span>
<span class="normal"><a href="#__codelineno-0-62">62</a></span>
<span class="normal"><a href="#__codelineno-0-63">63</a></span>
<span class="normal"><a href="#__codelineno-0-64">64</a></span>
<span class="normal"><a href="#__codelineno-0-65">65</a></span>
<span class="normal"><a href="#__codelineno-0-66">66</a></span>
<span class="normal"><a href="#__codelineno-0-67">67</a></span>
<span class="normal"><a href="#__codelineno-0-68">68</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-12" name="__codelineno-0-12"></a><span class="k">async</span> <span class="k">def</span> <span class="nf">call_remote_great_ai_async</span><span class="p">(</span>
<a id="__codelineno-0-13" name="__codelineno-0-13"></a> <span class="n">base_uri</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
<a id="__codelineno-0-14" name="__codelineno-0-14"></a> <span class="n">data</span><span class="p">:</span> <span class="n">Mapping</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">],</span>
<a id="__codelineno-0-15" name="__codelineno-0-15"></a> <span class="n">retry_count</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="mi">4</span><span class="p">,</span>
<a id="__codelineno-0-16" name="__codelineno-0-16"></a> <span class="n">timeout_in_seconds</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="mi">300</span><span class="p">,</span>
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="n">model_class</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="n">T</span><span class="p">]]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Trace</span><span class="p">[</span><span class="n">T</span><span class="p">]:</span>
<a id="__codelineno-0-19" name="__codelineno-0-19"></a>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="k">if</span> <span class="n">base_uri</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">):</span>
<a id="__codelineno-0-21" name="__codelineno-0-21"></a> <span class="n">base_uri</span> <span class="o">=</span> <span class="n">base_uri</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<a id="__codelineno-0-22" name="__codelineno-0-22"></a>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">base_uri</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/predict&quot;</span><span class="p">):</span>
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="n">base_uri</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_uri</span><span class="si">}</span><span class="s2">/predict&quot;</span>
<a id="__codelineno-0-25" name="__codelineno-0-25"></a>
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="n">transport</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">AsyncHTTPTransport</span><span class="p">(</span><span class="n">retries</span><span class="o">=</span><span class="n">retry_count</span><span class="p">)</span>
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="k">async</span> <span class="k">with</span> <span class="n">httpx</span><span class="o">.</span><span class="n">AsyncClient</span><span class="p">(</span>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="n">transport</span><span class="o">=</span><span class="n">transport</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout_in_seconds</span>
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="p">)</span> <span class="k">as</span> <span class="n">client</span><span class="p">:</span>
<a id="__codelineno-0-32" name="__codelineno-0-32"></a> <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">client</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="n">base_uri</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">data</span><span class="p">)</span>
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-34" name="__codelineno-0-34"></a> <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<a id="__codelineno-0-35" name="__codelineno-0-35"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-36" name="__codelineno-0-36"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span><span class="s2">&quot;Unexpected status code&quot;</span><span class="p">)</span>
<a id="__codelineno-0-37" name="__codelineno-0-37"></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<a id="__codelineno-0-38" name="__codelineno-0-38"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span> <span class="kn">from</span> <span class="nn">e</span>
<a id="__codelineno-0-19" name="__codelineno-0-19"></a> <span class="sd">&quot;&quot;&quot;Communicate with a GreatAI object through an HTTP request.</span>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a>
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="sd"> Send a POST request using [httpx](https://www.python-httpx.org/) to implement a</span>
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> remote call. Error-handling and retries are provided by httpx.</span>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a>
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> The return value is inflated into a Trace. If `model_class` is specified, the</span>
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> original output is deserialised.</span>
<a id="__codelineno-0-26" name="__codelineno-0-26"></a>
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> Args:</span>
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> base_uri: Address of the remote instance, example: &#39;http://localhost:6060&#39;</span>
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> data: The input sent as a json to the remote instance.</span>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> retry_count: Retry on any HTTP communication failure.</span>
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> timeout_in_seconds: Overall permissable max length of the request. `None` means</span>
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> no timeout.</span>
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> model_class: A subtype of BaseModel to be used for deserialising the `.output`</span>
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> of the trace.</span>
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-36" name="__codelineno-0-36"></a>
<a id="__codelineno-0-37" name="__codelineno-0-37"></a> <span class="k">if</span> <span class="n">base_uri</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/&quot;</span><span class="p">):</span>
<a id="__codelineno-0-38" name="__codelineno-0-38"></a> <span class="n">base_uri</span> <span class="o">=</span> <span class="n">base_uri</span><span class="p">[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
<a id="__codelineno-0-39" name="__codelineno-0-39"></a>
<a id="__codelineno-0-40" name="__codelineno-0-40"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-41" name="__codelineno-0-41"></a> <span class="n">trace</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<a id="__codelineno-0-42" name="__codelineno-0-42"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-43" name="__codelineno-0-43"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span>
<a id="__codelineno-0-44" name="__codelineno-0-44"></a> <span class="sa">f</span><span class="s2">&quot;JSON parsing failed </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<a id="__codelineno-0-45" name="__codelineno-0-45"></a> <span class="p">)</span>
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-47" name="__codelineno-0-47"></a> <span class="k">if</span> <span class="n">model_class</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="n">trace</span><span class="p">[</span><span class="s2">&quot;output&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">model_class</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">[</span><span class="s2">&quot;output&quot;</span><span class="p">])</span>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="k">return</span> <span class="n">Trace</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">)</span>
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span><span class="s2">&quot;Could not parse Trace&quot;</span><span class="p">)</span>
<a id="__codelineno-0-40" name="__codelineno-0-40"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">base_uri</span><span class="o">.</span><span class="n">endswith</span><span class="p">(</span><span class="s2">&quot;/predict&quot;</span><span class="p">):</span>
<a id="__codelineno-0-41" name="__codelineno-0-41"></a> <span class="n">base_uri</span> <span class="o">=</span> <span class="sa">f</span><span class="s2">&quot;</span><span class="si">{</span><span class="n">base_uri</span><span class="si">}</span><span class="s2">/predict&quot;</span>
<a id="__codelineno-0-42" name="__codelineno-0-42"></a>
<a id="__codelineno-0-43" name="__codelineno-0-43"></a> <span class="n">transport</span> <span class="o">=</span> <span class="n">httpx</span><span class="o">.</span><span class="n">AsyncHTTPTransport</span><span class="p">(</span><span class="n">retries</span><span class="o">=</span><span class="n">retry_count</span><span class="p">)</span>
<a id="__codelineno-0-44" name="__codelineno-0-44"></a>
<a id="__codelineno-0-45" name="__codelineno-0-45"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="k">async</span> <span class="k">with</span> <span class="n">httpx</span><span class="o">.</span><span class="n">AsyncClient</span><span class="p">(</span>
<a id="__codelineno-0-47" name="__codelineno-0-47"></a> <span class="n">transport</span><span class="o">=</span><span class="n">transport</span><span class="p">,</span> <span class="n">timeout</span><span class="o">=</span><span class="n">timeout_in_seconds</span>
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="p">)</span> <span class="k">as</span> <span class="n">client</span><span class="p">:</span>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">response</span> <span class="o">=</span> <span class="k">await</span> <span class="n">client</span><span class="o">.</span><span class="n">post</span><span class="p">(</span><span class="n">base_uri</span><span class="p">,</span> <span class="n">json</span><span class="o">=</span><span class="n">data</span><span class="p">)</span>
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">response</span><span class="o">.</span><span class="n">raise_for_status</span><span class="p">()</span>
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span><span class="s2">&quot;Unexpected status code&quot;</span><span class="p">)</span>
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="k">except</span> <span class="ne">Exception</span> <span class="k">as</span> <span class="n">e</span><span class="p">:</span>
<a id="__codelineno-0-55" name="__codelineno-0-55"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span> <span class="kn">from</span> <span class="nn">e</span>
<a id="__codelineno-0-56" name="__codelineno-0-56"></a>
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-58" name="__codelineno-0-58"></a> <span class="n">trace</span> <span class="o">=</span> <span class="n">response</span><span class="o">.</span><span class="n">json</span><span class="p">()</span>
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span>
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="sa">f</span><span class="s2">&quot;JSON parsing failed </span><span class="si">{</span><span class="n">response</span><span class="o">.</span><span class="n">text</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="p">)</span>
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="k">try</span><span class="p">:</span>
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="k">if</span> <span class="n">model_class</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="n">trace</span><span class="p">[</span><span class="s2">&quot;output&quot;</span><span class="p">]</span> <span class="o">=</span> <span class="n">model_class</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">[</span><span class="s2">&quot;output&quot;</span><span class="p">])</span>
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="k">return</span> <span class="n">Trace</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">)</span>
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="k">raise</span> <span class="n">RemoteCallError</span><span class="p">(</span><span class="s2">&quot;Could not parse Trace&quot;</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
</details>
</div>
@ -3299,7 +3419,7 @@ Returns:
<div class="doc doc-contents first">
<p>Add training data (with optional train-test splitting).</p>
<p>Add and tag datapoints, wrap them into traces. The <code>inputs</code> are available via the
<p>Add and tag data-points, wrap them into traces. The <code>inputs</code> are available via the
<code>.input</code> property, while <code>expected_outputs</code> under both the <code>.output</code> and <code>.feedback</code>
properties.</p>
<p>All generated traces are tagged with <code>ground_truth</code> by default. Additional tags can
@ -3316,6 +3436,18 @@ add up to 1.</p>
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="gp">... </span> <span class="n">test_split_ratio</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="gp">... </span> <span class="n">validation_split_ratio</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span>
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="gp">... </span><span class="p">)</span>
</code></pre></div>
<div class="highlight"><pre><span></span><code><a id="__codelineno-0-1" name="__codelineno-0-1" href="#__codelineno-0-1"></a><span class="gp">&gt;&gt;&gt; </span><span class="n">add_ground_truth</span><span class="p">(</span>
<a id="__codelineno-0-2" name="__codelineno-0-2" href="#__codelineno-0-2"></a><span class="gp">... </span> <span class="p">[</span><span class="mi">1</span><span class="p">,</span> <span class="mi">2</span><span class="p">],</span>
<a id="__codelineno-0-3" name="__codelineno-0-3" href="#__codelineno-0-3"></a><span class="gp">... </span> <span class="p">[</span><span class="s1">&#39;odd&#39;</span><span class="p">,</span> <span class="s1">&#39;even&#39;</span><span class="p">,</span> <span class="s1">&#39;odd&#39;</span><span class="p">],</span>
<a id="__codelineno-0-4" name="__codelineno-0-4" href="#__codelineno-0-4"></a><span class="gp">... </span> <span class="n">tags</span><span class="o">=</span><span class="s1">&#39;my_tag&#39;</span><span class="p">,</span>
<a id="__codelineno-0-5" name="__codelineno-0-5" href="#__codelineno-0-5"></a><span class="gp">... </span> <span class="n">train_split_ratio</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<a id="__codelineno-0-6" name="__codelineno-0-6" href="#__codelineno-0-6"></a><span class="gp">... </span> <span class="n">test_split_ratio</span><span class="o">=</span><span class="mi">1</span><span class="p">,</span>
<a id="__codelineno-0-7" name="__codelineno-0-7" href="#__codelineno-0-7"></a><span class="gp">... </span> <span class="n">validation_split_ratio</span><span class="o">=</span><span class="mf">0.5</span><span class="p">,</span>
<a id="__codelineno-0-8" name="__codelineno-0-8" href="#__codelineno-0-8"></a><span class="gp">... </span><span class="p">)</span>
<a id="__codelineno-0-9" name="__codelineno-0-9" href="#__codelineno-0-9"></a><span class="gt">Traceback (most recent call last):</span>
<a id="__codelineno-0-10" name="__codelineno-0-10" href="#__codelineno-0-10"></a> <span class="o">...</span>
<a id="__codelineno-0-11" name="__codelineno-0-11" href="#__codelineno-0-11"></a><span class="gr">AssertionError</span>: <span class="n">The length of the inputs and expected_outputs must be equal</span>
</code></pre></div>
<p><strong>Parameters:</strong></p>
@ -3353,7 +3485,7 @@ scikit-learn)</p></td>
<tr>
<td><code>tags</code></td>
<td>
<code><span title="typing.List">List</span>[str]</code>
<code><span title="typing.Union">Union</span>[<span title="typing.List">List</span>[str], str]</code>
</td>
<td><p>A single tag or a list of tags to append to each generated trace's tags.</p></td>
<td>
@ -3478,18 +3610,32 @@ scikit-learn)</p></td>
<span class="normal"><a href="#__codelineno-0-99"> 99</a></span>
<span class="normal"><a href="#__codelineno-0-100">100</a></span>
<span class="normal"><a href="#__codelineno-0-101">101</a></span>
<span class="normal"><a href="#__codelineno-0-102">102</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">def</span> <span class="nf">add_ground_truth</span><span class="p">(</span>
<span class="normal"><a href="#__codelineno-0-102">102</a></span>
<span class="normal"><a href="#__codelineno-0-103">103</a></span>
<span class="normal"><a href="#__codelineno-0-104">104</a></span>
<span class="normal"><a href="#__codelineno-0-105">105</a></span>
<span class="normal"><a href="#__codelineno-0-106">106</a></span>
<span class="normal"><a href="#__codelineno-0-107">107</a></span>
<span class="normal"><a href="#__codelineno-0-108">108</a></span>
<span class="normal"><a href="#__codelineno-0-109">109</a></span>
<span class="normal"><a href="#__codelineno-0-110">110</a></span>
<span class="normal"><a href="#__codelineno-0-111">111</a></span>
<span class="normal"><a href="#__codelineno-0-112">112</a></span>
<span class="normal"><a href="#__codelineno-0-113">113</a></span>
<span class="normal"><a href="#__codelineno-0-114">114</a></span>
<span class="normal"><a href="#__codelineno-0-115">115</a></span>
<span class="normal"><a href="#__codelineno-0-116">116</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">def</span> <span class="nf">add_ground_truth</span><span class="p">(</span>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="n">inputs</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span>
<a id="__codelineno-0-21" name="__codelineno-0-21"></a> <span class="n">expected_outputs</span><span class="p">:</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
<a id="__codelineno-0-22" name="__codelineno-0-22"></a> <span class="o">*</span><span class="p">,</span>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="n">tags</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[],</span>
<a id="__codelineno-0-23" name="__codelineno-0-23"></a> <span class="n">tags</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">],</span> <span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="p">[],</span>
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="n">train_split_ratio</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">1</span><span class="p">,</span>
<a id="__codelineno-0-25" name="__codelineno-0-25"></a> <span class="n">test_split_ratio</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span><span class="p">,</span>
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="n">validation_split_ratio</span><span class="p">:</span> <span class="nb">float</span> <span class="o">=</span> <span class="mi">0</span>
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="sd">&quot;&quot;&quot;Add training data (with optional train-test splitting).</span>
<a id="__codelineno-0-29" name="__codelineno-0-29"></a>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> Add and tag datapoints, wrap them into traces. The `inputs` are available via the</span>
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> Add and tag data-points, wrap them into traces. The `inputs` are available via the</span>
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> `.input` property, while `expected_outputs` under both the `.output` and `.feedback`</span>
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> properties.</span>
<a id="__codelineno-0-33" name="__codelineno-0-33"></a>
@ -3508,60 +3654,74 @@ scikit-learn)</p></td>
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> ... validation_split_ratio=0.5,</span>
<a id="__codelineno-0-47" name="__codelineno-0-47"></a><span class="sd"> ... )</span>
<a id="__codelineno-0-48" name="__codelineno-0-48"></a>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="sd"> Args:</span>
<a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="sd"> inputs: The inputs. (X in scikit-learn)</span>
<a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd"> expected_outputs: The ground-truth values corresponding to the inputs. (y in</span>
<a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="sd"> scikit-learn)</span>
<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd"> tags: A single tag or a list of tags to append to each generated trace&#39;s tags.</span>
<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd"> train_split_ratio: The probability-weight of giving each trace the `train` tag.</span>
<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd"> test_split_ratio: The probability-weight of giving each trace the `test` tag.</span>
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> validation_split_ratio: The probability-weight of giving each trace the</span>
<a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="sd"> `validation` tag.</span>
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-59" name="__codelineno-0-59"></a>
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="n">inputs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="n">expected_outputs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">expected_outputs</span><span class="p">)</span>
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span>
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="n">expected_outputs</span>
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="p">),</span> <span class="s2">&quot;The length of the inputs and expected_outputs must be equal&quot;</span>
<a id="__codelineno-0-65" name="__codelineno-0-65"></a>
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="n">sum_ratio</span> <span class="o">=</span> <span class="n">train_split_ratio</span> <span class="o">+</span> <span class="n">test_split_ratio</span> <span class="o">+</span> <span class="n">validation_split_ratio</span>
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="k">assert</span> <span class="n">sum_ratio</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;The sum of the split ratios must be a positive number&quot;</span>
<a id="__codelineno-0-68" name="__codelineno-0-68"></a>
<a id="__codelineno-0-69" name="__codelineno-0-69"></a> <span class="n">train_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="n">test_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="n">validation_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-72" name="__codelineno-0-72"></a>
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="n">values</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">expected_outputs</span><span class="p">))</span>
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="n">shuffle</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="n">split_tags</span> <span class="o">=</span> <span class="p">(</span>
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="p">[</span><span class="n">TRAIN_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">train_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="o">+</span> <span class="p">[</span><span class="n">TEST_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">test_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="o">+</span> <span class="p">[</span><span class="n">VALIDATION_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">validation_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="p">)</span>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">shuffle</span><span class="p">(</span><span class="n">split_tags</span><span class="p">)</span>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="sd"> &gt;&gt;&gt; add_ground_truth(</span>
<a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="sd"> ... [1, 2],</span>
<a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd"> ... [&#39;odd&#39;, &#39;even&#39;, &#39;odd&#39;],</span>
<a id="__codelineno-0-52" name="__codelineno-0-52"></a><span class="sd"> ... tags=&#39;my_tag&#39;,</span>
<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd"> ... train_split_ratio=1,</span>
<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd"> ... test_split_ratio=1,</span>
<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd"> ... validation_split_ratio=0.5,</span>
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> ... )</span>
<a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="sd"> Traceback (most recent call last):</span>
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> ...</span>
<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd"> AssertionError: The length of the inputs and expected_outputs must be equal</span>
<a id="__codelineno-0-60" name="__codelineno-0-60"></a>
<a id="__codelineno-0-61" name="__codelineno-0-61"></a><span class="sd"> Args:</span>
<a id="__codelineno-0-62" name="__codelineno-0-62"></a><span class="sd"> inputs: The inputs. (X in scikit-learn)</span>
<a id="__codelineno-0-63" name="__codelineno-0-63"></a><span class="sd"> expected_outputs: The ground-truth values corresponding to the inputs. (y in</span>
<a id="__codelineno-0-64" name="__codelineno-0-64"></a><span class="sd"> scikit-learn)</span>
<a id="__codelineno-0-65" name="__codelineno-0-65"></a><span class="sd"> tags: A single tag or a list of tags to append to each generated trace&#39;s tags.</span>
<a id="__codelineno-0-66" name="__codelineno-0-66"></a><span class="sd"> train_split_ratio: The probability-weight of giving each trace the `train` tag.</span>
<a id="__codelineno-0-67" name="__codelineno-0-67"></a><span class="sd"> test_split_ratio: The probability-weight of giving each trace the `test` tag.</span>
<a id="__codelineno-0-68" name="__codelineno-0-68"></a><span class="sd"> validation_split_ratio: The probability-weight of giving each trace the</span>
<a id="__codelineno-0-69" name="__codelineno-0-69"></a><span class="sd"> `validation` tag.</span>
<a id="__codelineno-0-70" name="__codelineno-0-70"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-71" name="__codelineno-0-71"></a>
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="n">inputs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span>
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="n">expected_outputs</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="n">expected_outputs</span><span class="p">)</span>
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="k">assert</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">)</span> <span class="o">==</span> <span class="nb">len</span><span class="p">(</span>
<a id="__codelineno-0-75" name="__codelineno-0-75"></a> <span class="n">expected_outputs</span>
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="p">),</span> <span class="s2">&quot;The length of the inputs and expected_outputs must be equal&quot;</span>
<a id="__codelineno-0-77" name="__codelineno-0-77"></a>
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="n">tags</span> <span class="o">=</span> <span class="n">tags</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tags</span><span class="p">,</span> <span class="nb">list</span><span class="p">)</span> <span class="k">else</span> <span class="p">[</span><span class="n">tags</span><span class="p">]</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">sum_ratio</span> <span class="o">=</span> <span class="n">train_split_ratio</span> <span class="o">+</span> <span class="n">test_split_ratio</span> <span class="o">+</span> <span class="n">validation_split_ratio</span>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="k">assert</span> <span class="n">sum_ratio</span> <span class="o">&gt;</span> <span class="mi">0</span><span class="p">,</span> <span class="s2">&quot;The sum of the split ratios must be a positive number&quot;</span>
<a id="__codelineno-0-82" name="__codelineno-0-82"></a>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">created</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">traces</span> <span class="o">=</span> <span class="p">[</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="n">cast</span><span class="p">(</span>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="n">Trace</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a> <span class="n">Trace</span><span class="p">(</span> <span class="c1"># avoid ValueError: &quot;Trace&quot; object has no field &quot;__orig_class__&quot;</span>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a> <span class="n">trace_id</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">uuid4</span><span class="p">()),</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="n">created</span><span class="o">=</span><span class="n">created</span><span class="p">,</span>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">original_execution_time_ms</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="n">logged_values</span><span class="o">=</span><span class="n">X</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span> <span class="k">else</span> <span class="p">{</span><span class="s2">&quot;input&quot;</span><span class="p">:</span> <span class="n">X</span><span class="p">},</span>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="n">models</span><span class="o">=</span><span class="p">[],</span>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="n">output</span><span class="o">=</span><span class="n">y</span><span class="p">,</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="n">feedback</span><span class="o">=</span><span class="n">y</span><span class="p">,</span>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="n">exception</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-96" name="__codelineno-0-96"></a> <span class="n">tags</span><span class="o">=</span><span class="p">[</span><span class="n">GROUND_TRUTH_TAG_NAME</span><span class="p">,</span> <span class="n">split_tag</span><span class="p">,</span> <span class="o">*</span><span class="n">tags</span><span class="p">],</span>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="p">),</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a> <span class="p">)</span>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="k">for</span> <span class="p">((</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">),</span> <span class="n">split_tag</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">split_tags</span><span class="p">)</span>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="p">]</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="n">get_context</span><span class="p">()</span><span class="o">.</span><span class="n">tracing_database</span><span class="o">.</span><span class="n">save_batch</span><span class="p">(</span><span class="n">traces</span><span class="p">)</span>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">train_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">test_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="n">validation_split_ratio</span> <span class="o">/=</span> <span class="n">sum_ratio</span>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a> <span class="n">values</span> <span class="o">=</span> <span class="nb">list</span><span class="p">(</span><span class="nb">zip</span><span class="p">(</span><span class="n">inputs</span><span class="p">,</span> <span class="n">expected_outputs</span><span class="p">))</span>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a> <span class="n">shuffle</span><span class="p">(</span><span class="n">values</span><span class="p">)</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">split_tags</span> <span class="o">=</span> <span class="p">(</span>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="p">[</span><span class="n">TRAIN_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">train_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="o">+</span> <span class="p">[</span><span class="n">TEST_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">test_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="o">+</span> <span class="p">[</span><span class="n">VALIDATION_SPLIT_TAG_NAME</span><span class="p">]</span> <span class="o">*</span> <span class="n">ceil</span><span class="p">(</span><span class="n">validation_split_ratio</span> <span class="o">*</span> <span class="nb">len</span><span class="p">(</span><span class="n">inputs</span><span class="p">))</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="p">)</span>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="n">shuffle</span><span class="p">(</span><span class="n">split_tags</span><span class="p">)</span>
<a id="__codelineno-0-96" name="__codelineno-0-96"></a>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="n">created</span> <span class="o">=</span> <span class="n">datetime</span><span class="o">.</span><span class="n">utcnow</span><span class="p">()</span><span class="o">.</span><span class="n">isoformat</span><span class="p">()</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a> <span class="n">traces</span> <span class="o">=</span> <span class="p">[</span>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="n">cast</span><span class="p">(</span>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="n">Trace</span><span class="p">[</span><span class="n">T</span><span class="p">],</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="n">Trace</span><span class="p">(</span> <span class="c1"># avoid ValueError: &quot;Trace&quot; object has no field &quot;__orig_class__&quot;</span>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="n">trace_id</span><span class="o">=</span><span class="nb">str</span><span class="p">(</span><span class="n">uuid4</span><span class="p">()),</span>
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="n">created</span><span class="o">=</span><span class="n">created</span><span class="p">,</span>
<a id="__codelineno-0-104" name="__codelineno-0-104"></a> <span class="n">original_execution_time_ms</span><span class="o">=</span><span class="mi">0</span><span class="p">,</span>
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="n">logged_values</span><span class="o">=</span><span class="n">X</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">X</span><span class="p">,</span> <span class="nb">dict</span><span class="p">)</span> <span class="k">else</span> <span class="p">{</span><span class="s2">&quot;input&quot;</span><span class="p">:</span> <span class="n">X</span><span class="p">},</span>
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="n">models</span><span class="o">=</span><span class="p">[],</span>
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="n">output</span><span class="o">=</span><span class="n">y</span><span class="p">,</span>
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="n">feedback</span><span class="o">=</span><span class="n">y</span><span class="p">,</span>
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="n">exception</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-110" name="__codelineno-0-110"></a> <span class="n">tags</span><span class="o">=</span><span class="p">[</span><span class="n">GROUND_TRUTH_TAG_NAME</span><span class="p">,</span> <span class="n">split_tag</span><span class="p">,</span> <span class="o">*</span><span class="n">tags</span><span class="p">],</span>
<a id="__codelineno-0-111" name="__codelineno-0-111"></a> <span class="p">),</span>
<a id="__codelineno-0-112" name="__codelineno-0-112"></a> <span class="p">)</span>
<a id="__codelineno-0-113" name="__codelineno-0-113"></a> <span class="k">for</span> <span class="p">((</span><span class="n">X</span><span class="p">,</span> <span class="n">y</span><span class="p">),</span> <span class="n">split_tag</span><span class="p">)</span> <span class="ow">in</span> <span class="nb">zip</span><span class="p">(</span><span class="n">values</span><span class="p">,</span> <span class="n">split_tags</span><span class="p">)</span>
<a id="__codelineno-0-114" name="__codelineno-0-114"></a> <span class="p">]</span>
<a id="__codelineno-0-115" name="__codelineno-0-115"></a>
<a id="__codelineno-0-116" name="__codelineno-0-116"></a> <span class="n">get_context</span><span class="p">()</span><span class="o">.</span><span class="n">tracing_database</span><span class="o">.</span><span class="n">save_batch</span><span class="p">(</span><span class="n">traces</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
</details>
</div>
@ -3581,7 +3741,7 @@ scikit-learn)</p></td>
<div class="doc doc-contents first">
<p>Return training samples.</p>
<p>Combines, filters, and returns datapoints that have been either added by
<p>Combines, filters, and returns data-points that have been either added by
<code>add_ground_truth</code> or were the result of a prediction after which their trace got
a feedback through the RESP API-s <code>/traces/{trace_id}/feedback</code> endpoint
(end-to-end feedback).</p>
@ -3704,7 +3864,7 @@ filtering.</p></td>
<a id="__codelineno-0-14" name="__codelineno-0-14"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">List</span><span class="p">[</span><span class="n">Trace</span><span class="p">]:</span>
<a id="__codelineno-0-15" name="__codelineno-0-15"></a> <span class="sd">&quot;&quot;&quot;Return training samples.</span>
<a id="__codelineno-0-16" name="__codelineno-0-16"></a>
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="sd"> Combines, filters, and returns datapoints that have been either added by</span>
<a id="__codelineno-0-17" name="__codelineno-0-17"></a><span class="sd"> Combines, filters, and returns data-points that have been either added by</span>
<a id="__codelineno-0-18" name="__codelineno-0-18"></a><span class="sd"> `add_ground_truth` or were the result of a prediction after which their trace got</span>
<a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="sd"> a feedback through the RESP API-s `/traces/{trace_id}/feedback` endpoint</span>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a><span class="sd"> (end-to-end feedback).</span>

10
reference/large-file/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

10
reference/utilities/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

10
reference/views/index.html
View file

@ -408,7 +408,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -422,7 +422,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -436,7 +436,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -449,8 +449,8 @@
<li class="md-nav__item">
<a href="../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

2
search/search_index.json
View file

File diff suppressed because one or more lines are too long

BIN
sitemap.xml.gz
View file

Binary file not shown.

10
tutorial/deploy/index.html
View file

@ -451,7 +451,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -465,7 +465,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -479,7 +479,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -492,8 +492,8 @@
<li class="md-nav__item">
<a href="../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>

12
tutorial/index.html
View file

@ -485,7 +485,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -499,7 +499,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -513,7 +513,7 @@
<li class="md-nav__item">
<a href="../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -526,8 +526,8 @@
<li class="md-nav__item">
<a href="../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>
@ -915,7 +915,7 @@
You can freely reference it knowing that the function is always provided with it.</li>
</ol>
<p>Finally, we test the model's inference function through the GreatAI dashboard. <a href="/how-to-guides/use-service">The only thing left is to deploy the hardened-service properly.</a></p>
<div style="display: flex; justify-content: center;">
<div style="display: flex; justify-content: space-evenly;">
<p><a class="md-button md-button--primary" href="/how-to-guides/create-service"><span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M18 22a2 2 0 0 0 2-2V4a2 2 0 0 0-2-2h-6v7L9.5 7.5 7 9V2H6a2 2 0 0 0-2 2v16a2 2 0 0 0 2 2h12Z"/></svg></span> Learn about all the features</a></p>
<p><a class="md-button md-button--secondary" href="/examples/simple/data"><span class="twemoji"><svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 24 24"><path d="M7 2v2h1v14a4 4 0 0 0 4 4 4 4 0 0 0 4-4V4h1V2H7m4 14c-.6 0-1-.4-1-1s.4-1 1-1 1 .4 1 1-.4 1-1 1m2-4c-.6 0-1-.4-1-1s.4-1 1-1 1 .4 1 1-.4 1-1 1m1-5h-4V4h4v3Z"/></svg></span> Look at more examples</a></p>
</div>

2
tutorial/index.md
View file

@ -72,7 +72,7 @@ def predict_domain(sentence, model):
Finally, we test the model's inference function through the GreatAI dashboard. [The only thing left is to deploy the hardened-service properly.](/how-to-guides/use-service)
<div style="display: flex; justify-content: center;" markdown>
<div style="display: flex; justify-content: space-evenly;" markdown>
[:material-book: Learn about all the features](/how-to-guides/create-service){ .md-button .md-button--primary }
[:material-test-tube: Look at more examples](/examples/simple/data){ .md-button .md-button--secondary }

10
tutorial/train/index.html
View file

@ -451,7 +451,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/use-service/" class="md-nav__link">
How to use a GreatAI service
How to deploy a GreatAI service
</a>
</li>
@ -465,7 +465,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/handle-training-data/" class="md-nav__link">
How to handle training data
How to manage training data
</a>
</li>
@ -479,7 +479,7 @@
<li class="md-nav__item">
<a href="../../how-to-guides/large_file/" class="md-nav__link">
How to use LargeFile
How to use LargeFile-s
</a>
</li>
@ -492,8 +492,8 @@
<li class="md-nav__item">
<a href="../../how-to-guides/call_remote.md" class="md-nav__link">
None
<a href="../../how-to-guides/call-remote/" class="md-nav__link">
Call remote GreatAI instances
</a>
</li>