andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Deployed 4996869 with MkDocs version: 1.3.1

Browse source
This commit is contained in:
2022-08-11 14:51:01 +00:00
parent c8060cc443
commit 9e9892e00c
5 changed files with 88 additions and 74 deletions
Download patch file Download diff file Expand all files Collapse all files

74
reference/index.html
View file

@ -5047,7 +5047,12 @@ not advised for production use.</p>
<span class="normal"><a href="#__codelineno-0-106">106</a></span>
<span class="normal"><a href="#__codelineno-0-107">107</a></span>
<span class="normal"><a href="#__codelineno-0-108">108</a></span>
<span class="normal"><a href="#__codelineno-0-109">109</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">class</span> <span class="nc">ParallelTinyDbDriver</span><span class="p">(</span><span class="n">TracingDatabaseDriver</span><span class="p">):</span>
<span class="normal"><a href="#__codelineno-0-109">109</a></span>
<span class="normal"><a href="#__codelineno-0-110">110</a></span>
<span class="normal"><a href="#__codelineno-0-111">111</a></span>
<span class="normal"><a href="#__codelineno-0-112">112</a></span>
<span class="normal"><a href="#__codelineno-0-113">113</a></span>
<span class="normal"><a href="#__codelineno-0-114">114</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-19" name="__codelineno-0-19"></a><span class="k">class</span> <span class="nc">ParallelTinyDbDriver</span><span class="p">(</span><span class="n">TracingDatabaseDriver</span><span class="p">):</span>
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="sd">&quot;&quot;&quot;TracingDatabaseDriver with TinyDB as a backend.</span>
<a id="__codelineno-0-21" name="__codelineno-0-21"></a>
<a id="__codelineno-0-22" name="__codelineno-0-22"></a><span class="sd"> Saves the database as a JSON into a single file. Highly inefficient on inserting,</span>
@ -5098,7 +5103,7 @@ not advised for production use.</p>
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">documents</span><span class="p">:</span>
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="k">return</span> <span class="p">[],</span> <span class="mi">0</span>
<a id="__codelineno-0-69" name="__codelineno-0-69"></a>
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">(</span><span class="n">documents</span><span class="p">)</span>
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">pd</span><span class="o">.</span><span class="n">DataFrame</span><span class="p">([</span><span class="n">Trace</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">d</span><span class="p">)</span><span class="o">.</span><span class="n">to_flat_dict</span><span class="p">()</span> <span class="k">for</span> <span class="n">d</span> <span class="ow">in</span> <span class="n">documents</span><span class="p">])</span>
<a id="__codelineno-0-71" name="__codelineno-0-71"></a>
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">conjunctive_filters</span><span class="p">:</span>
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="n">operator</span> <span class="o">=</span> <span class="n">f</span><span class="o">.</span><span class="n">operator</span><span class="o">.</span><span class="n">lower</span><span class="p">()</span>
@ -5107,37 +5112,42 @@ not advised for production use.</p>
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="nb">getattr</span><span class="p">(</span><span class="n">df</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">property</span><span class="p">],</span> <span class="n">operator_mapping</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">operator</span><span class="p">])(</span><span class="n">f</span><span class="o">.</span><span class="n">value</span><span class="p">)</span>
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="p">]</span>
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="k">elif</span> <span class="n">operator</span> <span class="o">==</span> <span class="s2">&quot;contains&quot;</span><span class="p">:</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span><span class="n">df</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">property</span><span class="p">]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="n">case</span><span class="o">=</span><span class="kc">False</span><span class="p">)]</span>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="k">if</span> <span class="n">sort_by</span><span class="p">:</span>
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">column_id</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sort_by</span><span class="p">],</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">ascending</span><span class="o">=</span><span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">direction</span> <span class="o">==</span> <span class="s2">&quot;asc&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sort_by</span><span class="p">],</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="p">)</span>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a> <span class="n">count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">skip</span><span class="p">:]</span> <span class="k">if</span> <span class="n">take</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">skip</span> <span class="p">:</span> <span class="n">skip</span> <span class="o">+</span> <span class="n">take</span><span class="p">]</span>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="k">return</span> <span class="p">[</span><span class="n">Trace</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">trace</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">iterrows</span><span class="p">()],</span> <span class="n">count</span>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">new_version</span><span class="p">:</span> <span class="n">Trace</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_safe_execute</span><span class="p">(</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="k">lambda</span> <span class="n">db</span><span class="p">:</span> <span class="n">db</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">new_version</span><span class="o">.</span><span class="n">dict</span><span class="p">(),</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">)</span>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="p">)</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">df</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">loc</span><span class="p">[</span>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">df</span><span class="p">[</span><span class="n">f</span><span class="o">.</span><span class="n">property</span><span class="p">]</span><span class="o">.</span><span class="n">str</span><span class="o">.</span><span class="n">contains</span><span class="p">(</span>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="nb">str</span><span class="p">(</span><span class="nb">int</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">value</span><span class="p">))</span> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">value</span><span class="p">,</span> <span class="nb">float</span><span class="p">)</span> <span class="k">else</span> <span class="n">f</span><span class="o">.</span><span class="n">value</span><span class="p">,</span>
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">case</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="p">)</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="p">]</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="k">if</span> <span class="n">sort_by</span><span class="p">:</span>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a> <span class="n">df</span><span class="o">.</span><span class="n">sort_values</span><span class="p">(</span>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a> <span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">column_id</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sort_by</span><span class="p">],</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="n">ascending</span><span class="o">=</span><span class="p">[</span><span class="n">col</span><span class="o">.</span><span class="n">direction</span> <span class="o">==</span> <span class="s2">&quot;asc&quot;</span> <span class="k">for</span> <span class="n">col</span> <span class="ow">in</span> <span class="n">sort_by</span><span class="p">],</span>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">inplace</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="p">)</span>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="n">count</span> <span class="o">=</span> <span class="nb">len</span><span class="p">(</span><span class="n">df</span><span class="p">)</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="n">result</span> <span class="o">=</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">skip</span><span class="p">:]</span> <span class="k">if</span> <span class="n">take</span> <span class="ow">is</span> <span class="kc">None</span> <span class="k">else</span> <span class="n">df</span><span class="o">.</span><span class="n">iloc</span><span class="p">[</span><span class="n">skip</span> <span class="p">:</span> <span class="n">skip</span> <span class="o">+</span> <span class="n">take</span><span class="p">]</span>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="k">return</span> <span class="p">[</span><span class="n">Trace</span><span class="o">.</span><span class="n">parse_obj</span><span class="p">(</span><span class="n">trace</span><span class="p">)</span> <span class="k">for</span> <span class="n">_</span><span class="p">,</span> <span class="n">trace</span> <span class="ow">in</span> <span class="n">result</span><span class="o">.</span><span class="n">iterrows</span><span class="p">()],</span> <span class="n">count</span>
<a id="__codelineno-0-96" name="__codelineno-0-96"></a>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_safe_execute</span><span class="p">(</span><span class="k">lambda</span> <span class="n">db</span><span class="p">:</span> <span class="n">db</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">))</span>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="k">def</span> <span class="nf">delete_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="k">with</span> <span class="n">lock</span><span class="p">:</span>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="k">with</span> <span class="n">TinyDB</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path_to_db</span><span class="p">)</span> <span class="k">as</span> <span class="n">db</span><span class="p">:</span>
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="k">for</span> <span class="nb">id</span> <span class="ow">in</span> <span class="n">ids</span><span class="p">:</span>
<a id="__codelineno-0-104" name="__codelineno-0-104"></a> <span class="n">db</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">)</span>
<a id="__codelineno-0-105" name="__codelineno-0-105"></a>
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="k">def</span> <span class="nf">_safe_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">TinyDB</span><span class="p">],</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="k">with</span> <span class="n">lock</span><span class="p">:</span>
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="k">with</span> <span class="n">TinyDB</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path_to_db</span><span class="p">)</span> <span class="k">as</span> <span class="n">db</span><span class="p">:</span>
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="n">db</span><span class="p">)</span>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="k">def</span> <span class="nf">update</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">id</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span> <span class="n">new_version</span><span class="p">:</span> <span class="n">Trace</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_safe_execute</span><span class="p">(</span>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="k">lambda</span> <span class="n">db</span><span class="p">:</span> <span class="n">db</span><span class="o">.</span><span class="n">update</span><span class="p">(</span><span class="n">new_version</span><span class="o">.</span><span class="n">dict</span><span class="p">(),</span> <span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">)</span>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="p">)</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="nb">id</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_safe_execute</span><span class="p">(</span><span class="k">lambda</span> <span class="n">db</span><span class="p">:</span> <span class="n">db</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">))</span>
<a id="__codelineno-0-104" name="__codelineno-0-104"></a>
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="k">def</span> <span class="nf">delete_batch</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">ids</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="nb">str</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="k">with</span> <span class="n">lock</span><span class="p">:</span>
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="k">with</span> <span class="n">TinyDB</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path_to_db</span><span class="p">)</span> <span class="k">as</span> <span class="n">db</span><span class="p">:</span>
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="k">for</span> <span class="nb">id</span> <span class="ow">in</span> <span class="n">ids</span><span class="p">:</span>
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="n">db</span><span class="o">.</span><span class="n">remove</span><span class="p">(</span><span class="k">lambda</span> <span class="n">d</span><span class="p">:</span> <span class="n">d</span><span class="p">[</span><span class="s2">&quot;trace_id&quot;</span><span class="p">]</span> <span class="o">==</span> <span class="nb">id</span><span class="p">)</span>
<a id="__codelineno-0-110" name="__codelineno-0-110"></a>
<a id="__codelineno-0-111" name="__codelineno-0-111"></a> <span class="k">def</span> <span class="nf">_safe_execute</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">TinyDB</span><span class="p">],</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-&gt;</span> <span class="n">Any</span><span class="p">:</span>
<a id="__codelineno-0-112" name="__codelineno-0-112"></a> <span class="k">with</span> <span class="n">lock</span><span class="p">:</span>
<a id="__codelineno-0-113" name="__codelineno-0-113"></a> <span class="k">with</span> <span class="n">TinyDB</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">path_to_db</span><span class="p">)</span> <span class="k">as</span> <span class="n">db</span><span class="p">:</span>
<a id="__codelineno-0-114" name="__codelineno-0-114"></a> <span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="n">db</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
</details>