Deployed 85d079f with MkDocs version: 1.3.1
This commit is contained in:
parent
95fdd502a9
commit
8a093ac9d2
7 changed files with 640 additions and 640 deletions
|
|
@ -2834,7 +2834,7 @@ decorating the same function with <code>use_model</code> multiple times.</p>
|
||||||
<tr>
|
<tr>
|
||||||
<td><code>version</code></td>
|
<td><code>version</code></td>
|
||||||
<td>
|
<td>
|
||||||
<code><span title="typing.Union">Union</span>[int, <span title="typing.Literal">Literal</span>['latest']]</code>
|
<code><span title="typing.Union">Union</span>[int, <span title="typing_extensions.Literal">Literal</span>[latest]]</code>
|
||||||
</td>
|
</td>
|
||||||
<td><p>The model's version as stored by the LargeFile implementation.</p></td>
|
<td><p>The model's version as stored by the LargeFile implementation.</p></td>
|
||||||
<td>
|
<td>
|
||||||
|
|
@ -2874,7 +2874,18 @@ decorating the same function with <code>use_model</code> multiple times.</p>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/models/use_model.py</code></summary>
|
<summary>Source code in <code>great_ai/models/use_model.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-18">18</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-19">19</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-20">20</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-21">21</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-22">22</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-23">23</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-24">24</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-25">25</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-26">26</a></span>
|
||||||
|
<span class="normal"><a href="#__codelineno-0-27">27</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
<span class="normal"><a href="#__codelineno-0-28">28</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
<span class="normal"><a href="#__codelineno-0-29">29</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
<span class="normal"><a href="#__codelineno-0-30">30</a></span>
|
||||||
|
|
@ -2921,76 +2932,65 @@ decorating the same function with <code>use_model</code> multiple times.</p>
|
||||||
<span class="normal"><a href="#__codelineno-0-71">71</a></span>
|
<span class="normal"><a href="#__codelineno-0-71">71</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-72">72</a></span>
|
<span class="normal"><a href="#__codelineno-0-72">72</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-73">73</a></span>
|
<span class="normal"><a href="#__codelineno-0-73">73</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-74">74</a></span>
|
<span class="normal"><a href="#__codelineno-0-74">74</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-16" name="__codelineno-0-16"></a><span class="k">def</span> <span class="nf">use_model</span><span class="p">(</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-75">75</a></span>
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-76">76</a></span>
|
<a id="__codelineno-0-18" name="__codelineno-0-18"></a> <span class="o">*</span><span class="p">,</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-77">77</a></span>
|
<a id="__codelineno-0-19" name="__codelineno-0-19"></a> <span class="n">version</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"latest"</span><span class="p">]]</span> <span class="o">=</span> <span class="s2">"latest"</span><span class="p">,</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-78">78</a></span>
|
<a id="__codelineno-0-20" name="__codelineno-0-20"></a> <span class="n">model_kwarg_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"model"</span><span class="p">,</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-79">79</a></span>
|
<a id="__codelineno-0-21" name="__codelineno-0-21"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">F</span><span class="p">],</span> <span class="n">F</span><span class="p">]:</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-80">80</a></span>
|
<a id="__codelineno-0-22" name="__codelineno-0-22"></a> <span class="sd">"""Inject a model into a function.</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-81">81</a></span>
|
<a id="__codelineno-0-23" name="__codelineno-0-23"></a>
|
||||||
<span class="normal"><a href="#__codelineno-0-82">82</a></span>
|
<a id="__codelineno-0-24" name="__codelineno-0-24"></a><span class="sd"> Load a model specified by `key` and `version` using the currently active `LargeFile`</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-83">83</a></span>
|
<a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="sd"> implementation. If it's a single object, it is deserialised using `dill`. If it's a</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-84">84</a></span>
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> directory of files, a `pathlib.Path` instance is given.</span>
|
||||||
<span class="normal"><a href="#__codelineno-0-85">85</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="k">def</span> <span class="nf">use_model</span><span class="p">(</span>
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
||||||
<a id="__codelineno-0-28" name="__codelineno-0-28"></a> <span class="n">key</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> By default, the function's `model` parameter is replaced by the loaded model. This</span>
|
||||||
<a id="__codelineno-0-29" name="__codelineno-0-29"></a> <span class="o">*</span><span class="p">,</span>
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> can be customised by changing `model_kwarg_name`. Multiple models can be loaded by</span>
|
||||||
<a id="__codelineno-0-30" name="__codelineno-0-30"></a> <span class="n">version</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="nb">int</span><span class="p">,</span> <span class="n">Literal</span><span class="p">[</span><span class="s2">"latest"</span><span class="p">]]</span> <span class="o">=</span> <span class="s2">"latest"</span><span class="p">,</span>
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> decorating the same function with `use_model` multiple times.</span>
|
||||||
<a id="__codelineno-0-31" name="__codelineno-0-31"></a> <span class="n">model_kwarg_name</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"model"</span><span class="p">,</span>
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a>
|
||||||
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Callable</span><span class="p">[[</span><span class="n">F</span><span class="p">],</span> <span class="n">F</span><span class="p">]:</span>
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> Examples:</span>
|
||||||
<a id="__codelineno-0-33" name="__codelineno-0-33"></a> <span class="sd">"""Inject a model into a function.</span>
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> >>> from great_ai import save_model</span>
|
||||||
<a id="__codelineno-0-34" name="__codelineno-0-34"></a>
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> >>> save_model(3, 'my_number')</span>
|
||||||
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> Load a model specified by `key` and `version` using the currently active `LargeFile`</span>
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> 'my_number:...'</span>
|
||||||
<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd"> implementation. If it's a single object, it is deserialised using `dill`. If it's a</span>
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a><span class="sd"> >>> @use_model('my_number')</span>
|
||||||
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> directory of files, a `pathlib.Path` instance is given.</span>
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> ... def my_function(a, model):</span>
|
||||||
<a id="__codelineno-0-38" name="__codelineno-0-38"></a>
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="sd"> ... return a + model</span>
|
||||||
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> By default, the function's `model` parameter is replaced by the loaded model. This</span>
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> >>> my_function(4)</span>
|
||||||
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> can be customised by changing `model_kwarg_name`. Multiple models can be loaded by</span>
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> 7</span>
|
||||||
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> decorating the same function with `use_model` multiple times.</span>
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a>
|
||||||
<a id="__codelineno-0-42" name="__codelineno-0-42"></a>
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> Examples:</span>
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> key: The model's name as stored by the LargeFile implementation.</span>
|
||||||
<a id="__codelineno-0-44" name="__codelineno-0-44"></a><span class="sd"> >>> from great_ai import save_model</span>
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a><span class="sd"> version: The model's version as stored by the LargeFile implementation.</span>
|
||||||
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> >>> save_model(3, 'my_number')</span>
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> model_kwarg_name: the parameter to use for injecting the loaded model</span>
|
||||||
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> 'my_number:...'</span>
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> Returns:</span>
|
||||||
<a id="__codelineno-0-47" name="__codelineno-0-47"></a><span class="sd"> >>> @use_model('my_number')</span>
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a><span class="sd"> A decorator for model injection.</span>
|
||||||
<a id="__codelineno-0-48" name="__codelineno-0-48"></a><span class="sd"> ... def my_function(a, model):</span>
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-49" name="__codelineno-0-49"></a><span class="sd"> ... return a + model</span>
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a>
|
||||||
<a id="__codelineno-0-50" name="__codelineno-0-50"></a><span class="sd"> >>> my_function(4)</span>
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="k">assert</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-51" name="__codelineno-0-51"></a><span class="sd"> 7</span>
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="nb">isinstance</span><span class="p">(</span><span class="n">version</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span> <span class="ow">or</span> <span class="n">version</span> <span class="o">==</span> <span class="s2">"latest"</span>
|
||||||
<a id="__codelineno-0-52" name="__codelineno-0-52"></a>
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="p">),</span> <span class="s2">"Only integers or the string literal `latest` is allowed as a version"</span>
|
||||||
<a id="__codelineno-0-53" name="__codelineno-0-53"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a>
|
||||||
<a id="__codelineno-0-54" name="__codelineno-0-54"></a><span class="sd"> key: The model's name as stored by the LargeFile implementation.</span>
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="n">model</span><span class="p">,</span> <span class="n">actual_version</span> <span class="o">=</span> <span class="n">_load_model</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-55" name="__codelineno-0-55"></a><span class="sd"> version: The model's version as stored by the LargeFile implementation.</span>
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a> <span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-56" name="__codelineno-0-56"></a><span class="sd"> model_kwarg_name: the parameter to use for injecting the loaded model</span>
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a> <span class="n">version</span><span class="o">=</span><span class="kc">None</span> <span class="k">if</span> <span class="n">version</span> <span class="o">==</span> <span class="s2">"latest"</span> <span class="k">else</span> <span class="n">version</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-57" name="__codelineno-0-57"></a><span class="sd"> Returns:</span>
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-58" name="__codelineno-0-58"></a><span class="sd"> A decorator for model injection.</span>
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a>
|
||||||
<a id="__codelineno-0-59" name="__codelineno-0-59"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="k">def</span> <span class="nf">decorator</span><span class="p">(</span><span class="n">func</span><span class="p">:</span> <span class="n">F</span><span class="p">)</span> <span class="o">-></span> <span class="n">F</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-60" name="__codelineno-0-60"></a>
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="n">assert_function_is_not_finalised</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="k">assert</span> <span class="p">(</span>
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a>
|
||||||
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="nb">isinstance</span><span class="p">(</span><span class="n">version</span><span class="p">,</span> <span class="nb">int</span><span class="p">)</span> <span class="ow">or</span> <span class="n">version</span> <span class="o">==</span> <span class="s2">"latest"</span>
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="n">store</span> <span class="o">=</span> <span class="n">get_function_metadata_store</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="p">),</span> <span class="s2">"Only integers or the string literal `latest` is allowed as a version"</span>
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="n">store</span><span class="o">.</span><span class="n">model_parameter_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">model_kwarg_name</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-64" name="__codelineno-0-64"></a>
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a>
|
||||||
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="n">model</span><span class="p">,</span> <span class="n">actual_version</span> <span class="o">=</span> <span class="n">_load_model</span><span class="p">(</span>
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="nd">@wraps</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span>
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="n">version</span><span class="o">=</span><span class="kc">None</span> <span class="k">if</span> <span class="n">version</span> <span class="o">==</span> <span class="s2">"latest"</span> <span class="k">else</span> <span class="n">version</span><span class="p">,</span>
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="n">tracing_context</span> <span class="o">=</span> <span class="n">TracingContext</span><span class="o">.</span><span class="n">get_current_tracing_context</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="p">)</span>
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="k">if</span> <span class="n">tracing_context</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-69" name="__codelineno-0-69"></a>
|
<a id="__codelineno-0-69" name="__codelineno-0-69"></a> <span class="n">tracing_context</span><span class="o">.</span><span class="n">log_model</span><span class="p">(</span><span class="n">Model</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="n">actual_version</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="k">def</span> <span class="nf">decorator</span><span class="p">(</span><span class="n">func</span><span class="p">:</span> <span class="n">F</span><span class="p">)</span> <span class="o">-></span> <span class="n">F</span><span class="p">:</span>
|
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span> <span class="o">**</span><span class="p">{</span><span class="n">model_kwarg_name</span><span class="p">:</span> <span class="n">model</span><span class="p">})</span>
|
||||||
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="n">assert_function_is_not_finalised</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
<a id="__codelineno-0-71" name="__codelineno-0-71"></a>
|
||||||
<a id="__codelineno-0-72" name="__codelineno-0-72"></a>
|
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">F</span><span class="p">,</span> <span class="n">wrapper</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="n">store</span> <span class="o">=</span> <span class="n">get_function_metadata_store</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
<a id="__codelineno-0-73" name="__codelineno-0-73"></a>
|
||||||
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="n">store</span><span class="o">.</span><span class="n">model_parameter_names</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">model_kwarg_name</span><span class="p">)</span>
|
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="k">return</span> <span class="n">decorator</span>
|
||||||
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
|
||||||
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="nd">@wraps</span><span class="p">(</span><span class="n">func</span><span class="p">)</span>
|
|
||||||
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="k">def</span> <span class="nf">wrapper</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">Any</span><span class="p">],</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="n">Dict</span><span class="p">[</span><span class="nb">str</span><span class="p">,</span> <span class="n">Any</span><span class="p">])</span> <span class="o">-></span> <span class="n">Any</span><span class="p">:</span>
|
|
||||||
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="n">tracing_context</span> <span class="o">=</span> <span class="n">TracingContext</span><span class="o">.</span><span class="n">get_current_tracing_context</span><span class="p">()</span>
|
|
||||||
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="k">if</span> <span class="n">tracing_context</span><span class="p">:</span>
|
|
||||||
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">tracing_context</span><span class="o">.</span><span class="n">log_model</span><span class="p">(</span><span class="n">Model</span><span class="p">(</span><span class="n">key</span><span class="o">=</span><span class="n">key</span><span class="p">,</span> <span class="n">version</span><span class="o">=</span><span class="n">actual_version</span><span class="p">))</span>
|
|
||||||
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="k">return</span> <span class="n">func</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">,</span> <span class="o">**</span><span class="p">{</span><span class="n">model_kwarg_name</span><span class="p">:</span> <span class="n">model</span><span class="p">})</span>
|
|
||||||
<a id="__codelineno-0-82" name="__codelineno-0-82"></a>
|
|
||||||
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="n">F</span><span class="p">,</span> <span class="n">wrapper</span><span class="p">)</span>
|
|
||||||
<a id="__codelineno-0-84" name="__codelineno-0-84"></a>
|
|
||||||
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="k">return</span> <span class="n">decorator</span>
|
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1866,9 +1866,7 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-23"> 23</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-25"> 25</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-24"> 24</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-25"> 25</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-26"> 26</a></span>
|
<span class="normal"><a href="#__codelineno-0-26"> 26</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-27"> 27</a></span>
|
<span class="normal"><a href="#__codelineno-0-27"> 27</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-28"> 28</a></span>
|
<span class="normal"><a href="#__codelineno-0-28"> 28</a></span>
|
||||||
|
|
@ -2184,325 +2182,327 @@ automatic cache-pruning.</p></td>
|
||||||
<span class="normal"><a href="#__codelineno-0-338">338</a></span>
|
<span class="normal"><a href="#__codelineno-0-338">338</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-339">339</a></span>
|
<span class="normal"><a href="#__codelineno-0-339">339</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-340">340</a></span>
|
<span class="normal"><a href="#__codelineno-0-340">340</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-341">341</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-23" name="__codelineno-0-23"></a><span class="k">class</span> <span class="nc">LargeFileBase</span><span class="p">(</span><span class="n">ABC</span><span class="p">):</span>
|
<span class="normal"><a href="#__codelineno-0-341">341</a></span>
|
||||||
<a id="__codelineno-0-24" name="__codelineno-0-24"></a> <span class="sd">"""Base for LargeFile implementations with different backends.</span>
|
<span class="normal"><a href="#__codelineno-0-342">342</a></span>
|
||||||
<a id="__codelineno-0-25" name="__codelineno-0-25"></a>
|
<span class="normal"><a href="#__codelineno-0-343">343</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-25" name="__codelineno-0-25"></a><span class="k">class</span> <span class="nc">LargeFileBase</span><span class="p">(</span><span class="n">ABC</span><span class="p">):</span>
|
||||||
<a id="__codelineno-0-26" name="__codelineno-0-26"></a><span class="sd"> Store large files remotely using the familiar API of `open()`. With built-in</span>
|
<a id="__codelineno-0-26" name="__codelineno-0-26"></a> <span class="sd">"""Base for LargeFile implementations with different backends.</span>
|
||||||
<a id="__codelineno-0-27" name="__codelineno-0-27"></a><span class="sd"> versioning, pruning and local cache.</span>
|
<a id="__codelineno-0-27" name="__codelineno-0-27"></a>
|
||||||
<a id="__codelineno-0-28" name="__codelineno-0-28"></a>
|
<a id="__codelineno-0-28" name="__codelineno-0-28"></a><span class="sd"> Store large files remotely using the familiar API of `open()`. With built-in</span>
|
||||||
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> By default, files are stored in the ".cache" folder and the least recently used is</span>
|
<a id="__codelineno-0-29" name="__codelineno-0-29"></a><span class="sd"> versioning, pruning and local cache.</span>
|
||||||
<a id="__codelineno-0-30" name="__codelineno-0-30"></a><span class="sd"> deleted after the overall size reaches 30 GBs.</span>
|
<a id="__codelineno-0-30" name="__codelineno-0-30"></a>
|
||||||
<a id="__codelineno-0-31" name="__codelineno-0-31"></a>
|
<a id="__codelineno-0-31" name="__codelineno-0-31"></a><span class="sd"> By default, files are stored in the ".cache" folder and the least recently used is</span>
|
||||||
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> Examples:</span>
|
<a id="__codelineno-0-32" name="__codelineno-0-32"></a><span class="sd"> deleted after the overall size reaches 30 GBs.</span>
|
||||||
<a id="__codelineno-0-33" name="__codelineno-0-33"></a><span class="sd"> >>> LargeFileBase.cache_path = Path(".cache")</span>
|
<a id="__codelineno-0-33" name="__codelineno-0-33"></a>
|
||||||
<a id="__codelineno-0-34" name="__codelineno-0-34"></a>
|
<a id="__codelineno-0-34" name="__codelineno-0-34"></a><span class="sd"> Examples:</span>
|
||||||
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> >>> LargeFileBase.max_cache_size = "30GB"</span>
|
<a id="__codelineno-0-35" name="__codelineno-0-35"></a><span class="sd"> >>> LargeFileBase.cache_path = Path(".cache")</span>
|
||||||
<a id="__codelineno-0-36" name="__codelineno-0-36"></a>
|
<a id="__codelineno-0-36" name="__codelineno-0-36"></a>
|
||||||
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> Attributes:</span>
|
<a id="__codelineno-0-37" name="__codelineno-0-37"></a><span class="sd"> >>> LargeFileBase.max_cache_size = "30GB"</span>
|
||||||
<a id="__codelineno-0-38" name="__codelineno-0-38"></a><span class="sd"> initialized: Tell whether `configure_credentials` or</span>
|
<a id="__codelineno-0-38" name="__codelineno-0-38"></a>
|
||||||
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> `configure_credentials_from_file` has been already called.</span>
|
<a id="__codelineno-0-39" name="__codelineno-0-39"></a><span class="sd"> Attributes:</span>
|
||||||
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> cache_path: Storage location for cached files.</span>
|
<a id="__codelineno-0-40" name="__codelineno-0-40"></a><span class="sd"> initialized: Tell whether `configure_credentials` or</span>
|
||||||
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> max_cache_size: Delete files until the folder at `cache_path` is smaller than</span>
|
<a id="__codelineno-0-41" name="__codelineno-0-41"></a><span class="sd"> `configure_credentials_from_file` has been already called.</span>
|
||||||
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> this value. Examples: "5 GB", "10MB", "0.3 TB". Set to `None` for no</span>
|
<a id="__codelineno-0-42" name="__codelineno-0-42"></a><span class="sd"> cache_path: Storage location for cached files.</span>
|
||||||
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> automatic cache-pruning.</span>
|
<a id="__codelineno-0-43" name="__codelineno-0-43"></a><span class="sd"> max_cache_size: Delete files until the folder at `cache_path` is smaller than</span>
|
||||||
<a id="__codelineno-0-44" name="__codelineno-0-44"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-44" name="__codelineno-0-44"></a><span class="sd"> this value. Examples: "5 GB", "10MB", "0.3 TB". Set to `None` for no</span>
|
||||||
<a id="__codelineno-0-45" name="__codelineno-0-45"></a>
|
<a id="__codelineno-0-45" name="__codelineno-0-45"></a><span class="sd"> automatic cache-pruning.</span>
|
||||||
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="n">initialized</span> <span class="o">=</span> <span class="kc">False</span>
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-47" name="__codelineno-0-47"></a> <span class="n">cache_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">".cache"</span><span class="p">)</span>
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a>
|
||||||
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="n">max_cache_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"30GB"</span>
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="n">initialized</span> <span class="o">=</span> <span class="kc">False</span>
|
||||||
<a id="__codelineno-0-49" name="__codelineno-0-49"></a>
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">cache_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="s2">".cache"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="n">max_cache_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="s2">"30GB"</span>
|
||||||
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="bp">self</span><span class="p">,</span>
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a>
|
||||||
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="k">def</span> <span class="fm">__init__</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"r"</span><span class="p">,</span>
|
<a id="__codelineno-0-53" name="__codelineno-0-53"></a> <span class="bp">self</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="o">*</span><span class="p">,</span>
|
<a id="__codelineno-0-54" name="__codelineno-0-54"></a> <span class="n">name</span><span class="p">:</span> <span class="nb">str</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-55" name="__codelineno-0-55"></a> <span class="n">buffering</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
|
<a id="__codelineno-0-55" name="__codelineno-0-55"></a> <span class="n">mode</span><span class="p">:</span> <span class="nb">str</span> <span class="o">=</span> <span class="s2">"r"</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-56" name="__codelineno-0-56"></a> <span class="n">encoding</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-56" name="__codelineno-0-56"></a> <span class="o">*</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="n">errors</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-57" name="__codelineno-0-57"></a> <span class="n">buffering</span><span class="p">:</span> <span class="nb">int</span> <span class="o">=</span> <span class="o">-</span><span class="mi">1</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-58" name="__codelineno-0-58"></a> <span class="n">newline</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-58" name="__codelineno-0-58"></a> <span class="n">encoding</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="n">version</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-59" name="__codelineno-0-59"></a> <span class="n">errors</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="n">keep_last_n</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-60" name="__codelineno-0-60"></a> <span class="n">newline</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">str</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="n">cache_only_mode</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
<a id="__codelineno-0-61" name="__codelineno-0-61"></a> <span class="n">version</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="p">):</span>
|
<a id="__codelineno-0-62" name="__codelineno-0-62"></a> <span class="n">keep_last_n</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
|
<a id="__codelineno-0-63" name="__codelineno-0-63"></a> <span class="n">cache_only_mode</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="n">version</span>
|
<a id="__codelineno-0-64" name="__codelineno-0-64"></a> <span class="p">):</span>
|
||||||
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span> <span class="o">=</span> <span class="n">mode</span>
|
<a id="__codelineno-0-65" name="__codelineno-0-65"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span> <span class="o">=</span> <span class="n">name</span>
|
||||||
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="n">keep_last_n</span>
|
<a id="__codelineno-0-66" name="__codelineno-0-66"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="n">version</span>
|
||||||
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_cache_only_mode</span> <span class="o">=</span> <span class="n">cache_only_mode</span>
|
<a id="__codelineno-0-67" name="__codelineno-0-67"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span> <span class="o">=</span> <span class="n">mode</span>
|
||||||
<a id="__codelineno-0-68" name="__codelineno-0-68"></a>
|
<a id="__codelineno-0-68" name="__codelineno-0-68"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="n">keep_last_n</span>
|
||||||
<a id="__codelineno-0-69" name="__codelineno-0-69"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span> <span class="o">=</span> <span class="n">buffering</span>
|
<a id="__codelineno-0-69" name="__codelineno-0-69"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_cache_only_mode</span> <span class="o">=</span> <span class="n">cache_only_mode</span>
|
||||||
<a id="__codelineno-0-70" name="__codelineno-0-70"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span> <span class="o">=</span> <span class="n">encoding</span>
|
<a id="__codelineno-0-70" name="__codelineno-0-70"></a>
|
||||||
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_errors</span> <span class="o">=</span> <span class="n">errors</span>
|
<a id="__codelineno-0-71" name="__codelineno-0-71"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span> <span class="o">=</span> <span class="n">buffering</span>
|
||||||
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_newline</span> <span class="o">=</span> <span class="n">newline</span>
|
<a id="__codelineno-0-72" name="__codelineno-0-72"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span> <span class="o">=</span> <span class="n">encoding</span>
|
||||||
<a id="__codelineno-0-73" name="__codelineno-0-73"></a>
|
<a id="__codelineno-0-73" name="__codelineno-0-73"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_errors</span> <span class="o">=</span> <span class="n">errors</span>
|
||||||
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="n">LargeFileBase</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
<a id="__codelineno-0-74" name="__codelineno-0-74"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_newline</span> <span class="o">=</span> <span class="n">newline</span>
|
||||||
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
<a id="__codelineno-0-75" name="__codelineno-0-75"></a>
|
||||||
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_find_instances</span><span class="p">()</span>
|
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="n">LargeFileBase</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_check_mode_and_set_version</span><span class="p">()</span>
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a>
|
||||||
<a id="__codelineno-0-78" name="__codelineno-0-78"></a>
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_find_instances</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="nd">@classmethod</span>
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_check_mode_and_set_version</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="k">def</span> <span class="nf">configure_credentials_from_file</span><span class="p">(</span>
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a>
|
||||||
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="bp">cls</span><span class="p">,</span>
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="nd">@classmethod</span>
|
||||||
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">secrets</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">],</span>
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="k">def</span> <span class="nf">configure_credentials_from_file</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="bp">cls</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="sd">"""Load file and feed its content to `configure_credentials`.</span>
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">secrets</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">],</span>
|
||||||
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd"> Extra keys are ignored.</span>
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="sd">"""Load file and feed its content to `configure_credentials`.</span>
|
||||||
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a>
|
||||||
<a id="__codelineno-0-88" name="__codelineno-0-88"></a>
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> Extra keys are ignored.</span>
|
||||||
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">secrets</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">):</span>
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">secrets</span> <span class="o">=</span> <span class="n">ConfigFile</span><span class="p">(</span><span class="n">secrets</span><span class="p">)</span>
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a>
|
||||||
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">(</span><span class="o">**</span><span class="p">{</span><span class="n">k</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">secrets</span><span class="o">.</span><span class="n">items</span><span class="p">()})</span>
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">secrets</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">):</span>
|
||||||
<a id="__codelineno-0-92" name="__codelineno-0-92"></a>
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="n">secrets</span> <span class="o">=</span> <span class="n">ConfigFile</span><span class="p">(</span><span class="n">secrets</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="nd">@classmethod</span>
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">(</span><span class="o">**</span><span class="p">{</span><span class="n">k</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">secrets</span><span class="o">.</span><span class="n">items</span><span class="p">()})</span>
|
||||||
<a id="__codelineno-0-94" name="__codelineno-0-94"></a> <span class="k">def</span> <span class="nf">configure_credentials</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a>
|
||||||
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="sd">"""Configure required credentials for the LargeFile backend."""</span>
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="nd">@classmethod</span>
|
||||||
<a id="__codelineno-0-96" name="__codelineno-0-96"></a>
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a> <span class="k">def</span> <span class="nf">configure_credentials</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">initialized</span> <span class="o">=</span> <span class="kc">True</span>
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="sd">"""Configure required credentials for the LargeFile backend."""</span>
|
||||||
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
||||||
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">IO</span><span class="p">:</span>
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">initialized</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
<a id="__codelineno-0-100" name="__codelineno-0-100"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="p">:</span> <span class="n">IO</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a>
|
||||||
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="n">tempfile</span><span class="o">.</span><span class="n">NamedTemporaryFile</span><span class="p">(</span>
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a> <span class="k">def</span> <span class="fm">__enter__</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">IO</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="n">mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">,</span>
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="p">:</span> <span class="n">IO</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="n">buffering</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span><span class="p">,</span>
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a> <span class="n">tempfile</span><span class="o">.</span><span class="n">NamedTemporaryFile</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-104" name="__codelineno-0-104"></a> <span class="n">encoding</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span><span class="p">,</span>
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a> <span class="n">mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="n">newline</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_newline</span><span class="p">,</span>
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a> <span class="n">buffering</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="n">errors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_errors</span><span class="p">,</span>
|
<a id="__codelineno-0-106" name="__codelineno-0-106"></a> <span class="n">encoding</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="n">delete</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
<a id="__codelineno-0-107" name="__codelineno-0-107"></a> <span class="n">newline</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_newline</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="n">prefix</span><span class="o">=</span><span class="s2">"large_file-"</span><span class="p">,</span>
|
<a id="__codelineno-0-108" name="__codelineno-0-108"></a> <span class="n">errors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_errors</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="p">)</span>
|
<a id="__codelineno-0-109" name="__codelineno-0-109"></a> <span class="n">delete</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-110" name="__codelineno-0-110"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span>
|
<a id="__codelineno-0-110" name="__codelineno-0-110"></a> <span class="n">prefix</span><span class="o">=</span><span class="s2">"large_file-"</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-111" name="__codelineno-0-111"></a> <span class="k">else</span> <span class="nb">open</span><span class="p">(</span>
|
<a id="__codelineno-0-111" name="__codelineno-0-111"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-112" name="__codelineno-0-112"></a> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(),</span>
|
<a id="__codelineno-0-112" name="__codelineno-0-112"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span>
|
||||||
<a id="__codelineno-0-113" name="__codelineno-0-113"></a> <span class="n">mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">,</span>
|
<a id="__codelineno-0-113" name="__codelineno-0-113"></a> <span class="k">else</span> <span class="nb">open</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-114" name="__codelineno-0-114"></a> <span class="n">buffering</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span><span class="p">,</span>
|
<a id="__codelineno-0-114" name="__codelineno-0-114"></a> <span class="bp">self</span><span class="o">.</span><span class="n">get</span><span class="p">(),</span>
|
||||||
<a id="__codelineno-0-115" name="__codelineno-0-115"></a> <span class="n">encoding</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span><span class="p">,</span>
|
<a id="__codelineno-0-115" name="__codelineno-0-115"></a> <span class="n">mode</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-116" name="__codelineno-0-116"></a> <span class="n">newline</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_newline</span><span class="p">,</span>
|
<a id="__codelineno-0-116" name="__codelineno-0-116"></a> <span class="n">buffering</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_buffering</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-117" name="__codelineno-0-117"></a> <span class="n">errors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_errors</span><span class="p">,</span>
|
<a id="__codelineno-0-117" name="__codelineno-0-117"></a> <span class="n">encoding</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_encoding</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-118" name="__codelineno-0-118"></a> <span class="p">)</span>
|
<a id="__codelineno-0-118" name="__codelineno-0-118"></a> <span class="n">newline</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_newline</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-119" name="__codelineno-0-119"></a> <span class="p">)</span>
|
<a id="__codelineno-0-119" name="__codelineno-0-119"></a> <span class="n">errors</span><span class="o">=</span><span class="bp">self</span><span class="o">.</span><span class="n">_errors</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-120" name="__codelineno-0-120"></a>
|
<a id="__codelineno-0-120" name="__codelineno-0-120"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-121" name="__codelineno-0-121"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span>
|
<a id="__codelineno-0-121" name="__codelineno-0-121"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-122" name="__codelineno-0-122"></a>
|
<a id="__codelineno-0-122" name="__codelineno-0-122"></a>
|
||||||
<a id="__codelineno-0-123" name="__codelineno-0-123"></a> <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span>
|
<a id="__codelineno-0-123" name="__codelineno-0-123"></a> <span class="k">return</span> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span>
|
||||||
<a id="__codelineno-0-124" name="__codelineno-0-124"></a> <span class="bp">self</span><span class="p">,</span>
|
<a id="__codelineno-0-124" name="__codelineno-0-124"></a>
|
||||||
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">]],</span>
|
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="k">def</span> <span class="fm">__exit__</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">exc</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">],</span>
|
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="bp">self</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">traceback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TracebackType</span><span class="p">],</span>
|
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="nb">type</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Type</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">]],</span>
|
||||||
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">Literal</span><span class="p">[</span><span class="kc">False</span><span class="p">]:</span>
|
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">exc</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="ne">BaseException</span><span class="p">],</span>
|
||||||
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">traceback</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">TracebackType</span><span class="p">],</span>
|
||||||
<a id="__codelineno-0-130" name="__codelineno-0-130"></a>
|
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="p">)</span> <span class="o">-></span> <span class="n">Literal</span><span class="p">[</span><span class="kc">False</span><span class="p">]:</span>
|
||||||
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="k">if</span> <span class="nb">type</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">close</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
<a id="__codelineno-0-132" name="__codelineno-0-132"></a>
|
||||||
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="bp">self</span><span class="o">.</span><span class="n">push</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
|
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="k">if</span> <span class="nb">type</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="bp">self</span><span class="o">.</span><span class="n">push</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">name</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Could not finish operation."</span><span class="p">)</span>
|
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_file</span><span class="o">.</span><span class="n">name</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-137" name="__codelineno-0-137"></a>
|
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="k">return</span> <span class="kc">False</span>
|
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">logger</span><span class="o">.</span><span class="n">exception</span><span class="p">(</span><span class="s2">"Could not finish operation."</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-139" name="__codelineno-0-139"></a>
|
<a id="__codelineno-0-139" name="__codelineno-0-139"></a>
|
||||||
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="nd">@property</span>
|
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="k">return</span> <span class="kc">False</span>
|
||||||
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="k">def</span> <span class="nf">version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
<a id="__codelineno-0-141" name="__codelineno-0-141"></a>
|
||||||
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="sd">"""Numeric version of the file proxied by this LargeFile instance."""</span>
|
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="nd">@property</span>
|
||||||
<a id="__codelineno-0-143" name="__codelineno-0-143"></a>
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="k">def</span> <span class="nf">version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="sd">"""Numeric version of the file proxied by this LargeFile instance."""</span>
|
||||||
<a id="__codelineno-0-145" name="__codelineno-0-145"></a>
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a>
|
||||||
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="nd">@lru_cache</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
<a id="__codelineno-0-147" name="__codelineno-0-147"></a>
|
||||||
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="sd">"""Return path to the proxy of a file (or directory).</span>
|
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="nd">@lru_cache</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-149" name="__codelineno-0-149"></a>
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd"> If not available in the local cache, an attempt is made to download it.</span>
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="sd">"""Return path to the proxy of a file (or directory).</span>
|
||||||
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
|
||||||
<a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd"> If not available in the local cache, an attempt is made to download it.</span>
|
||||||
<a id="__codelineno-0-153" name="__codelineno-0-153"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a>
|
||||||
<a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-155" name="__codelineno-0-155"></a>
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
||||||
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">remote_path</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">i</span><span class="o">.</span><span class="n">remote_path</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">if</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span>
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a>
|
||||||
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="p">)</span>
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">remote_path</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-159" name="__codelineno-0-159"></a>
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">i</span><span class="o">.</span><span class="n">remote_path</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">if</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span>
|
||||||
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">destination</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span>
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">destination</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
<a id="__codelineno-0-161" name="__codelineno-0-161"></a>
|
||||||
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> does not exist locally"</span><span class="p">)</span>
|
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">destination</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span>
|
||||||
<a id="__codelineno-0-163" name="__codelineno-0-163"></a>
|
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">destination</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||||
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> does not exist locally"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">local_root_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
|
<a id="__codelineno-0-165" name="__codelineno-0-165"></a>
|
||||||
<a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="n">tmp_file_archive</span> <span class="o">=</span> <span class="p">(</span>
|
<a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="n">local_root_path</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="n">local_root_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="p">)</span>
|
<a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="n">tmp_file_archive</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_download</span><span class="p">(</span>
|
<a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="n">local_root_path</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-170" name="__codelineno-0-170"></a> <span class="n">remote_path</span><span class="p">,</span> <span class="n">tmp_file_archive</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span>
|
<a id="__codelineno-0-170" name="__codelineno-0-170"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-171" name="__codelineno-0-171"></a> <span class="p">)</span>
|
<a id="__codelineno-0-171" name="__codelineno-0-171"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_download</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-172" name="__codelineno-0-172"></a>
|
<a id="__codelineno-0-172" name="__codelineno-0-172"></a> <span class="n">remote_path</span><span class="p">,</span> <span class="n">tmp_file_archive</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span>
|
||||||
<a id="__codelineno-0-173" name="__codelineno-0-173"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Decompressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-173" name="__codelineno-0-173"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-174" name="__codelineno-0-174"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">unpack_archive</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">tmp_file_archive</span><span class="p">),</span> <span class="n">tmp</span><span class="p">,</span> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">)</span>
|
<a id="__codelineno-0-174" name="__codelineno-0-174"></a>
|
||||||
<a id="__codelineno-0-175" name="__codelineno-0-175"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">local_root_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">destination</span><span class="p">))</span>
|
<a id="__codelineno-0-175" name="__codelineno-0-175"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Decompressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-176" name="__codelineno-0-176"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-176" name="__codelineno-0-176"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">unpack_archive</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">tmp_file_archive</span><span class="p">),</span> <span class="n">tmp</span><span class="p">,</span> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-177" name="__codelineno-0-177"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> found in cache"</span><span class="p">)</span>
|
<a id="__codelineno-0-177" name="__codelineno-0-177"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">local_root_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">destination</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-178" name="__codelineno-0-178"></a>
|
<a id="__codelineno-0-178" name="__codelineno-0-178"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-179" name="__codelineno-0-179"></a> <span class="k">return</span> <span class="n">destination</span>
|
<a id="__codelineno-0-179" name="__codelineno-0-179"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> found in cache"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-180" name="__codelineno-0-180"></a>
|
<a id="__codelineno-0-180" name="__codelineno-0-180"></a>
|
||||||
<a id="__codelineno-0-181" name="__codelineno-0-181"></a> <span class="k">def</span> <span class="nf">push</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-181" name="__codelineno-0-181"></a> <span class="k">return</span> <span class="n">destination</span>
|
||||||
<a id="__codelineno-0-182" name="__codelineno-0-182"></a> <span class="sd">"""Upload a file (or directory) as a new version of `key`.</span>
|
<a id="__codelineno-0-182" name="__codelineno-0-182"></a>
|
||||||
<a id="__codelineno-0-183" name="__codelineno-0-183"></a>
|
<a id="__codelineno-0-183" name="__codelineno-0-183"></a> <span class="k">def</span> <span class="nf">push</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-184" name="__codelineno-0-184"></a><span class="sd"> The file/directory is compressed before upload.</span>
|
<a id="__codelineno-0-184" name="__codelineno-0-184"></a> <span class="sd">"""Upload a file (or directory) as a new version of `key`.</span>
|
||||||
<a id="__codelineno-0-185" name="__codelineno-0-185"></a>
|
<a id="__codelineno-0-185" name="__codelineno-0-185"></a>
|
||||||
<a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd"> The file/directory is compressed before upload.</span>
|
||||||
<a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
<a id="__codelineno-0-187" name="__codelineno-0-187"></a>
|
||||||
<a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-189" name="__codelineno-0-189"></a>
|
<a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
||||||
<a id="__codelineno-0-190" name="__codelineno-0-190"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
<a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-191" name="__codelineno-0-191"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
<a id="__codelineno-0-191" name="__codelineno-0-191"></a>
|
||||||
<a id="__codelineno-0-192" name="__codelineno-0-192"></a>
|
<a id="__codelineno-0-192" name="__codelineno-0-192"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
<a id="__codelineno-0-193" name="__codelineno-0-193"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
<a id="__codelineno-0-193" name="__codelineno-0-193"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-194" name="__codelineno-0-194"></a> <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
|
<a id="__codelineno-0-194" name="__codelineno-0-194"></a>
|
||||||
<a id="__codelineno-0-195" name="__codelineno-0-195"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying file for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-195" name="__codelineno-0-195"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-196" name="__codelineno-0-196"></a> <span class="n">copy</span><span class="p">:</span> <span class="n">Any</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span>
|
<a id="__codelineno-0-196" name="__codelineno-0-196"></a> <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
|
||||||
<a id="__codelineno-0-197" name="__codelineno-0-197"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-197" name="__codelineno-0-197"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying file for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-198" name="__codelineno-0-198"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying directory for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-198" name="__codelineno-0-198"></a> <span class="n">copy</span><span class="p">:</span> <span class="n">Any</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span>
|
||||||
<a id="__codelineno-0-199" name="__codelineno-0-199"></a> <span class="n">copy</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copytree</span>
|
<a id="__codelineno-0-199" name="__codelineno-0-199"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-200" name="__codelineno-0-200"></a>
|
<a id="__codelineno-0-200" name="__codelineno-0-200"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying directory for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-201" name="__codelineno-0-201"></a> <span class="k">try</span><span class="p">:</span>
|
<a id="__codelineno-0-201" name="__codelineno-0-201"></a> <span class="n">copy</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copytree</span>
|
||||||
<a id="__codelineno-0-202" name="__codelineno-0-202"></a> <span class="c1"># Make local copy in the cache</span>
|
<a id="__codelineno-0-202" name="__codelineno-0-202"></a>
|
||||||
<a id="__codelineno-0-203" name="__codelineno-0-203"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">,</span> <span class="n">ignore_errors</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
<a id="__codelineno-0-203" name="__codelineno-0-203"></a> <span class="k">try</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-204" name="__codelineno-0-204"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
<a id="__codelineno-0-204" name="__codelineno-0-204"></a> <span class="c1"># Make local copy in the cache</span>
|
||||||
<a id="__codelineno-0-205" name="__codelineno-0-205"></a> <span class="k">except</span> <span class="n">shutil</span><span class="o">.</span><span class="n">SameFileError</span><span class="p">:</span>
|
<a id="__codelineno-0-205" name="__codelineno-0-205"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">,</span> <span class="n">ignore_errors</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-206" name="__codelineno-0-206"></a> <span class="k">pass</span> <span class="c1"># No worries</span>
|
<a id="__codelineno-0-206" name="__codelineno-0-206"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-207" name="__codelineno-0-207"></a>
|
<a id="__codelineno-0-207" name="__codelineno-0-207"></a> <span class="k">except</span> <span class="n">shutil</span><span class="o">.</span><span class="n">SameFileError</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-208" name="__codelineno-0-208"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
<a id="__codelineno-0-208" name="__codelineno-0-208"></a> <span class="k">pass</span> <span class="c1"># No worries</span>
|
||||||
<a id="__codelineno-0-209" name="__codelineno-0-209"></a>
|
<a id="__codelineno-0-209" name="__codelineno-0-209"></a>
|
||||||
<a id="__codelineno-0-210" name="__codelineno-0-210"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp2</span><span class="p">:</span>
|
<a id="__codelineno-0-210" name="__codelineno-0-210"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-211" name="__codelineno-0-211"></a> <span class="c1"># A directory has to be zipped and it cannot contain the output of the zipping</span>
|
<a id="__codelineno-0-211" name="__codelineno-0-211"></a>
|
||||||
<a id="__codelineno-0-212" name="__codelineno-0-212"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Compressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-212" name="__codelineno-0-212"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp2</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-213" name="__codelineno-0-213"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">make_archive</span><span class="p">(</span>
|
<a id="__codelineno-0-213" name="__codelineno-0-213"></a> <span class="c1"># A directory has to be zipped and it cannot contain the output of the zipping</span>
|
||||||
<a id="__codelineno-0-214" name="__codelineno-0-214"></a> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span>
|
<a id="__codelineno-0-214" name="__codelineno-0-214"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Compressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-215" name="__codelineno-0-215"></a> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">,</span>
|
<a id="__codelineno-0-215" name="__codelineno-0-215"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">make_archive</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-216" name="__codelineno-0-216"></a> <span class="n">tmp</span><span class="p">,</span>
|
<a id="__codelineno-0-216" name="__codelineno-0-216"></a> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span>
|
||||||
<a id="__codelineno-0-217" name="__codelineno-0-217"></a> <span class="p">)</span>
|
<a id="__codelineno-0-217" name="__codelineno-0-217"></a> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-218" name="__codelineno-0-218"></a>
|
<a id="__codelineno-0-218" name="__codelineno-0-218"></a> <span class="n">tmp</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-219" name="__codelineno-0-219"></a> <span class="n">file_to_be_uploaded</span> <span class="o">=</span> <span class="p">(</span>
|
<a id="__codelineno-0-219" name="__codelineno-0-219"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-220" name="__codelineno-0-220"></a> <span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-220" name="__codelineno-0-220"></a>
|
||||||
<a id="__codelineno-0-221" name="__codelineno-0-221"></a> <span class="p">)</span>
|
<a id="__codelineno-0-221" name="__codelineno-0-221"></a> <span class="n">file_to_be_uploaded</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-222" name="__codelineno-0-222"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_upload</span><span class="p">(</span><span class="n">file_to_be_uploaded</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span><span class="p">)</span>
|
<a id="__codelineno-0-222" name="__codelineno-0-222"></a> <span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-223" name="__codelineno-0-223"></a>
|
<a id="__codelineno-0-223" name="__codelineno-0-223"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-224" name="__codelineno-0-224"></a> <span class="bp">self</span><span class="o">.</span><span class="n">clean_up</span><span class="p">()</span>
|
<a id="__codelineno-0-224" name="__codelineno-0-224"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_upload</span><span class="p">(</span><span class="n">file_to_be_uploaded</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-225" name="__codelineno-0-225"></a>
|
<a id="__codelineno-0-225" name="__codelineno-0-225"></a>
|
||||||
<a id="__codelineno-0-226" name="__codelineno-0-226"></a> <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-226" name="__codelineno-0-226"></a> <span class="bp">self</span><span class="o">.</span><span class="n">clean_up</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-227" name="__codelineno-0-227"></a> <span class="sd">"""Delete all versions of the files under this `key`."""</span>
|
<a id="__codelineno-0-227" name="__codelineno-0-227"></a>
|
||||||
<a id="__codelineno-0-228" name="__codelineno-0-228"></a>
|
<a id="__codelineno-0-228" name="__codelineno-0-228"></a> <span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-229" name="__codelineno-0-229"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="mi">0</span>
|
<a id="__codelineno-0-229" name="__codelineno-0-229"></a> <span class="sd">"""Delete all versions of the files under this `key`."""</span>
|
||||||
<a id="__codelineno-0-230" name="__codelineno-0-230"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
<a id="__codelineno-0-230" name="__codelineno-0-230"></a>
|
||||||
<a id="__codelineno-0-231" name="__codelineno-0-231"></a>
|
<a id="__codelineno-0-231" name="__codelineno-0-231"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
<a id="__codelineno-0-232" name="__codelineno-0-232"></a> <span class="nd">@property</span>
|
<a id="__codelineno-0-232" name="__codelineno-0-232"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-233" name="__codelineno-0-233"></a> <span class="k">def</span> <span class="nf">versions_pretty</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
<a id="__codelineno-0-233" name="__codelineno-0-233"></a>
|
||||||
<a id="__codelineno-0-234" name="__codelineno-0-234"></a> <span class="sd">"""Formatted string of all available versions."""</span>
|
<a id="__codelineno-0-234" name="__codelineno-0-234"></a> <span class="nd">@property</span>
|
||||||
<a id="__codelineno-0-235" name="__codelineno-0-235"></a> <span class="k">return</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">))</span>
|
<a id="__codelineno-0-235" name="__codelineno-0-235"></a> <span class="k">def</span> <span class="nf">versions_pretty</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-236" name="__codelineno-0-236"></a>
|
<a id="__codelineno-0-236" name="__codelineno-0-236"></a> <span class="sd">"""Formatted string of all available versions."""</span>
|
||||||
<a id="__codelineno-0-237" name="__codelineno-0-237"></a> <span class="k">def</span> <span class="nf">clean_up</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-237" name="__codelineno-0-237"></a> <span class="k">return</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-238" name="__codelineno-0-238"></a> <span class="sd">"""Delete local and remote versions according to currently set cache and retention policy."""</span>
|
<a id="__codelineno-0-238" name="__codelineno-0-238"></a>
|
||||||
<a id="__codelineno-0-239" name="__codelineno-0-239"></a>
|
<a id="__codelineno-0-239" name="__codelineno-0-239"></a> <span class="k">def</span> <span class="nf">clean_up</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-240" name="__codelineno-0-240"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
<a id="__codelineno-0-240" name="__codelineno-0-240"></a> <span class="sd">"""Delete local and remote versions according to currently set cache and retention policy."""</span>
|
||||||
<a id="__codelineno-0-241" name="__codelineno-0-241"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prune_cache</span><span class="p">()</span>
|
<a id="__codelineno-0-241" name="__codelineno-0-241"></a>
|
||||||
<a id="__codelineno-0-242" name="__codelineno-0-242"></a>
|
<a id="__codelineno-0-242" name="__codelineno-0-242"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-243" name="__codelineno-0-243"></a> <span class="nd">@property</span>
|
<a id="__codelineno-0-243" name="__codelineno-0-243"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prune_cache</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-244" name="__codelineno-0-244"></a> <span class="k">def</span> <span class="nf">_local_name</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
<a id="__codelineno-0-244" name="__codelineno-0-244"></a>
|
||||||
<a id="__codelineno-0-245" name="__codelineno-0-245"></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}{</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="si">}{</span><span class="bp">self</span><span class="o">.</span><span class="n">version</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-245" name="__codelineno-0-245"></a> <span class="nd">@property</span>
|
||||||
<a id="__codelineno-0-246" name="__codelineno-0-246"></a>
|
<a id="__codelineno-0-246" name="__codelineno-0-246"></a> <span class="k">def</span> <span class="nf">_local_name</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-247" name="__codelineno-0-247"></a> <span class="k">def</span> <span class="nf">_find_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-247" name="__codelineno-0-247"></a> <span class="k">return</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}{</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="si">}{</span><span class="bp">self</span><span class="o">.</span><span class="n">version</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-248" name="__codelineno-0-248"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cache_only_mode</span><span class="p">:</span>
|
<a id="__codelineno-0-248" name="__codelineno-0-248"></a>
|
||||||
<a id="__codelineno-0-249" name="__codelineno-0-249"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_find_instances_from_cache</span><span class="p">()</span>
|
<a id="__codelineno-0-249" name="__codelineno-0-249"></a> <span class="k">def</span> <span class="nf">_find_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-250" name="__codelineno-0-250"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-250" name="__codelineno-0-250"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_cache_only_mode</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-251" name="__codelineno-0-251"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_find_remote_instances</span><span class="p">()</span>
|
<a id="__codelineno-0-251" name="__codelineno-0-251"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_find_instances_from_cache</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-252" name="__codelineno-0-252"></a>
|
<a id="__codelineno-0-252" name="__codelineno-0-252"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-253" name="__codelineno-0-253"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span>
|
<a id="__codelineno-0-253" name="__codelineno-0-253"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_find_remote_instances</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-254" name="__codelineno-0-254"></a>
|
<a id="__codelineno-0-254" name="__codelineno-0-254"></a>
|
||||||
<a id="__codelineno-0-255" name="__codelineno-0-255"></a> <span class="k">def</span> <span class="nf">_find_instances_from_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">DataInstance</span><span class="p">]:</span>
|
<a id="__codelineno-0-255" name="__codelineno-0-255"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">,</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">i</span><span class="p">:</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-256" name="__codelineno-0-256"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fetching cached versions of </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-256" name="__codelineno-0-256"></a>
|
||||||
<a id="__codelineno-0-257" name="__codelineno-0-257"></a>
|
<a id="__codelineno-0-257" name="__codelineno-0-257"></a> <span class="k">def</span> <span class="nf">_find_instances_from_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">DataInstance</span><span class="p">]:</span>
|
||||||
<a id="__codelineno-0-258" name="__codelineno-0-258"></a> <span class="n">candidates</span> <span class="o">=</span> <span class="p">[</span>
|
<a id="__codelineno-0-258" name="__codelineno-0-258"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Fetching cached versions of </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-259" name="__codelineno-0-259"></a> <span class="n">DataInstance</span><span class="p">(</span>
|
<a id="__codelineno-0-259" name="__codelineno-0-259"></a>
|
||||||
<a id="__codelineno-0-260" name="__codelineno-0-260"></a> <span class="n">name</span><span class="o">=</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
<a id="__codelineno-0-260" name="__codelineno-0-260"></a> <span class="n">candidates</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
<a id="__codelineno-0-261" name="__codelineno-0-261"></a> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
<a id="__codelineno-0-261" name="__codelineno-0-261"></a> <span class="n">DataInstance</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-262" name="__codelineno-0-262"></a> <span class="p">),</span>
|
<a id="__codelineno-0-262" name="__codelineno-0-262"></a> <span class="n">name</span><span class="o">=</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="o">.</span><span class="n">join</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-263" name="__codelineno-0-263"></a> <span class="n">version</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span>
|
<a id="__codelineno-0-263" name="__codelineno-0-263"></a> <span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="p">)[:</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
|
||||||
<a id="__codelineno-0-264" name="__codelineno-0-264"></a> <span class="n">remote_path</span><span class="o">=</span><span class="n">f</span><span class="p">,</span>
|
<a id="__codelineno-0-264" name="__codelineno-0-264"></a> <span class="p">),</span>
|
||||||
<a id="__codelineno-0-265" name="__codelineno-0-265"></a> <span class="p">)</span>
|
<a id="__codelineno-0-265" name="__codelineno-0-265"></a> <span class="n">version</span><span class="o">=</span><span class="nb">int</span><span class="p">(</span><span class="n">f</span><span class="o">.</span><span class="n">name</span><span class="o">.</span><span class="n">split</span><span class="p">(</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="p">)[</span><span class="o">-</span><span class="mi">1</span><span class="p">]),</span>
|
||||||
<a id="__codelineno-0-266" name="__codelineno-0-266"></a> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span>
|
<a id="__codelineno-0-266" name="__codelineno-0-266"></a> <span class="n">remote_path</span><span class="o">=</span><span class="n">f</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-267" name="__codelineno-0-267"></a> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}{</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="si">}</span><span class="s2">*"</span>
|
<a id="__codelineno-0-267" name="__codelineno-0-267"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-268" name="__codelineno-0-268"></a> <span class="p">)</span>
|
<a id="__codelineno-0-268" name="__codelineno-0-268"></a> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-269" name="__codelineno-0-269"></a> <span class="p">]</span>
|
<a id="__codelineno-0-269" name="__codelineno-0-269"></a> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}{</span><span class="n">CACHE_NAME_VERSION_SEPARATOR</span><span class="si">}</span><span class="s2">*"</span>
|
||||||
<a id="__codelineno-0-270" name="__codelineno-0-270"></a>
|
<a id="__codelineno-0-270" name="__codelineno-0-270"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-271" name="__codelineno-0-271"></a> <span class="k">return</span> <span class="p">[</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">candidates</span> <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span>
|
<a id="__codelineno-0-271" name="__codelineno-0-271"></a> <span class="p">]</span>
|
||||||
<a id="__codelineno-0-272" name="__codelineno-0-272"></a>
|
<a id="__codelineno-0-272" name="__codelineno-0-272"></a>
|
||||||
<a id="__codelineno-0-273" name="__codelineno-0-273"></a> <span class="k">def</span> <span class="nf">_check_mode_and_set_version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-273" name="__codelineno-0-273"></a> <span class="k">return</span> <span class="p">[</span><span class="n">c</span> <span class="k">for</span> <span class="n">c</span> <span class="ow">in</span> <span class="n">candidates</span> <span class="k">if</span> <span class="n">c</span><span class="o">.</span><span class="n">name</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="p">]</span>
|
||||||
<a id="__codelineno-0-274" name="__codelineno-0-274"></a> <span class="k">if</span> <span class="s2">"+"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
<a id="__codelineno-0-274" name="__codelineno-0-274"></a>
|
||||||
<a id="__codelineno-0-275" name="__codelineno-0-275"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
<a id="__codelineno-0-275" name="__codelineno-0-275"></a> <span class="k">def</span> <span class="nf">_check_mode_and_set_version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-276" name="__codelineno-0-276"></a> <span class="sa">f</span><span class="s2">"File mode `</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="si">}</span><span class="s2">` is not allowed3, remove the `+`."</span>
|
<a id="__codelineno-0-276" name="__codelineno-0-276"></a> <span class="k">if</span> <span class="s2">"+"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-277" name="__codelineno-0-277"></a> <span class="p">)</span>
|
<a id="__codelineno-0-277" name="__codelineno-0-277"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-278" name="__codelineno-0-278"></a>
|
<a id="__codelineno-0-278" name="__codelineno-0-278"></a> <span class="sa">f</span><span class="s2">"File mode `</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="si">}</span><span class="s2">` is not allowed3, remove the `+`."</span>
|
||||||
<a id="__codelineno-0-279" name="__codelineno-0-279"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
<a id="__codelineno-0-279" name="__codelineno-0-279"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-280" name="__codelineno-0-280"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-280" name="__codelineno-0-280"></a>
|
||||||
<a id="__codelineno-0-281" name="__codelineno-0-281"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Providing a version is not allowed in write mode."</span><span class="p">)</span>
|
<a id="__codelineno-0-281" name="__codelineno-0-281"></a> <span class="k">if</span> <span class="s2">"w"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-282" name="__codelineno-0-282"></a>
|
<a id="__codelineno-0-282" name="__codelineno-0-282"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-283" name="__codelineno-0-283"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">version</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">else</span> <span class="mi">0</span>
|
<a id="__codelineno-0-283" name="__codelineno-0-283"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Providing a version is not allowed in write mode."</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-284" name="__codelineno-0-284"></a>
|
<a id="__codelineno-0-284" name="__codelineno-0-284"></a>
|
||||||
<a id="__codelineno-0-285" name="__codelineno-0-285"></a> <span class="k">elif</span> <span class="s2">"r"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
<a id="__codelineno-0-285" name="__codelineno-0-285"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">version</span> <span class="o">+</span> <span class="mi">1</span> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">else</span> <span class="mi">0</span>
|
||||||
<a id="__codelineno-0-286" name="__codelineno-0-286"></a> <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span>
|
<a id="__codelineno-0-286" name="__codelineno-0-286"></a>
|
||||||
<a id="__codelineno-0-287" name="__codelineno-0-287"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span>
|
<a id="__codelineno-0-287" name="__codelineno-0-287"></a> <span class="k">elif</span> <span class="s2">"r"</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_mode</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-288" name="__codelineno-0-288"></a> <span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> not found. No versions are available."</span>
|
<a id="__codelineno-0-288" name="__codelineno-0-288"></a> <span class="k">if</span> <span class="ow">not</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-289" name="__codelineno-0-289"></a> <span class="p">)</span>
|
<a id="__codelineno-0-289" name="__codelineno-0-289"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-290" name="__codelineno-0-290"></a>
|
<a id="__codelineno-0-290" name="__codelineno-0-290"></a> <span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> not found. No versions are available."</span>
|
||||||
<a id="__codelineno-0-291" name="__codelineno-0-291"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-291" name="__codelineno-0-291"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-292" name="__codelineno-0-292"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">version</span>
|
<a id="__codelineno-0-292" name="__codelineno-0-292"></a>
|
||||||
<a id="__codelineno-0-293" name="__codelineno-0-293"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
<a id="__codelineno-0-293" name="__codelineno-0-293"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-294" name="__codelineno-0-294"></a> <span class="sa">f</span><span class="s2">"Latest version of </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> is </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="si">}</span><span class="s2"> "</span>
|
<a id="__codelineno-0-294" name="__codelineno-0-294"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span><span class="o">.</span><span class="n">version</span>
|
||||||
<a id="__codelineno-0-295" name="__codelineno-0-295"></a> <span class="o">+</span> <span class="sa">f</span><span class="s2">"(from versions: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">versions_pretty</span><span class="si">}</span><span class="s2">)"</span>
|
<a id="__codelineno-0-295" name="__codelineno-0-295"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-296" name="__codelineno-0-296"></a> <span class="p">)</span>
|
<a id="__codelineno-0-296" name="__codelineno-0-296"></a> <span class="sa">f</span><span class="s2">"Latest version of </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> is </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="si">}</span><span class="s2"> "</span>
|
||||||
<a id="__codelineno-0-297" name="__codelineno-0-297"></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">]:</span>
|
<a id="__codelineno-0-297" name="__codelineno-0-297"></a> <span class="o">+</span> <span class="sa">f</span><span class="s2">"(from versions: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">versions_pretty</span><span class="si">}</span><span class="s2">)"</span>
|
||||||
<a id="__codelineno-0-298" name="__codelineno-0-298"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span>
|
<a id="__codelineno-0-298" name="__codelineno-0-298"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-299" name="__codelineno-0-299"></a> <span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> not found with version </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="si">}</span><span class="s2">. "</span>
|
<a id="__codelineno-0-299" name="__codelineno-0-299"></a> <span class="k">elif</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span> <span class="ow">not</span> <span class="ow">in</span> <span class="p">[</span><span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">]:</span>
|
||||||
<a id="__codelineno-0-300" name="__codelineno-0-300"></a> <span class="o">+</span> <span class="sa">f</span><span class="s2">"(from versions: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">versions_pretty</span><span class="si">}</span><span class="s2">)"</span>
|
<a id="__codelineno-0-300" name="__codelineno-0-300"></a> <span class="k">raise</span> <span class="ne">FileNotFoundError</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-301" name="__codelineno-0-301"></a> <span class="p">)</span>
|
<a id="__codelineno-0-301" name="__codelineno-0-301"></a> <span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_name</span><span class="si">}</span><span class="s2"> not found with version </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="si">}</span><span class="s2">. "</span>
|
||||||
<a id="__codelineno-0-302" name="__codelineno-0-302"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-302" name="__codelineno-0-302"></a> <span class="o">+</span> <span class="sa">f</span><span class="s2">"(from versions: </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">versions_pretty</span><span class="si">}</span><span class="s2">)"</span>
|
||||||
<a id="__codelineno-0-303" name="__codelineno-0-303"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unsupported file mode."</span><span class="p">)</span>
|
<a id="__codelineno-0-303" name="__codelineno-0-303"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-304" name="__codelineno-0-304"></a>
|
<a id="__codelineno-0-304" name="__codelineno-0-304"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-305" name="__codelineno-0-305"></a> <span class="k">def</span> <span class="nf">_prune_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-305" name="__codelineno-0-305"></a> <span class="k">raise</span> <span class="ne">ValueError</span><span class="p">(</span><span class="s2">"Unsupported file mode."</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-306" name="__codelineno-0-306"></a> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
<a id="__codelineno-0-306" name="__codelineno-0-306"></a>
|
||||||
<a id="__codelineno-0-307" name="__codelineno-0-307"></a>
|
<a id="__codelineno-0-307" name="__codelineno-0-307"></a> <span class="k">def</span> <span class="nf">_prune_cache</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-308" name="__codelineno-0-308"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-308" name="__codelineno-0-308"></a> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">mkdir</span><span class="p">(</span><span class="n">parents</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span> <span class="n">exist_ok</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-309" name="__codelineno-0-309"></a> <span class="k">return</span>
|
<a id="__codelineno-0-309" name="__codelineno-0-309"></a>
|
||||||
<a id="__codelineno-0-310" name="__codelineno-0-310"></a>
|
<a id="__codelineno-0-310" name="__codelineno-0-310"></a> <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-311" name="__codelineno-0-311"></a> <span class="n">allowed_size</span> <span class="o">=</span> <span class="n">human_readable_to_byte</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span><span class="p">)</span>
|
<a id="__codelineno-0-311" name="__codelineno-0-311"></a> <span class="k">return</span>
|
||||||
<a id="__codelineno-0-312" name="__codelineno-0-312"></a> <span class="k">assert</span> <span class="n">allowed_size</span> <span class="o">>=</span> <span class="mi">0</span>
|
<a id="__codelineno-0-312" name="__codelineno-0-312"></a>
|
||||||
<a id="__codelineno-0-313" name="__codelineno-0-313"></a>
|
<a id="__codelineno-0-313" name="__codelineno-0-313"></a> <span class="n">allowed_size</span> <span class="o">=</span> <span class="n">human_readable_to_byte</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-314" name="__codelineno-0-314"></a> <span class="n">least_recently_read</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
|
<a id="__codelineno-0-314" name="__codelineno-0-314"></a> <span class="k">assert</span> <span class="n">allowed_size</span> <span class="o">>=</span> <span class="mi">0</span>
|
||||||
<a id="__codelineno-0-315" name="__codelineno-0-315"></a> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="s2">"*"</span><span class="p">)],</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">f</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">st_atime</span>
|
<a id="__codelineno-0-315" name="__codelineno-0-315"></a>
|
||||||
<a id="__codelineno-0-316" name="__codelineno-0-316"></a> <span class="p">)</span>
|
<a id="__codelineno-0-316" name="__codelineno-0-316"></a> <span class="n">least_recently_read</span> <span class="o">=</span> <span class="nb">sorted</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-317" name="__codelineno-0-317"></a>
|
<a id="__codelineno-0-317" name="__codelineno-0-317"></a> <span class="p">[</span><span class="n">f</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span><span class="o">.</span><span class="n">glob</span><span class="p">(</span><span class="s2">"*"</span><span class="p">)],</span> <span class="n">key</span><span class="o">=</span><span class="k">lambda</span> <span class="n">f</span><span class="p">:</span> <span class="n">f</span><span class="o">.</span><span class="n">stat</span><span class="p">()</span><span class="o">.</span><span class="n">st_atime</span>
|
||||||
<a id="__codelineno-0-318" name="__codelineno-0-318"></a> <span class="k">while</span> <span class="nb">sum</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">least_recently_read</span><span class="p">)</span> <span class="o">></span> <span class="n">allowed_size</span><span class="p">:</span>
|
<a id="__codelineno-0-318" name="__codelineno-0-318"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-319" name="__codelineno-0-319"></a> <span class="n">file</span> <span class="o">=</span> <span class="n">least_recently_read</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
<a id="__codelineno-0-319" name="__codelineno-0-319"></a>
|
||||||
<a id="__codelineno-0-320" name="__codelineno-0-320"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
<a id="__codelineno-0-320" name="__codelineno-0-320"></a> <span class="k">while</span> <span class="nb">sum</span><span class="p">(</span><span class="n">os</span><span class="o">.</span><span class="n">path</span><span class="o">.</span><span class="n">getsize</span><span class="p">(</span><span class="n">f</span><span class="p">)</span> <span class="k">for</span> <span class="n">f</span> <span class="ow">in</span> <span class="n">least_recently_read</span><span class="p">)</span> <span class="o">></span> <span class="n">allowed_size</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-321" name="__codelineno-0-321"></a> <span class="sa">f</span><span class="s2">"Deleting file from cache to meet quota (max_cache_size=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">file</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-321" name="__codelineno-0-321"></a> <span class="n">file</span> <span class="o">=</span> <span class="n">least_recently_read</span><span class="o">.</span><span class="n">pop</span><span class="p">(</span><span class="mi">0</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-322" name="__codelineno-0-322"></a> <span class="p">)</span>
|
<a id="__codelineno-0-322" name="__codelineno-0-322"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-323" name="__codelineno-0-323"></a> <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
|
<a id="__codelineno-0-323" name="__codelineno-0-323"></a> <span class="sa">f</span><span class="s2">"Deleting file from cache to meet quota (max_cache_size=</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">max_cache_size</span><span class="si">}</span><span class="s2">): </span><span class="si">{</span><span class="n">file</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-324" name="__codelineno-0-324"></a>
|
<a id="__codelineno-0-324" name="__codelineno-0-324"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-325" name="__codelineno-0-325"></a> <span class="nd">@abstractmethod</span>
|
<a id="__codelineno-0-325" name="__codelineno-0-325"></a> <span class="n">os</span><span class="o">.</span><span class="n">unlink</span><span class="p">(</span><span class="n">file</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-326" name="__codelineno-0-326"></a> <span class="k">def</span> <span class="nf">_find_remote_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">DataInstance</span><span class="p">]:</span>
|
<a id="__codelineno-0-326" name="__codelineno-0-326"></a>
|
||||||
<a id="__codelineno-0-327" name="__codelineno-0-327"></a> <span class="k">pass</span>
|
<a id="__codelineno-0-327" name="__codelineno-0-327"></a> <span class="nd">@abstractmethod</span>
|
||||||
<a id="__codelineno-0-328" name="__codelineno-0-328"></a>
|
<a id="__codelineno-0-328" name="__codelineno-0-328"></a> <span class="k">def</span> <span class="nf">_find_remote_instances</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="n">List</span><span class="p">[</span><span class="n">DataInstance</span><span class="p">]:</span>
|
||||||
<a id="__codelineno-0-329" name="__codelineno-0-329"></a> <span class="nd">@abstractmethod</span>
|
<a id="__codelineno-0-329" name="__codelineno-0-329"></a> <span class="k">pass</span>
|
||||||
<a id="__codelineno-0-330" name="__codelineno-0-330"></a> <span class="k">def</span> <span class="nf">_download</span><span class="p">(</span>
|
<a id="__codelineno-0-330" name="__codelineno-0-330"></a>
|
||||||
<a id="__codelineno-0-331" name="__codelineno-0-331"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="n">local_path</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span>
|
<a id="__codelineno-0-331" name="__codelineno-0-331"></a> <span class="nd">@abstractmethod</span>
|
||||||
<a id="__codelineno-0-332" name="__codelineno-0-332"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-332" name="__codelineno-0-332"></a> <span class="k">def</span> <span class="nf">_download</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-333" name="__codelineno-0-333"></a> <span class="k">pass</span>
|
<a id="__codelineno-0-333" name="__codelineno-0-333"></a> <span class="bp">self</span><span class="p">,</span> <span class="n">remote_path</span><span class="p">:</span> <span class="n">Any</span><span class="p">,</span> <span class="n">local_path</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span>
|
||||||
<a id="__codelineno-0-334" name="__codelineno-0-334"></a>
|
<a id="__codelineno-0-334" name="__codelineno-0-334"></a> <span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-335" name="__codelineno-0-335"></a> <span class="nd">@abstractmethod</span>
|
<a id="__codelineno-0-335" name="__codelineno-0-335"></a> <span class="k">pass</span>
|
||||||
<a id="__codelineno-0-336" name="__codelineno-0-336"></a> <span class="k">def</span> <span class="nf">_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-336" name="__codelineno-0-336"></a>
|
||||||
<a id="__codelineno-0-337" name="__codelineno-0-337"></a> <span class="k">pass</span>
|
<a id="__codelineno-0-337" name="__codelineno-0-337"></a> <span class="nd">@abstractmethod</span>
|
||||||
<a id="__codelineno-0-338" name="__codelineno-0-338"></a>
|
<a id="__codelineno-0-338" name="__codelineno-0-338"></a> <span class="k">def</span> <span class="nf">_upload</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">local_path</span><span class="p">:</span> <span class="n">Path</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-339" name="__codelineno-0-339"></a> <span class="nd">@abstractmethod</span>
|
<a id="__codelineno-0-339" name="__codelineno-0-339"></a> <span class="k">pass</span>
|
||||||
<a id="__codelineno-0-340" name="__codelineno-0-340"></a> <span class="k">def</span> <span class="nf">_delete_old_remote_versions</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-340" name="__codelineno-0-340"></a>
|
||||||
<a id="__codelineno-0-341" name="__codelineno-0-341"></a> <span class="k">pass</span>
|
<a id="__codelineno-0-341" name="__codelineno-0-341"></a> <span class="nd">@abstractmethod</span>
|
||||||
|
<a id="__codelineno-0-342" name="__codelineno-0-342"></a> <span class="k">def</span> <span class="nf">_delete_old_remote_versions</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
|
<a id="__codelineno-0-343" name="__codelineno-0-343"></a> <span class="k">pass</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
|
|
@ -2534,15 +2534,15 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-237">237</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-239">239</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-238">238</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-239">239</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-240">240</a></span>
|
<span class="normal"><a href="#__codelineno-0-240">240</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-241">241</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-237" name="__codelineno-0-237"></a><span class="k">def</span> <span class="nf">clean_up</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-241">241</a></span>
|
||||||
<a id="__codelineno-0-238" name="__codelineno-0-238"></a> <span class="sd">"""Delete local and remote versions according to currently set cache and retention policy."""</span>
|
<span class="normal"><a href="#__codelineno-0-242">242</a></span>
|
||||||
<a id="__codelineno-0-239" name="__codelineno-0-239"></a>
|
<span class="normal"><a href="#__codelineno-0-243">243</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-239" name="__codelineno-0-239"></a><span class="k">def</span> <span class="nf">clean_up</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-240" name="__codelineno-0-240"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
<a id="__codelineno-0-240" name="__codelineno-0-240"></a> <span class="sd">"""Delete local and remote versions according to currently set cache and retention policy."""</span>
|
||||||
<a id="__codelineno-0-241" name="__codelineno-0-241"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prune_cache</span><span class="p">()</span>
|
<a id="__codelineno-0-241" name="__codelineno-0-241"></a>
|
||||||
|
<a id="__codelineno-0-242" name="__codelineno-0-242"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
||||||
|
<a id="__codelineno-0-243" name="__codelineno-0-243"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_prune_cache</span><span class="p">()</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2569,15 +2569,15 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-93">93</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-95">95</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-94">94</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-95">95</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-96">96</a></span>
|
<span class="normal"><a href="#__codelineno-0-96">96</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-97">97</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="nd">@classmethod</span>
|
<span class="normal"><a href="#__codelineno-0-97">97</a></span>
|
||||||
<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="k">def</span> <span class="nf">configure_credentials</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-98">98</a></span>
|
||||||
<a id="__codelineno-0-95" name="__codelineno-0-95"></a> <span class="sd">"""Configure required credentials for the LargeFile backend."""</span>
|
<span class="normal"><a href="#__codelineno-0-99">99</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="nd">@classmethod</span>
|
||||||
<a id="__codelineno-0-96" name="__codelineno-0-96"></a>
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="k">def</span> <span class="nf">configure_credentials</span><span class="p">(</span><span class="bp">cls</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">:</span> <span class="nb">str</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">initialized</span> <span class="o">=</span> <span class="kc">True</span>
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a> <span class="sd">"""Configure required credentials for the LargeFile backend."""</span>
|
||||||
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
||||||
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">initialized</span> <span class="o">=</span> <span class="kc">True</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2605,9 +2605,7 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-79">79</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-81">81</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-80">80</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-81">81</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-82">82</a></span>
|
<span class="normal"><a href="#__codelineno-0-82">82</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-83">83</a></span>
|
<span class="normal"><a href="#__codelineno-0-83">83</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-84">84</a></span>
|
<span class="normal"><a href="#__codelineno-0-84">84</a></span>
|
||||||
|
|
@ -2617,19 +2615,21 @@ automatic cache-pruning.</p></td>
|
||||||
<span class="normal"><a href="#__codelineno-0-88">88</a></span>
|
<span class="normal"><a href="#__codelineno-0-88">88</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-89">89</a></span>
|
<span class="normal"><a href="#__codelineno-0-89">89</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-90">90</a></span>
|
<span class="normal"><a href="#__codelineno-0-90">90</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-91">91</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-79" name="__codelineno-0-79"></a><span class="nd">@classmethod</span>
|
<span class="normal"><a href="#__codelineno-0-91">91</a></span>
|
||||||
<a id="__codelineno-0-80" name="__codelineno-0-80"></a><span class="k">def</span> <span class="nf">configure_credentials_from_file</span><span class="p">(</span>
|
<span class="normal"><a href="#__codelineno-0-92">92</a></span>
|
||||||
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="bp">cls</span><span class="p">,</span>
|
<span class="normal"><a href="#__codelineno-0-93">93</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-81" name="__codelineno-0-81"></a><span class="nd">@classmethod</span>
|
||||||
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">secrets</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">],</span>
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a><span class="k">def</span> <span class="nf">configure_credentials_from_file</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="bp">cls</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="sd">"""Load file and feed its content to `configure_credentials`.</span>
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="n">secrets</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">],</span>
|
||||||
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd"> Extra keys are ignored.</span>
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a> <span class="sd">"""Load file and feed its content to `configure_credentials`.</span>
|
||||||
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a>
|
||||||
<a id="__codelineno-0-88" name="__codelineno-0-88"></a>
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> Extra keys are ignored.</span>
|
||||||
<a id="__codelineno-0-89" name="__codelineno-0-89"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">secrets</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">):</span>
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-90" name="__codelineno-0-90"></a> <span class="n">secrets</span> <span class="o">=</span> <span class="n">ConfigFile</span><span class="p">(</span><span class="n">secrets</span><span class="p">)</span>
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a>
|
||||||
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">(</span><span class="o">**</span><span class="p">{</span><span class="n">k</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">secrets</span><span class="o">.</span><span class="n">items</span><span class="p">()})</span>
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a> <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">secrets</span><span class="p">,</span> <span class="n">ConfigFile</span><span class="p">):</span>
|
||||||
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a> <span class="n">secrets</span> <span class="o">=</span> <span class="n">ConfigFile</span><span class="p">(</span><span class="n">secrets</span><span class="p">)</span>
|
||||||
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a> <span class="bp">cls</span><span class="o">.</span><span class="n">configure_credentials</span><span class="p">(</span><span class="o">**</span><span class="p">{</span><span class="n">k</span><span class="o">.</span><span class="n">lower</span><span class="p">():</span> <span class="n">v</span> <span class="k">for</span> <span class="n">k</span><span class="p">,</span> <span class="n">v</span> <span class="ow">in</span> <span class="n">secrets</span><span class="o">.</span><span class="n">items</span><span class="p">()})</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2652,15 +2652,15 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-226">226</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-228">228</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-227">227</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-228">228</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-229">229</a></span>
|
<span class="normal"><a href="#__codelineno-0-229">229</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-230">230</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-226" name="__codelineno-0-226"></a><span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-230">230</a></span>
|
||||||
<a id="__codelineno-0-227" name="__codelineno-0-227"></a> <span class="sd">"""Delete all versions of the files under this `key`."""</span>
|
<span class="normal"><a href="#__codelineno-0-231">231</a></span>
|
||||||
<a id="__codelineno-0-228" name="__codelineno-0-228"></a>
|
<span class="normal"><a href="#__codelineno-0-232">232</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-228" name="__codelineno-0-228"></a><span class="k">def</span> <span class="nf">delete</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-229" name="__codelineno-0-229"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="mi">0</span>
|
<a id="__codelineno-0-229" name="__codelineno-0-229"></a> <span class="sd">"""Delete all versions of the files under this `key`."""</span>
|
||||||
<a id="__codelineno-0-230" name="__codelineno-0-230"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
<a id="__codelineno-0-230" name="__codelineno-0-230"></a>
|
||||||
|
<a id="__codelineno-0-231" name="__codelineno-0-231"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_keep_last_n</span> <span class="o">=</span> <span class="mi">0</span>
|
||||||
|
<a id="__codelineno-0-232" name="__codelineno-0-232"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_delete_old_remote_versions</span><span class="p">()</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2712,9 +2712,7 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-146">146</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-148">148</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-147">147</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-148">148</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-149">149</a></span>
|
<span class="normal"><a href="#__codelineno-0-149">149</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-150">150</a></span>
|
<span class="normal"><a href="#__codelineno-0-150">150</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-151">151</a></span>
|
<span class="normal"><a href="#__codelineno-0-151">151</a></span>
|
||||||
|
|
@ -2745,40 +2743,42 @@ automatic cache-pruning.</p></td>
|
||||||
<span class="normal"><a href="#__codelineno-0-176">176</a></span>
|
<span class="normal"><a href="#__codelineno-0-176">176</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-177">177</a></span>
|
<span class="normal"><a href="#__codelineno-0-177">177</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-178">178</a></span>
|
<span class="normal"><a href="#__codelineno-0-178">178</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-179">179</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-146" name="__codelineno-0-146"></a><span class="nd">@lru_cache</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
<span class="normal"><a href="#__codelineno-0-179">179</a></span>
|
||||||
<a id="__codelineno-0-147" name="__codelineno-0-147"></a><span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-180">180</a></span>
|
||||||
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="sd">"""Return path to the proxy of a file (or directory).</span>
|
<span class="normal"><a href="#__codelineno-0-181">181</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-148" name="__codelineno-0-148"></a><span class="nd">@lru_cache</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-149" name="__codelineno-0-149"></a>
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a><span class="k">def</span> <span class="nf">get</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="n">Path</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-150" name="__codelineno-0-150"></a><span class="sd"> If not available in the local cache, an attempt is made to download it.</span>
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="sd">"""Return path to the proxy of a file (or directory).</span>
|
||||||
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
|
||||||
<a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a><span class="sd"> If not available in the local cache, an attempt is made to download it.</span>
|
||||||
<a id="__codelineno-0-153" name="__codelineno-0-153"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a>
|
||||||
<a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-155" name="__codelineno-0-155"></a>
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
||||||
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">remote_path</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">i</span><span class="o">.</span><span class="n">remote_path</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">if</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span>
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a>
|
||||||
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="p">)</span>
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">remote_path</span> <span class="o">=</span> <span class="nb">next</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-159" name="__codelineno-0-159"></a>
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">i</span><span class="o">.</span><span class="n">remote_path</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span> <span class="k">if</span> <span class="n">i</span><span class="o">.</span><span class="n">version</span> <span class="o">==</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span>
|
||||||
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">destination</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span>
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">destination</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
<a id="__codelineno-0-161" name="__codelineno-0-161"></a>
|
||||||
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> does not exist locally"</span><span class="p">)</span>
|
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">destination</span> <span class="o">=</span> <span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span>
|
||||||
<a id="__codelineno-0-163" name="__codelineno-0-163"></a>
|
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="k">if</span> <span class="ow">not</span> <span class="n">destination</span><span class="o">.</span><span class="n">exists</span><span class="p">():</span>
|
||||||
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> does not exist locally"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">local_root_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
|
<a id="__codelineno-0-165" name="__codelineno-0-165"></a>
|
||||||
<a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="n">tmp_file_archive</span> <span class="o">=</span> <span class="p">(</span>
|
<a id="__codelineno-0-166" name="__codelineno-0-166"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="n">local_root_path</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-167" name="__codelineno-0-167"></a> <span class="n">local_root_path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="p">)</span>
|
<a id="__codelineno-0-168" name="__codelineno-0-168"></a> <span class="n">tmp_file_archive</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_download</span><span class="p">(</span>
|
<a id="__codelineno-0-169" name="__codelineno-0-169"></a> <span class="n">local_root_path</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-170" name="__codelineno-0-170"></a> <span class="n">remote_path</span><span class="p">,</span> <span class="n">tmp_file_archive</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span>
|
<a id="__codelineno-0-170" name="__codelineno-0-170"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-171" name="__codelineno-0-171"></a> <span class="p">)</span>
|
<a id="__codelineno-0-171" name="__codelineno-0-171"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_download</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-172" name="__codelineno-0-172"></a>
|
<a id="__codelineno-0-172" name="__codelineno-0-172"></a> <span class="n">remote_path</span><span class="p">,</span> <span class="n">tmp_file_archive</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span>
|
||||||
<a id="__codelineno-0-173" name="__codelineno-0-173"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Decompressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-173" name="__codelineno-0-173"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-174" name="__codelineno-0-174"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">unpack_archive</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">tmp_file_archive</span><span class="p">),</span> <span class="n">tmp</span><span class="p">,</span> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">)</span>
|
<a id="__codelineno-0-174" name="__codelineno-0-174"></a>
|
||||||
<a id="__codelineno-0-175" name="__codelineno-0-175"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">local_root_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">destination</span><span class="p">))</span>
|
<a id="__codelineno-0-175" name="__codelineno-0-175"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Decompressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-176" name="__codelineno-0-176"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-176" name="__codelineno-0-176"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">unpack_archive</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">tmp_file_archive</span><span class="p">),</span> <span class="n">tmp</span><span class="p">,</span> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-177" name="__codelineno-0-177"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> found in cache"</span><span class="p">)</span>
|
<a id="__codelineno-0-177" name="__codelineno-0-177"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">move</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">local_root_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">destination</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-178" name="__codelineno-0-178"></a>
|
<a id="__codelineno-0-178" name="__codelineno-0-178"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-179" name="__codelineno-0-179"></a> <span class="k">return</span> <span class="n">destination</span>
|
<a id="__codelineno-0-179" name="__codelineno-0-179"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"File </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2"> found in cache"</span><span class="p">)</span>
|
||||||
|
<a id="__codelineno-0-180" name="__codelineno-0-180"></a>
|
||||||
|
<a id="__codelineno-0-181" name="__codelineno-0-181"></a> <span class="k">return</span> <span class="n">destination</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2826,9 +2826,7 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-181">181</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-183">183</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-182">182</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-183">183</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-184">184</a></span>
|
<span class="normal"><a href="#__codelineno-0-184">184</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-185">185</a></span>
|
<span class="normal"><a href="#__codelineno-0-185">185</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-186">186</a></span>
|
<span class="normal"><a href="#__codelineno-0-186">186</a></span>
|
||||||
|
|
@ -2869,50 +2867,52 @@ automatic cache-pruning.</p></td>
|
||||||
<span class="normal"><a href="#__codelineno-0-221">221</a></span>
|
<span class="normal"><a href="#__codelineno-0-221">221</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-222">222</a></span>
|
<span class="normal"><a href="#__codelineno-0-222">222</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-223">223</a></span>
|
<span class="normal"><a href="#__codelineno-0-223">223</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-224">224</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-181" name="__codelineno-0-181"></a><span class="k">def</span> <span class="nf">push</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-224">224</a></span>
|
||||||
<a id="__codelineno-0-182" name="__codelineno-0-182"></a> <span class="sd">"""Upload a file (or directory) as a new version of `key`.</span>
|
<span class="normal"><a href="#__codelineno-0-225">225</a></span>
|
||||||
<a id="__codelineno-0-183" name="__codelineno-0-183"></a>
|
<span class="normal"><a href="#__codelineno-0-226">226</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-183" name="__codelineno-0-183"></a><span class="k">def</span> <span class="nf">push</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Path</span><span class="p">,</span> <span class="nb">str</span><span class="p">],</span> <span class="n">hide_progress</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">)</span> <span class="o">-></span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-184" name="__codelineno-0-184"></a><span class="sd"> The file/directory is compressed before upload.</span>
|
<a id="__codelineno-0-184" name="__codelineno-0-184"></a> <span class="sd">"""Upload a file (or directory) as a new version of `key`.</span>
|
||||||
<a id="__codelineno-0-185" name="__codelineno-0-185"></a>
|
<a id="__codelineno-0-185" name="__codelineno-0-185"></a>
|
||||||
<a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-186" name="__codelineno-0-186"></a><span class="sd"> The file/directory is compressed before upload.</span>
|
||||||
<a id="__codelineno-0-187" name="__codelineno-0-187"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
<a id="__codelineno-0-187" name="__codelineno-0-187"></a>
|
||||||
<a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-188" name="__codelineno-0-188"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-189" name="__codelineno-0-189"></a>
|
<a id="__codelineno-0-189" name="__codelineno-0-189"></a><span class="sd"> hide_progress: Do not show a progress update after each 10% of progress.</span>
|
||||||
<a id="__codelineno-0-190" name="__codelineno-0-190"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
<a id="__codelineno-0-190" name="__codelineno-0-190"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-191" name="__codelineno-0-191"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
<a id="__codelineno-0-191" name="__codelineno-0-191"></a>
|
||||||
<a id="__codelineno-0-192" name="__codelineno-0-192"></a>
|
<a id="__codelineno-0-192" name="__codelineno-0-192"></a> <span class="k">if</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">path</span><span class="p">,</span> <span class="nb">str</span><span class="p">):</span>
|
||||||
<a id="__codelineno-0-193" name="__codelineno-0-193"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
<a id="__codelineno-0-193" name="__codelineno-0-193"></a> <span class="n">path</span> <span class="o">=</span> <span class="n">Path</span><span class="p">(</span><span class="n">path</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-194" name="__codelineno-0-194"></a> <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
|
<a id="__codelineno-0-194" name="__codelineno-0-194"></a>
|
||||||
<a id="__codelineno-0-195" name="__codelineno-0-195"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying file for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-195" name="__codelineno-0-195"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-196" name="__codelineno-0-196"></a> <span class="n">copy</span><span class="p">:</span> <span class="n">Any</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span>
|
<a id="__codelineno-0-196" name="__codelineno-0-196"></a> <span class="k">if</span> <span class="n">path</span><span class="o">.</span><span class="n">is_file</span><span class="p">():</span>
|
||||||
<a id="__codelineno-0-197" name="__codelineno-0-197"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-197" name="__codelineno-0-197"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying file for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-198" name="__codelineno-0-198"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying directory for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-198" name="__codelineno-0-198"></a> <span class="n">copy</span><span class="p">:</span> <span class="n">Any</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copy</span>
|
||||||
<a id="__codelineno-0-199" name="__codelineno-0-199"></a> <span class="n">copy</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copytree</span>
|
<a id="__codelineno-0-199" name="__codelineno-0-199"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-200" name="__codelineno-0-200"></a>
|
<a id="__codelineno-0-200" name="__codelineno-0-200"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Copying directory for </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-201" name="__codelineno-0-201"></a> <span class="k">try</span><span class="p">:</span>
|
<a id="__codelineno-0-201" name="__codelineno-0-201"></a> <span class="n">copy</span> <span class="o">=</span> <span class="n">shutil</span><span class="o">.</span><span class="n">copytree</span>
|
||||||
<a id="__codelineno-0-202" name="__codelineno-0-202"></a> <span class="c1"># Make local copy in the cache</span>
|
<a id="__codelineno-0-202" name="__codelineno-0-202"></a>
|
||||||
<a id="__codelineno-0-203" name="__codelineno-0-203"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">,</span> <span class="n">ignore_errors</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
<a id="__codelineno-0-203" name="__codelineno-0-203"></a> <span class="k">try</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-204" name="__codelineno-0-204"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
<a id="__codelineno-0-204" name="__codelineno-0-204"></a> <span class="c1"># Make local copy in the cache</span>
|
||||||
<a id="__codelineno-0-205" name="__codelineno-0-205"></a> <span class="k">except</span> <span class="n">shutil</span><span class="o">.</span><span class="n">SameFileError</span><span class="p">:</span>
|
<a id="__codelineno-0-205" name="__codelineno-0-205"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">rmtree</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">,</span> <span class="n">ignore_errors</span><span class="o">=</span><span class="kc">True</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-206" name="__codelineno-0-206"></a> <span class="k">pass</span> <span class="c1"># No worries</span>
|
<a id="__codelineno-0-206" name="__codelineno-0-206"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">cache_path</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-207" name="__codelineno-0-207"></a>
|
<a id="__codelineno-0-207" name="__codelineno-0-207"></a> <span class="k">except</span> <span class="n">shutil</span><span class="o">.</span><span class="n">SameFileError</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-208" name="__codelineno-0-208"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
<a id="__codelineno-0-208" name="__codelineno-0-208"></a> <span class="k">pass</span> <span class="c1"># No worries</span>
|
||||||
<a id="__codelineno-0-209" name="__codelineno-0-209"></a>
|
<a id="__codelineno-0-209" name="__codelineno-0-209"></a>
|
||||||
<a id="__codelineno-0-210" name="__codelineno-0-210"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp2</span><span class="p">:</span>
|
<a id="__codelineno-0-210" name="__codelineno-0-210"></a> <span class="n">copy</span><span class="p">(</span><span class="nb">str</span><span class="p">(</span><span class="n">path</span><span class="p">),</span> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-211" name="__codelineno-0-211"></a> <span class="c1"># A directory has to be zipped and it cannot contain the output of the zipping</span>
|
<a id="__codelineno-0-211" name="__codelineno-0-211"></a>
|
||||||
<a id="__codelineno-0-212" name="__codelineno-0-212"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Compressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
<a id="__codelineno-0-212" name="__codelineno-0-212"></a> <span class="k">with</span> <span class="n">tempfile</span><span class="o">.</span><span class="n">TemporaryDirectory</span><span class="p">()</span> <span class="k">as</span> <span class="n">tmp2</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-213" name="__codelineno-0-213"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">make_archive</span><span class="p">(</span>
|
<a id="__codelineno-0-213" name="__codelineno-0-213"></a> <span class="c1"># A directory has to be zipped and it cannot contain the output of the zipping</span>
|
||||||
<a id="__codelineno-0-214" name="__codelineno-0-214"></a> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span>
|
<a id="__codelineno-0-214" name="__codelineno-0-214"></a> <span class="n">logger</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="sa">f</span><span class="s2">"Compressing </span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}</span><span class="s2">"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-215" name="__codelineno-0-215"></a> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">,</span>
|
<a id="__codelineno-0-215" name="__codelineno-0-215"></a> <span class="n">shutil</span><span class="o">.</span><span class="n">make_archive</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-216" name="__codelineno-0-216"></a> <span class="n">tmp</span><span class="p">,</span>
|
<a id="__codelineno-0-216" name="__codelineno-0-216"></a> <span class="nb">str</span><span class="p">(</span><span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="p">),</span>
|
||||||
<a id="__codelineno-0-217" name="__codelineno-0-217"></a> <span class="p">)</span>
|
<a id="__codelineno-0-217" name="__codelineno-0-217"></a> <span class="n">COMPRESSION_ALGORITHM</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-218" name="__codelineno-0-218"></a>
|
<a id="__codelineno-0-218" name="__codelineno-0-218"></a> <span class="n">tmp</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-219" name="__codelineno-0-219"></a> <span class="n">file_to_be_uploaded</span> <span class="o">=</span> <span class="p">(</span>
|
<a id="__codelineno-0-219" name="__codelineno-0-219"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-220" name="__codelineno-0-220"></a> <span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
<a id="__codelineno-0-220" name="__codelineno-0-220"></a>
|
||||||
<a id="__codelineno-0-221" name="__codelineno-0-221"></a> <span class="p">)</span>
|
<a id="__codelineno-0-221" name="__codelineno-0-221"></a> <span class="n">file_to_be_uploaded</span> <span class="o">=</span> <span class="p">(</span>
|
||||||
<a id="__codelineno-0-222" name="__codelineno-0-222"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_upload</span><span class="p">(</span><span class="n">file_to_be_uploaded</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span><span class="p">)</span>
|
<a id="__codelineno-0-222" name="__codelineno-0-222"></a> <span class="n">Path</span><span class="p">(</span><span class="n">tmp2</span><span class="p">)</span> <span class="o">/</span> <span class="sa">f</span><span class="s2">"</span><span class="si">{</span><span class="bp">self</span><span class="o">.</span><span class="n">_local_name</span><span class="si">}{</span><span class="n">ARCHIVE_EXTENSION</span><span class="si">}</span><span class="s2">"</span>
|
||||||
<a id="__codelineno-0-223" name="__codelineno-0-223"></a>
|
<a id="__codelineno-0-223" name="__codelineno-0-223"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-224" name="__codelineno-0-224"></a> <span class="bp">self</span><span class="o">.</span><span class="n">clean_up</span><span class="p">()</span>
|
<a id="__codelineno-0-224" name="__codelineno-0-224"></a> <span class="bp">self</span><span class="o">.</span><span class="n">_upload</span><span class="p">(</span><span class="n">file_to_be_uploaded</span><span class="p">,</span> <span class="n">hide_progress</span><span class="o">=</span><span class="n">hide_progress</span><span class="p">)</span>
|
||||||
|
<a id="__codelineno-0-225" name="__codelineno-0-225"></a>
|
||||||
|
<a id="__codelineno-0-226" name="__codelineno-0-226"></a> <span class="bp">self</span><span class="o">.</span><span class="n">clean_up</span><span class="p">()</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2939,15 +2939,15 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-140">140</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-142">142</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-141">141</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-142">142</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-143">143</a></span>
|
<span class="normal"><a href="#__codelineno-0-143">143</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-144">144</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-140" name="__codelineno-0-140"></a><span class="nd">@property</span>
|
<span class="normal"><a href="#__codelineno-0-144">144</a></span>
|
||||||
<a id="__codelineno-0-141" name="__codelineno-0-141"></a><span class="k">def</span> <span class="nf">version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
<span class="normal"><a href="#__codelineno-0-145">145</a></span>
|
||||||
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="sd">"""Numeric version of the file proxied by this LargeFile instance."""</span>
|
<span class="normal"><a href="#__codelineno-0-146">146</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-142" name="__codelineno-0-142"></a><span class="nd">@property</span>
|
||||||
<a id="__codelineno-0-143" name="__codelineno-0-143"></a>
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a><span class="k">def</span> <span class="nf">version</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">int</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="sd">"""Numeric version of the file proxied by this LargeFile instance."""</span>
|
||||||
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a>
|
||||||
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="k">return</span> <span class="n">cast</span><span class="p">(</span><span class="nb">int</span><span class="p">,</span> <span class="bp">self</span><span class="o">.</span><span class="n">_version</span><span class="p">)</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -2974,13 +2974,13 @@ automatic cache-pruning.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
<summary>Source code in <code>great_ai/large_file/large_file/large_file_base.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-232">232</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-234">234</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-233">233</a></span>
|
<span class="normal"><a href="#__codelineno-0-235">235</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-234">234</a></span>
|
<span class="normal"><a href="#__codelineno-0-236">236</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-235">235</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-232" name="__codelineno-0-232"></a><span class="nd">@property</span>
|
<span class="normal"><a href="#__codelineno-0-237">237</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-234" name="__codelineno-0-234"></a><span class="nd">@property</span>
|
||||||
<a id="__codelineno-0-233" name="__codelineno-0-233"></a><span class="k">def</span> <span class="nf">versions_pretty</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
<a id="__codelineno-0-235" name="__codelineno-0-235"></a><span class="k">def</span> <span class="nf">versions_pretty</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span> <span class="o">-></span> <span class="nb">str</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-234" name="__codelineno-0-234"></a> <span class="sd">"""Formatted string of all available versions."""</span>
|
<a id="__codelineno-0-236" name="__codelineno-0-236"></a> <span class="sd">"""Formatted string of all available versions."""</span>
|
||||||
<a id="__codelineno-0-235" name="__codelineno-0-235"></a> <span class="k">return</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">))</span>
|
<a id="__codelineno-0-237" name="__codelineno-0-237"></a> <span class="k">return</span> <span class="s2">", "</span><span class="o">.</span><span class="n">join</span><span class="p">((</span><span class="nb">str</span><span class="p">(</span><span class="n">i</span><span class="o">.</span><span class="n">version</span><span class="p">)</span> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">_instances</span><span class="p">))</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -2516,7 +2516,7 @@ language tagging.</p>
|
||||||
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="k">assert</span> <span class="mi">0</span> <span class="o"><=</span> <span class="n">target_recall</span> <span class="o"><=</span> <span class="mi">1</span>
|
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="k">assert</span> <span class="mi">0</span> <span class="o"><=</span> <span class="n">target_recall</span> <span class="o"><=</span> <span class="mi">1</span>
|
||||||
<a id="__codelineno-0-47" name="__codelineno-0-47"></a>
|
<a id="__codelineno-0-47" name="__codelineno-0-47"></a>
|
||||||
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="k">if</span> <span class="n">plot</span> <span class="ow">and</span> <span class="n">axes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="k">if</span> <span class="n">plot</span> <span class="ow">and</span> <span class="n">axes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
|
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">20</span><span class="p">))</span>
|
||||||
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="n">fig</span><span class="o">.</span><span class="n">patch</span><span class="o">.</span><span class="n">set_facecolor</span><span class="p">(</span><span class="s2">"white"</span><span class="p">)</span>
|
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="n">fig</span><span class="o">.</span><span class="n">patch</span><span class="o">.</span><span class="n">set_facecolor</span><span class="p">(</span><span class="s2">"white"</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">axes</span><span class="p">()</span>
|
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">axes</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="k">else</span><span class="p">:</span>
|
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="k">else</span><span class="p">:</span>
|
||||||
|
|
@ -3391,8 +3391,7 @@ thread and <code>ignore_exceptions=False</code>.</p></td>
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/utilities/parallel_map/threaded_parallel_map.py</code></summary>
|
<summary>Source code in <code>great_ai/utilities/parallel_map/threaded_parallel_map.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-75"> 75</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
|
|
||||||
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
|
||||||
|
|
@ -3480,96 +3479,97 @@ thread and <code>ignore_exceptions=False</code>.</p></td>
|
||||||
<span class="normal"><a href="#__codelineno-0-161">161</a></span>
|
<span class="normal"><a href="#__codelineno-0-161">161</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-162">162</a></span>
|
<span class="normal"><a href="#__codelineno-0-162">162</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-163">163</a></span>
|
<span class="normal"><a href="#__codelineno-0-163">163</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-164">164</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-75" name="__codelineno-0-75"></a><span class="k">def</span> <span class="nf">threaded_parallel_map</span><span class="p">(</span>
|
<span class="normal"><a href="#__codelineno-0-164">164</a></span>
|
||||||
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
|
<span class="normal"><a href="#__codelineno-0-165">165</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-76" name="__codelineno-0-76"></a><span class="k">def</span> <span class="nf">threaded_parallel_map</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
|
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
|
||||||
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="o">*</span><span class="p">,</span>
|
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
|
||||||
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="o">*</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
|
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="sd">"""Execute a map operation on an iterable stream.</span>
|
<a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="p">)</span> <span class="o">-></span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
|
||||||
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
|
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="sd">"""Execute a map operation on an iterable stream.</span>
|
||||||
<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd"> Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]</span>
|
<a id="__codelineno-0-86" name="__codelineno-0-86"></a>
|
||||||
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> but uses threads instead of processes. Hence, it is not helpful in CPU-bound</span>
|
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]</span>
|
||||||
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> situations.</span>
|
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> but uses threads instead of processes. Hence, it is not helpful in CPU-bound</span>
|
||||||
<a id="__codelineno-0-89" name="__codelineno-0-89"></a>
|
<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd"> situations.</span>
|
||||||
<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
|
<a id="__codelineno-0-90" name="__codelineno-0-90"></a>
|
||||||
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> functions. Exceptions encountered in the map function are sent to the host thread</span>
|
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
|
||||||
<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd"> where they are either raised (default) or ignored. Each process processes a single</span>
|
<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd"> functions. Exceptions encountered in the map function are sent to the host thread</span>
|
||||||
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> chunk at once.</span>
|
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> where they are either raised (default) or ignored. Each process processes a single</span>
|
||||||
<a id="__codelineno-0-94" name="__codelineno-0-94"></a>
|
<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd"> chunk at once.</span>
|
||||||
<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd"> Examples:</span>
|
<a id="__codelineno-0-95" name="__codelineno-0-95"></a>
|
||||||
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd"> >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))</span>
|
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd"> Examples:</span>
|
||||||
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> [1, 4, 9]</span>
|
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> >>> list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))</span>
|
||||||
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
|
<a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="sd"> [1, 4, 9]</span>
|
||||||
<a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="sd"> Args:</span>
|
<a id="__codelineno-0-99" name="__codelineno-0-99"></a>
|
||||||
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
|
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> Args:</span>
|
||||||
<a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
|
<a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
|
||||||
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
|
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
|
||||||
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
|
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
|
||||||
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
|
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
|
||||||
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
|
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
|
||||||
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> calculated automatically if not given.</span>
|
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
|
||||||
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
|
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> calculated automatically if not given.</span>
|
||||||
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
|
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
|
||||||
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> process.</span>
|
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
|
||||||
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> concurrency: Number of new threads to start.</span>
|
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> process.</span>
|
||||||
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
|
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> concurrency: Number of new threads to start.</span>
|
||||||
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
|
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
|
||||||
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> difficult-to-process items.</span>
|
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
|
||||||
<a id="__codelineno-0-114" name="__codelineno-0-114"></a>
|
<a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd"> difficult-to-process items.</span>
|
||||||
<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd"> Yields:</span>
|
<a id="__codelineno-0-115" name="__codelineno-0-115"></a>
|
||||||
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
|
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> Yields:</span>
|
||||||
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
|
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
|
||||||
<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd"> the input if `unordered=True`.</span>
|
<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
|
||||||
<a id="__codelineno-0-119" name="__codelineno-0-119"></a>
|
<a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd"> the input if `unordered=True`.</span>
|
||||||
<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd"> Raises:</span>
|
<a id="__codelineno-0-120" name="__codelineno-0-120"></a>
|
||||||
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
|
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> Raises:</span>
|
||||||
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> thread and `ignore_exceptions=False`.</span>
|
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
|
||||||
<a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd"> """</span>
|
<a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd"> thread and `ignore_exceptions=False`.</span>
|
||||||
<a id="__codelineno-0-124" name="__codelineno-0-124"></a>
|
<a id="__codelineno-0-124" name="__codelineno-0-124"></a><span class="sd"> """</span>
|
||||||
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
|
<a id="__codelineno-0-125" name="__codelineno-0-125"></a>
|
||||||
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
|
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
|
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="p">)</span>
|
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-131" name="__codelineno-0-131"></a>
|
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="n">input_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
<a id="__codelineno-0-132" name="__codelineno-0-132"></a>
|
||||||
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">output_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">input_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">output_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-135" name="__codelineno-0-135"></a>
|
<a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">threads</span> <span class="o">=</span> <span class="p">[</span>
|
<a id="__codelineno-0-136" name="__codelineno-0-136"></a>
|
||||||
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span>
|
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">threads</span> <span class="o">=</span> <span class="p">[</span>
|
||||||
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">"threaded_parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
|
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">"threaded_parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">"</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
|
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
|
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="p">),</span>
|
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="p">)</span>
|
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="p">),</span>
|
||||||
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
|
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="p">]</span>
|
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
|
||||||
<a id="__codelineno-0-150" name="__codelineno-0-150"></a>
|
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="p">]</span>
|
||||||
<a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
|
||||||
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
||||||
<a id="__codelineno-0-153" name="__codelineno-0-153"></a>
|
<a id="__codelineno-0-153" name="__codelineno-0-153"></a> <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
|
<a id="__codelineno-0-154" name="__codelineno-0-154"></a>
|
||||||
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
|
||||||
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
|
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
|
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
|
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="p">)</span>
|
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
|
||||||
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="p">)</span>
|
||||||
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
|
||||||
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
|
||||||
|
<a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
|
|
@ -1474,11 +1474,11 @@ function, value of <code>ENVIRONMENT</code>, its split if has any, and either
|
||||||
|
|
||||||
<details class="quote">
|
<details class="quote">
|
||||||
<summary>Source code in <code>great_ai/views/outputs/sequence_labeling_output.py</code></summary>
|
<summary>Source code in <code>great_ai/views/outputs/sequence_labeling_output.py</code></summary>
|
||||||
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-13">13</a></span>
|
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-15">15</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-14">14</a></span>
|
<span class="normal"><a href="#__codelineno-0-16">16</a></span>
|
||||||
<span class="normal"><a href="#__codelineno-0-15">15</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-13" name="__codelineno-0-13"></a><span class="k">class</span> <span class="nc">SequenceLabelingOutput</span><span class="p">(</span><span class="n">HashableBaseModel</span><span class="p">):</span>
|
<span class="normal"><a href="#__codelineno-0-17">17</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-15" name="__codelineno-0-15"></a><span class="k">class</span> <span class="nc">SequenceLabelingOutput</span><span class="p">(</span><span class="n">HashableBaseModel</span><span class="p">):</span>
|
||||||
<a id="__codelineno-0-14" name="__codelineno-0-14"></a> <span class="n">labeled_tokens</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">LabeledToken</span><span class="p">]</span>
|
<a id="__codelineno-0-16" name="__codelineno-0-16"></a> <span class="n">labeled_tokens</span><span class="p">:</span> <span class="n">List</span><span class="p">[</span><span class="n">LabeledToken</span><span class="p">]</span>
|
||||||
<a id="__codelineno-0-15" name="__codelineno-0-15"></a> <span class="n">explanation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span>
|
<a id="__codelineno-0-17" name="__codelineno-0-17"></a> <span class="n">explanation</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="n">Any</span><span class="p">]</span>
|
||||||
</code></pre></div></td></tr></table></div>
|
</code></pre></div></td></tr></table></div>
|
||||||
</details>
|
</details>
|
||||||
|
|
||||||
|
|
|
||||||
File diff suppressed because one or more lines are too long
52
sitemap.xml
52
sitemap.xml
|
|
@ -2,132 +2,132 @@
|
||||||
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
<urlset xmlns="http://www.sitemaps.org/schemas/sitemap/0.9">
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
<url>
|
<url>
|
||||||
<loc>None</loc>
|
<loc>None</loc>
|
||||||
<lastmod>2022-07-25</lastmod>
|
<lastmod>2022-07-29</lastmod>
|
||||||
<changefreq>daily</changefreq>
|
<changefreq>daily</changefreq>
|
||||||
</url>
|
</url>
|
||||||
</urlset>
|
</urlset>
|
||||||
BIN
sitemap.xml.gz
BIN
sitemap.xml.gz
Binary file not shown.
Loading…
Add table
Add a link
Reference in a new issue