andras/great-ai
1
0
Fork 0
Code Issues Pull requests 1 Projects Releases 4 Packages Wiki Activity Actions

Deployed 85d079f with MkDocs version: 1.3.1

Browse source
This commit is contained in:
2022-07-29 07:52:03 +00:00
parent 95fdd502a9
commit 8a093ac9d2
7 changed files with 640 additions and 640 deletions
Download patch file Download diff file Expand all files Collapse all files

186
reference/utilities/index.html
View file

@ -2516,7 +2516,7 @@ language tagging.</p>
<a id="__codelineno-0-46" name="__codelineno-0-46"></a> <span class="k">assert</span> <span class="mi">0</span> <span class="o">&lt;=</span> <span class="n">target_recall</span> <span class="o">&lt;=</span> <span class="mi">1</span>
<a id="__codelineno-0-47" name="__codelineno-0-47"></a>
<a id="__codelineno-0-48" name="__codelineno-0-48"></a> <span class="k">if</span> <span class="n">plot</span> <span class="ow">and</span> <span class="n">axes</span> <span class="ow">is</span> <span class="kc">None</span><span class="p">:</span>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">10</span><span class="p">))</span>
<a id="__codelineno-0-49" name="__codelineno-0-49"></a> <span class="n">fig</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">figure</span><span class="p">(</span><span class="n">figsize</span><span class="o">=</span><span class="p">(</span><span class="mi">20</span><span class="p">,</span> <span class="mi">20</span><span class="p">))</span>
<a id="__codelineno-0-50" name="__codelineno-0-50"></a> <span class="n">fig</span><span class="o">.</span><span class="n">patch</span><span class="o">.</span><span class="n">set_facecolor</span><span class="p">(</span><span class="s2">&quot;white&quot;</span><span class="p">)</span>
<a id="__codelineno-0-51" name="__codelineno-0-51"></a> <span class="n">ax</span> <span class="o">=</span> <span class="n">plt</span><span class="o">.</span><span class="n">axes</span><span class="p">()</span>
<a id="__codelineno-0-52" name="__codelineno-0-52"></a> <span class="k">else</span><span class="p">:</span>
@ -3391,8 +3391,7 @@ thread and <code>ignore_exceptions=False</code>.</p></td>
<details class="quote">
<summary>Source code in <code>great_ai/utilities/parallel_map/threaded_parallel_map.py</code></summary>
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-75"> 75</a></span>
<span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
<div class="highlight"><table class="highlighttable"><tr><td class="linenos"><div class="linenodiv"><pre><span></span><span class="normal"><a href="#__codelineno-0-76"> 76</a></span>
<span class="normal"><a href="#__codelineno-0-77"> 77</a></span>
<span class="normal"><a href="#__codelineno-0-78"> 78</a></span>
<span class="normal"><a href="#__codelineno-0-79"> 79</a></span>
@ -3480,96 +3479,97 @@ thread and <code>ignore_exceptions=False</code>.</p></td>
<span class="normal"><a href="#__codelineno-0-161">161</a></span>
<span class="normal"><a href="#__codelineno-0-162">162</a></span>
<span class="normal"><a href="#__codelineno-0-163">163</a></span>
<span class="normal"><a href="#__codelineno-0-164">164</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-75" name="__codelineno-0-75"></a><span class="k">def</span> <span class="nf">threaded_parallel_map</span><span class="p">(</span>
<a id="__codelineno-0-76" name="__codelineno-0-76"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="o">*</span><span class="p">,</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a> <span class="sd">&quot;&quot;&quot;Execute a map operation on an iterable stream.</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a><span class="sd"> Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]</span>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> but uses threads instead of processes. Hence, it is not helpful in CPU-bound</span>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> situations.</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> functions. Exceptions encountered in the map function are sent to the host thread</span>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd"> where they are either raised (default) or ignored. Each process processes a single</span>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> chunk at once.</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a><span class="sd"> Examples:</span>
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd"> &gt;&gt;&gt; list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))</span>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> [1, 4, 9]</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a><span class="sd"> Args:</span>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> calculated automatically if not given.</span>
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> process.</span>
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> concurrency: Number of new threads to start.</span>
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> difficult-to-process items.</span>
<a id="__codelineno-0-114" name="__codelineno-0-114"></a>
<a id="__codelineno-0-115" name="__codelineno-0-115"></a><span class="sd"> Yields:</span>
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd"> the input if `unordered=True`.</span>
<a id="__codelineno-0-119" name="__codelineno-0-119"></a>
<a id="__codelineno-0-120" name="__codelineno-0-120"></a><span class="sd"> Raises:</span>
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> thread and `ignore_exceptions=False`.</span>
<a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-124" name="__codelineno-0-124"></a>
<a id="__codelineno-0-125" name="__codelineno-0-125"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="p">)</span>
<a id="__codelineno-0-131" name="__codelineno-0-131"></a>
<a id="__codelineno-0-132" name="__codelineno-0-132"></a> <span class="n">input_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">output_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
<a id="__codelineno-0-135" name="__codelineno-0-135"></a>
<a id="__codelineno-0-136" name="__codelineno-0-136"></a> <span class="n">threads</span> <span class="o">=</span> <span class="p">[</span>
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span>
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;threaded_parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="p">),</span>
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="p">)</span>
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="p">]</span>
<a id="__codelineno-0-150" name="__codelineno-0-150"></a>
<a id="__codelineno-0-151" name="__codelineno-0-151"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<a id="__codelineno-0-153" name="__codelineno-0-153"></a>
<a id="__codelineno-0-154" name="__codelineno-0-154"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="p">)</span>
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
<span class="normal"><a href="#__codelineno-0-164">164</a></span>
<span class="normal"><a href="#__codelineno-0-165">165</a></span></pre></div></td><td class="code"><div><pre><span></span><code><a id="__codelineno-0-76" name="__codelineno-0-76"></a><span class="k">def</span> <span class="nf">threaded_parallel_map</span><span class="p">(</span>
<a id="__codelineno-0-77" name="__codelineno-0-77"></a> <span class="n">func</span><span class="p">:</span> <span class="n">Callable</span><span class="p">[[</span><span class="n">T</span><span class="p">],</span> <span class="n">Union</span><span class="p">[</span><span class="n">V</span><span class="p">,</span> <span class="n">Awaitable</span><span class="p">[</span><span class="n">V</span><span class="p">]]],</span>
<a id="__codelineno-0-78" name="__codelineno-0-78"></a> <span class="n">input_values</span><span class="p">:</span> <span class="n">Union</span><span class="p">[</span><span class="n">Iterable</span><span class="p">[</span><span class="n">T</span><span class="p">],</span> <span class="n">Sequence</span><span class="p">[</span><span class="n">T</span><span class="p">]],</span>
<a id="__codelineno-0-79" name="__codelineno-0-79"></a> <span class="o">*</span><span class="p">,</span>
<a id="__codelineno-0-80" name="__codelineno-0-80"></a> <span class="n">chunk_size</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-81" name="__codelineno-0-81"></a> <span class="n">ignore_exceptions</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
<a id="__codelineno-0-82" name="__codelineno-0-82"></a> <span class="n">concurrency</span><span class="p">:</span> <span class="n">Optional</span><span class="p">[</span><span class="nb">int</span><span class="p">]</span> <span class="o">=</span> <span class="kc">None</span><span class="p">,</span>
<a id="__codelineno-0-83" name="__codelineno-0-83"></a> <span class="n">unordered</span><span class="p">:</span> <span class="nb">bool</span> <span class="o">=</span> <span class="kc">False</span><span class="p">,</span>
<a id="__codelineno-0-84" name="__codelineno-0-84"></a><span class="p">)</span> <span class="o">-&gt;</span> <span class="n">Iterable</span><span class="p">[</span><span class="n">Optional</span><span class="p">[</span><span class="n">V</span><span class="p">]]:</span>
<a id="__codelineno-0-85" name="__codelineno-0-85"></a> <span class="sd">&quot;&quot;&quot;Execute a map operation on an iterable stream.</span>
<a id="__codelineno-0-86" name="__codelineno-0-86"></a>
<a id="__codelineno-0-87" name="__codelineno-0-87"></a><span class="sd"> Similar to [parallel_map][great_ai.utilities.parallel_map.parallel_map.parallel_map]</span>
<a id="__codelineno-0-88" name="__codelineno-0-88"></a><span class="sd"> but uses threads instead of processes. Hence, it is not helpful in CPU-bound</span>
<a id="__codelineno-0-89" name="__codelineno-0-89"></a><span class="sd"> situations.</span>
<a id="__codelineno-0-90" name="__codelineno-0-90"></a>
<a id="__codelineno-0-91" name="__codelineno-0-91"></a><span class="sd"> A custom parallel map operation supporting both synchronous and `async` map</span>
<a id="__codelineno-0-92" name="__codelineno-0-92"></a><span class="sd"> functions. Exceptions encountered in the map function are sent to the host thread</span>
<a id="__codelineno-0-93" name="__codelineno-0-93"></a><span class="sd"> where they are either raised (default) or ignored. Each process processes a single</span>
<a id="__codelineno-0-94" name="__codelineno-0-94"></a><span class="sd"> chunk at once.</span>
<a id="__codelineno-0-95" name="__codelineno-0-95"></a>
<a id="__codelineno-0-96" name="__codelineno-0-96"></a><span class="sd"> Examples:</span>
<a id="__codelineno-0-97" name="__codelineno-0-97"></a><span class="sd"> &gt;&gt;&gt; list(threaded_parallel_map(lambda x: x ** 2, [1, 2, 3]))</span>
<a id="__codelineno-0-98" name="__codelineno-0-98"></a><span class="sd"> [1, 4, 9]</span>
<a id="__codelineno-0-99" name="__codelineno-0-99"></a>
<a id="__codelineno-0-100" name="__codelineno-0-100"></a><span class="sd"> Args:</span>
<a id="__codelineno-0-101" name="__codelineno-0-101"></a><span class="sd"> func: The function that should be applied to each element of `input_values`.</span>
<a id="__codelineno-0-102" name="__codelineno-0-102"></a><span class="sd"> It can `async`, in that case, a new event loop is started for each chunk.</span>
<a id="__codelineno-0-103" name="__codelineno-0-103"></a><span class="sd"> input_values: An iterable of items that `func` is applied to.</span>
<a id="__codelineno-0-104" name="__codelineno-0-104"></a><span class="sd"> chunk_size: Tune the number of items processed in each step. Larger numbers</span>
<a id="__codelineno-0-105" name="__codelineno-0-105"></a><span class="sd"> result in smaller communication overhead but less parallelism at the start</span>
<a id="__codelineno-0-106" name="__codelineno-0-106"></a><span class="sd"> and end. If `chunk_size` has a `__len__` property, the `chunk_size` is</span>
<a id="__codelineno-0-107" name="__codelineno-0-107"></a><span class="sd"> calculated automatically if not given.</span>
<a id="__codelineno-0-108" name="__codelineno-0-108"></a><span class="sd"> ignore_exceptions: Ignore chunks if `next()` raises an exception on</span>
<a id="__codelineno-0-109" name="__codelineno-0-109"></a><span class="sd"> `input_values`. And return `None` if `func` raised an exception in a worker</span>
<a id="__codelineno-0-110" name="__codelineno-0-110"></a><span class="sd"> process.</span>
<a id="__codelineno-0-111" name="__codelineno-0-111"></a><span class="sd"> concurrency: Number of new threads to start.</span>
<a id="__codelineno-0-112" name="__codelineno-0-112"></a><span class="sd"> unordered: Do not preserve the order of the elements, yield them as soon as they</span>
<a id="__codelineno-0-113" name="__codelineno-0-113"></a><span class="sd"> have been processed. This decreases the latency caused by</span>
<a id="__codelineno-0-114" name="__codelineno-0-114"></a><span class="sd"> difficult-to-process items.</span>
<a id="__codelineno-0-115" name="__codelineno-0-115"></a>
<a id="__codelineno-0-116" name="__codelineno-0-116"></a><span class="sd"> Yields:</span>
<a id="__codelineno-0-117" name="__codelineno-0-117"></a><span class="sd"> The next result obtained from applying `func` to each input value. May</span>
<a id="__codelineno-0-118" name="__codelineno-0-118"></a><span class="sd"> contain `None`-s if `ignore_exceptions=True`. May have different order than</span>
<a id="__codelineno-0-119" name="__codelineno-0-119"></a><span class="sd"> the input if `unordered=True`.</span>
<a id="__codelineno-0-120" name="__codelineno-0-120"></a>
<a id="__codelineno-0-121" name="__codelineno-0-121"></a><span class="sd"> Raises:</span>
<a id="__codelineno-0-122" name="__codelineno-0-122"></a><span class="sd"> WorkerException: If there was an error in the `func` function in a background</span>
<a id="__codelineno-0-123" name="__codelineno-0-123"></a><span class="sd"> thread and `ignore_exceptions=False`.</span>
<a id="__codelineno-0-124" name="__codelineno-0-124"></a><span class="sd"> &quot;&quot;&quot;</span>
<a id="__codelineno-0-125" name="__codelineno-0-125"></a>
<a id="__codelineno-0-126" name="__codelineno-0-126"></a> <span class="n">config</span> <span class="o">=</span> <span class="n">get_config</span><span class="p">(</span>
<a id="__codelineno-0-127" name="__codelineno-0-127"></a> <span class="n">function</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
<a id="__codelineno-0-128" name="__codelineno-0-128"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
<a id="__codelineno-0-129" name="__codelineno-0-129"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">chunk_size</span><span class="p">,</span>
<a id="__codelineno-0-130" name="__codelineno-0-130"></a> <span class="n">concurrency</span><span class="o">=</span><span class="n">concurrency</span><span class="p">,</span>
<a id="__codelineno-0-131" name="__codelineno-0-131"></a> <span class="p">)</span>
<a id="__codelineno-0-132" name="__codelineno-0-132"></a>
<a id="__codelineno-0-133" name="__codelineno-0-133"></a> <span class="n">input_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
<a id="__codelineno-0-134" name="__codelineno-0-134"></a> <span class="n">output_queue</span><span class="p">:</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span> <span class="o">=</span> <span class="n">queue</span><span class="o">.</span><span class="n">Queue</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span> <span class="o">*</span> <span class="mi">2</span><span class="p">)</span>
<a id="__codelineno-0-135" name="__codelineno-0-135"></a> <span class="n">should_stop</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Event</span><span class="p">()</span>
<a id="__codelineno-0-136" name="__codelineno-0-136"></a>
<a id="__codelineno-0-137" name="__codelineno-0-137"></a> <span class="n">threads</span> <span class="o">=</span> <span class="p">[</span>
<a id="__codelineno-0-138" name="__codelineno-0-138"></a> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span>
<a id="__codelineno-0-139" name="__codelineno-0-139"></a> <span class="n">name</span><span class="o">=</span><span class="sa">f</span><span class="s2">&quot;threaded_parallel_map_</span><span class="si">{</span><span class="n">config</span><span class="o">.</span><span class="n">function_name</span><span class="si">}</span><span class="s2">_</span><span class="si">{</span><span class="n">i</span><span class="si">}</span><span class="s2">&quot;</span><span class="p">,</span>
<a id="__codelineno-0-140" name="__codelineno-0-140"></a> <span class="n">target</span><span class="o">=</span><span class="n">mapper_function</span><span class="p">,</span>
<a id="__codelineno-0-141" name="__codelineno-0-141"></a> <span class="n">daemon</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
<a id="__codelineno-0-142" name="__codelineno-0-142"></a> <span class="n">kwargs</span><span class="o">=</span><span class="nb">dict</span><span class="p">(</span>
<a id="__codelineno-0-143" name="__codelineno-0-143"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
<a id="__codelineno-0-144" name="__codelineno-0-144"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
<a id="__codelineno-0-145" name="__codelineno-0-145"></a> <span class="n">should_stop</span><span class="o">=</span><span class="n">should_stop</span><span class="p">,</span>
<a id="__codelineno-0-146" name="__codelineno-0-146"></a> <span class="n">func</span><span class="o">=</span><span class="n">func</span><span class="p">,</span>
<a id="__codelineno-0-147" name="__codelineno-0-147"></a> <span class="p">),</span>
<a id="__codelineno-0-148" name="__codelineno-0-148"></a> <span class="p">)</span>
<a id="__codelineno-0-149" name="__codelineno-0-149"></a> <span class="k">for</span> <span class="n">i</span> <span class="ow">in</span> <span class="nb">range</span><span class="p">(</span><span class="n">config</span><span class="o">.</span><span class="n">concurrency</span><span class="p">)</span>
<a id="__codelineno-0-150" name="__codelineno-0-150"></a> <span class="p">]</span>
<a id="__codelineno-0-151" name="__codelineno-0-151"></a>
<a id="__codelineno-0-152" name="__codelineno-0-152"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
<a id="__codelineno-0-153" name="__codelineno-0-153"></a> <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
<a id="__codelineno-0-154" name="__codelineno-0-154"></a>
<a id="__codelineno-0-155" name="__codelineno-0-155"></a> <span class="k">yield from</span> <span class="n">manage_communication</span><span class="p">(</span>
<a id="__codelineno-0-156" name="__codelineno-0-156"></a> <span class="n">input_values</span><span class="o">=</span><span class="n">input_values</span><span class="p">,</span>
<a id="__codelineno-0-157" name="__codelineno-0-157"></a> <span class="n">chunk_size</span><span class="o">=</span><span class="n">config</span><span class="o">.</span><span class="n">chunk_size</span><span class="p">,</span>
<a id="__codelineno-0-158" name="__codelineno-0-158"></a> <span class="n">input_queue</span><span class="o">=</span><span class="n">input_queue</span><span class="p">,</span>
<a id="__codelineno-0-159" name="__codelineno-0-159"></a> <span class="n">output_queue</span><span class="o">=</span><span class="n">output_queue</span><span class="p">,</span>
<a id="__codelineno-0-160" name="__codelineno-0-160"></a> <span class="n">unordered</span><span class="o">=</span><span class="n">unordered</span><span class="p">,</span>
<a id="__codelineno-0-161" name="__codelineno-0-161"></a> <span class="n">ignore_exceptions</span><span class="o">=</span><span class="n">ignore_exceptions</span><span class="p">,</span>
<a id="__codelineno-0-162" name="__codelineno-0-162"></a> <span class="p">)</span>
<a id="__codelineno-0-163" name="__codelineno-0-163"></a> <span class="n">should_stop</span><span class="o">.</span><span class="n">set</span><span class="p">()</span>
<a id="__codelineno-0-164" name="__codelineno-0-164"></a> <span class="k">for</span> <span class="n">t</span> <span class="ow">in</span> <span class="n">threads</span><span class="p">:</span>
<a id="__codelineno-0-165" name="__codelineno-0-165"></a> <span class="n">t</span><span class="o">.</span><span class="n">join</span><span class="p">(</span><span class="mi">1</span><span class="p">)</span>
</code></pre></div></td></tr></table></div>
</details>
</div>