HTML code for sharing модуль для распараллеливания работы source code

HTML Copy To Clipboarddone!

<div class="highlight" style="background: #f8f8f8"><pre style="line-height: 125%"><div id="lc1"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">threading</span>
</div><div id="lc2"><span style="color: #008000; font-weight: bold">import</span> <span style="color: #0000FF; font-weight: bold">time</span>
</div><div id="lc3"><span style="color: #008000; font-weight: bold">from</span> <span style="color: #0000FF; font-weight: bold">Queue</span> <span style="color: #008000; font-weight: bold">import</span> Queue, Empty <span style="color: #008000; font-weight: bold">as</span> QueueEmpty
</div><div id="lc4">
</div><div id="lc5"><span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">make_work</span>(callback, tasks, limit, results<span style="color: #666666">=</span><span style="color: #008000">None</span>):
</div><div id="lc6">    <span style="color: #BA2121; font-style: italic">&quot;&quot;&quot;</span>
</div><div id="lc7"><span style="color: #BA2121; font-style: italic">    Run up to limit threads, do tasks.</span>
</div><div id="lc8">
</div><div id="lc9"><span style="color: #BA2121; font-style: italic">    Put results to results queue if it is specified.</span>
</div><div id="lc10"><span style="color: #BA2121; font-style: italic">    &quot;&quot;&quot;</span>
</div><div id="lc11">
</div><div id="lc12">    pool <span style="color: #666666">=</span> []
</div><div id="lc13">
</div><div id="lc14">    <span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000">isinstance</span>(tasks, Queue):
</div><div id="lc15">        new_tasks <span style="color: #666666">=</span> Queue()
</div><div id="lc16">        <span style="color: #008000; font-weight: bold">for</span> x <span style="color: #AA22FF; font-weight: bold">in</span> tasks:
</div><div id="lc17">            new_tasks<span style="color: #666666">.</span>put(x)
</div><div id="lc18">        tasks <span style="color: #666666">=</span> new_tasks
</div><div id="lc19">
</div><div id="lc20">    <span style="color: #408080; font-style: italic"># If callback result should be placed to the results queue</span>
</div><div id="lc21">    <span style="color: #408080; font-style: italic"># wrap the callback to the function that will catch result and</span>
</div><div id="lc22">    <span style="color: #408080; font-style: italic"># put it to the queue</span>
</div><div id="lc23">    <span style="color: #008000; font-weight: bold">if</span> results:
</div><div id="lc24">        old_callback <span style="color: #666666">=</span> callback
</div><div id="lc25">        <span style="color: #008000; font-weight: bold">def</span> <span style="color: #0000FF">callback</span>(<span style="color: #666666">*</span>args):
</div><div id="lc26">            results<span style="color: #666666">.</span>put(old_callback(<span style="color: #666666">*</span>args))
</div><div id="lc27">    
</div><div id="lc28">    <span style="color: #008000; font-weight: bold">while</span> <span style="color: #008000">True</span>:
</div><div id="lc29">        pool <span style="color: #666666">=</span> <span style="color: #008000">filter</span>(<span style="color: #008000; font-weight: bold">lambda</span> x: x<span style="color: #666666">.</span>isAlive(), pool)
</div><div id="lc30">
</div><div id="lc31">        <span style="color: #008000; font-weight: bold">for</span> x <span style="color: #AA22FF; font-weight: bold">in</span> <span style="color: #008000">xrange</span>(limit <span style="color: #666666">-</span> <span style="color: #008000">len</span>(pool)):
</div><div id="lc32">            <span style="color: #008000; font-weight: bold">try</span>:
</div><div id="lc33">                task <span style="color: #666666">=</span> tasks<span style="color: #666666">.</span>get(<span style="color: #008000">False</span>)
</div><div id="lc34">            <span style="color: #008000; font-weight: bold">except</span> QueueEmpty:
</div><div id="lc35">                <span style="color: #008000; font-weight: bold">pass</span>
</div><div id="lc36">            <span style="color: #008000; font-weight: bold">else</span>:
</div><div id="lc37">                <span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> <span style="color: #008000">isinstance</span>(task, (<span style="color: #008000">list</span>, <span style="color: #008000">tuple</span>)):
</div><div id="lc38">                    task <span style="color: #666666">=</span> [task]
</div><div id="lc39">                t <span style="color: #666666">=</span> threading<span style="color: #666666">.</span>Thread(target<span style="color: #666666">=</span>callback, args<span style="color: #666666">=</span>task)
</div><div id="lc40">                t<span style="color: #666666">.</span>start()
</div><div id="lc41">                pool<span style="color: #666666">.</span>append(t)
</div><div id="lc42">
</div><div id="lc43">        <span style="color: #008000; font-weight: bold">if</span> <span style="color: #AA22FF; font-weight: bold">not</span> pool:
</div><div id="lc44">            <span style="color: #008000; font-weight: bold">return</span>
</div><div id="lc45">
</div><div id="lc46">        time<span style="color: #666666">.</span>sleep(<span style="color: #666666">0.01</span>)
</div></pre></div>
<p style="padding: 0; margin: 2em 0 1em 0; font-size: 0.8em; color: #666;"><span style="padding: 0 1em; background-color: #f5f5f5;"> Colored with <a style="color: #333" href="http://dumpz.org">dumpz.org</a></span></p>

BBCODE Copy To Clipboarddone!

[color=#008000][b]import[/b][/color] [color=#0000FF][b]threading[/b][/color]
[color=#008000][b]import[/b][/color] [color=#0000FF][b]time[/b][/color]
[color=#008000][b]from[/b][/color] [color=#0000FF][b]Queue[/b][/color] [color=#008000][b]import[/b][/color] Queue, Empty [color=#008000][b]as[/b][/color] QueueEmpty

[color=#008000][b]def[/b][/color] [color=#0000FF]make_work[/color](callback, tasks, limit, results[color=#666666]=[/color][color=#008000]None[/color]):
    [color=#BA2121][i]"""
    Run up to limit threads, do tasks.

Put results to results queue if it is specified.
    """[/i][/color]

pool [color=#666666]=[/color] []

[color=#008000][b]if[/b][/color] [color=#AA22FF][b]not[/b][/color] [color=#008000]isinstance[/color](tasks, Queue):
        new_tasks [color=#666666]=[/color] Queue()
        [color=#008000][b]for[/b][/color] x [color=#AA22FF][b]in[/b][/color] tasks:
            new_tasks[color=#666666].[/color]put(x)
        tasks [color=#666666]=[/color] new_tasks

[color=#408080][i]# If callback result should be placed to the results queue[/i][/color]
    [color=#408080][i]# wrap the callback to the function that will catch result and[/i][/color]
    [color=#408080][i]# put it to the queue[/i][/color]
    [color=#008000][b]if[/b][/color] results:
        old_callback [color=#666666]=[/color] callback
        [color=#008000][b]def[/b][/color] [color=#0000FF]callback[/color]([color=#666666]*[/color]args):
            results[color=#666666].[/color]put(old_callback([color=#666666]*[/color]args))
    
    [color=#008000][b]while[/b][/color] [color=#008000]True[/color]:
        pool [color=#666666]=[/color] [color=#008000]filter[/color]([color=#008000][b]lambda[/b][/color] x: x[color=#666666].[/color]isAlive(), pool)

[color=#008000][b]for[/b][/color] x [color=#AA22FF][b]in[/b][/color] [color=#008000]xrange[/color](limit [color=#666666]-[/color] [color=#008000]len[/color](pool)):
            [color=#008000][b]try[/b][/color]:
                task [color=#666666]=[/color] tasks[color=#666666].[/color]get([color=#008000]False[/color])
            [color=#008000][b]except[/b][/color] QueueEmpty:
                [color=#008000][b]pass[/b][/color]
            [color=#008000][b]else[/b][/color]:
                [color=#008000][b]if[/b][/color] [color=#AA22FF][b]not[/b][/color] [color=#008000]isinstance[/color](task, ([color=#008000]list[/color], [color=#008000]tuple[/color])):
                    task [color=#666666]=[/color] [task]
                t [color=#666666]=[/color] threading[color=#666666].[/color]Thread(target[color=#666666]=[/color]callback, args[color=#666666]=[/color]task)
                t[color=#666666].[/color]start()
                pool[color=#666666].[/color]append(t)

[color=#008000][b]if[/b][/color] [color=#AA22FF][b]not[/b][/color] pool:
            [color=#008000][b]return[/b][/color]

time[color=#666666].[/color]sleep([color=#666666]0.01[/color])

--------------------------------------
Colored with http://dumpz.org

Light HTML Copy To Clipboarddone!

That method generates HTML with CSS classes. Hence you'll need the style sheet file

<div class="highlight"><pre><div id="lc1"><span class="kn">import</span> <span class="nn">threading</span>
</div><div id="lc2"><span class="kn">import</span> <span class="nn">time</span>
</div><div id="lc3"><span class="kn">from</span> <span class="nn">Queue</span> <span class="kn">import</span> <span class="n">Queue</span><span class="p">,</span> <span class="n">Empty</span> <span class="k">as</span> <span class="n">QueueEmpty</span>
</div><div id="lc4">
</div><div id="lc5"><span class="k">def</span> <span class="nf">make_work</span><span class="p">(</span><span class="n">callback</span><span class="p">,</span> <span class="n">tasks</span><span class="p">,</span> <span class="n">limit</span><span class="p">,</span> <span class="n">results</span><span class="o">=</span><span class="bp">None</span><span class="p">):</span>
</div><div id="lc6">    <span class="sd">&quot;&quot;&quot;</span>
</div><div id="lc7"><span class="sd">    Run up to limit threads, do tasks.</span>
</div><div id="lc8">
</div><div id="lc9"><span class="sd">    Put results to results queue if it is specified.</span>
</div><div id="lc10"><span class="sd">    &quot;&quot;&quot;</span>
</div><div id="lc11">
</div><div id="lc12">    <span class="n">pool</span> <span class="o">=</span> <span class="p">[]</span>
</div><div id="lc13">
</div><div id="lc14">    <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">tasks</span><span class="p">,</span> <span class="n">Queue</span><span class="p">):</span>
</div><div id="lc15">        <span class="n">new_tasks</span> <span class="o">=</span> <span class="n">Queue</span><span class="p">()</span>
</div><div id="lc16">        <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="n">tasks</span><span class="p">:</span>
</div><div id="lc17">            <span class="n">new_tasks</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">x</span><span class="p">)</span>
</div><div id="lc18">        <span class="n">tasks</span> <span class="o">=</span> <span class="n">new_tasks</span>
</div><div id="lc19">
</div><div id="lc20">    <span class="c"># If callback result should be placed to the results queue</span>
</div><div id="lc21">    <span class="c"># wrap the callback to the function that will catch result and</span>
</div><div id="lc22">    <span class="c"># put it to the queue</span>
</div><div id="lc23">    <span class="k">if</span> <span class="n">results</span><span class="p">:</span>
</div><div id="lc24">        <span class="n">old_callback</span> <span class="o">=</span> <span class="n">callback</span>
</div><div id="lc25">        <span class="k">def</span> <span class="nf">callback</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">):</span>
</div><div id="lc26">            <span class="n">results</span><span class="o">.</span><span class="n">put</span><span class="p">(</span><span class="n">old_callback</span><span class="p">(</span><span class="o">*</span><span class="n">args</span><span class="p">))</span>
</div><div id="lc27">    
</div><div id="lc28">    <span class="k">while</span> <span class="bp">True</span><span class="p">:</span>
</div><div id="lc29">        <span class="n">pool</span> <span class="o">=</span> <span class="nb">filter</span><span class="p">(</span><span class="k">lambda</span> <span class="n">x</span><span class="p">:</span> <span class="n">x</span><span class="o">.</span><span class="n">isAlive</span><span class="p">(),</span> <span class="n">pool</span><span class="p">)</span>
</div><div id="lc30">
</div><div id="lc31">        <span class="k">for</span> <span class="n">x</span> <span class="ow">in</span> <span class="nb">xrange</span><span class="p">(</span><span class="n">limit</span> <span class="o">-</span> <span class="nb">len</span><span class="p">(</span><span class="n">pool</span><span class="p">)):</span>
</div><div id="lc32">            <span class="k">try</span><span class="p">:</span>
</div><div id="lc33">                <span class="n">task</span> <span class="o">=</span> <span class="n">tasks</span><span class="o">.</span><span class="n">get</span><span class="p">(</span><span class="bp">False</span><span class="p">)</span>
</div><div id="lc34">            <span class="k">except</span> <span class="n">QueueEmpty</span><span class="p">:</span>
</div><div id="lc35">                <span class="k">pass</span>
</div><div id="lc36">            <span class="k">else</span><span class="p">:</span>
</div><div id="lc37">                <span class="k">if</span> <span class="ow">not</span> <span class="nb">isinstance</span><span class="p">(</span><span class="n">task</span><span class="p">,</span> <span class="p">(</span><span class="nb">list</span><span class="p">,</span> <span class="nb">tuple</span><span class="p">)):</span>
</div><div id="lc38">                    <span class="n">task</span> <span class="o">=</span> <span class="p">[</span><span class="n">task</span><span class="p">]</span>
</div><div id="lc39">                <span class="n">t</span> <span class="o">=</span> <span class="n">threading</span><span class="o">.</span><span class="n">Thread</span><span class="p">(</span><span class="n">target</span><span class="o">=</span><span class="n">callback</span><span class="p">,</span> <span class="n">args</span><span class="o">=</span><span class="n">task</span><span class="p">)</span>
</div><div id="lc40">                <span class="n">t</span><span class="o">.</span><span class="n">start</span><span class="p">()</span>
</div><div id="lc41">                <span class="n">pool</span><span class="o">.</span><span class="n">append</span><span class="p">(</span><span class="n">t</span><span class="p">)</span>
</div><div id="lc42">
</div><div id="lc43">        <span class="k">if</span> <span class="ow">not</span> <span class="n">pool</span><span class="p">:</span>
</div><div id="lc44">            <span class="k">return</span>
</div><div id="lc45">
</div><div id="lc46">        <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="mf">0.01</span><span class="p">)</span>
</div></pre></div>
<p style="padding: 0; margin: 2em 0 1em 0; font-size: 0.8em; color: #666;"><span style="padding: 0 1em; background-color: #f5f5f5;"> Colored with <a style="color: #333" href="http://dumpz.org">dumpz.org</a></span></p>

Habrahabr.ru HTML Copy To Clipboarddone!

Use that HTML for publications on habrahabr.ru

<blockquote><code><font color="#008000"><strong>import</strong></font>&nbsp;<font color="#0000FF"><strong>threading</strong></font>
<font color="#008000"><strong>import</strong></font>&nbsp;<font color="#0000FF"><strong>time</strong></font>
<font color="#008000"><strong>from</strong></font>&nbsp;<font color="#0000FF"><strong>Queue</strong></font>&nbsp;<font color="#008000"><strong>import</strong></font>&nbsp;Queue,&nbsp;Empty&nbsp;<font color="#008000"><strong>as</strong></font>&nbsp;QueueEmpty

<font color="#008000"><strong>def</strong></font>&nbsp;<font color="#0000FF">make_work</font>(callback,&nbsp;tasks,&nbsp;limit,&nbsp;results<font color="#666666">=</font><font color="#008000">None</font>):
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#BA2121"><em>&quot;&quot;&quot;
&nbsp;&nbsp;&nbsp;&nbsp;Run&nbsp;up&nbsp;to&nbsp;limit&nbsp;threads,&nbsp;do&nbsp;tasks.

&nbsp;&nbsp;&nbsp;&nbsp;Put&nbsp;results&nbsp;to&nbsp;results&nbsp;queue&nbsp;if&nbsp;it&nbsp;is&nbsp;specified.
&nbsp;&nbsp;&nbsp;&nbsp;&quot;&quot;&quot;</em></font>

&nbsp;&nbsp;&nbsp;&nbsp;pool&nbsp;<font color="#666666">=</font>&nbsp;[]

&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>if</strong></font>&nbsp;<font color="#AA22FF"><strong>not</strong></font>&nbsp;<font color="#008000">isinstance</font>(tasks,&nbsp;Queue):
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;new_tasks&nbsp;<font color="#666666">=</font>&nbsp;Queue()
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>for</strong></font>&nbsp;x&nbsp;<font color="#AA22FF"><strong>in</strong></font>&nbsp;tasks:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;new_tasks<font color="#666666">.</font>put(x)
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;tasks&nbsp;<font color="#666666">=</font>&nbsp;new_tasks

&nbsp;&nbsp;&nbsp;&nbsp;<font color="#408080"><em>#&nbsp;If&nbsp;callback&nbsp;result&nbsp;should&nbsp;be&nbsp;placed&nbsp;to&nbsp;the&nbsp;results&nbsp;queue</em></font>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#408080"><em>#&nbsp;wrap&nbsp;the&nbsp;callback&nbsp;to&nbsp;the&nbsp;function&nbsp;that&nbsp;will&nbsp;catch&nbsp;result&nbsp;and</em></font>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#408080"><em>#&nbsp;put&nbsp;it&nbsp;to&nbsp;the&nbsp;queue</em></font>
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>if</strong></font>&nbsp;results:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;old_callback&nbsp;<font color="#666666">=</font>&nbsp;callback
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>def</strong></font>&nbsp;<font color="#0000FF">callback</font>(<font color="#666666">*</font>args):
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;results<font color="#666666">.</font>put(old_callback(<font color="#666666">*</font>args))
&nbsp;&nbsp;&nbsp;&nbsp;
&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>while</strong></font>&nbsp;<font color="#008000">True</font>:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;pool&nbsp;<font color="#666666">=</font>&nbsp;<font color="#008000">filter</font>(<font color="#008000"><strong>lambda</strong></font>&nbsp;x:&nbsp;x<font color="#666666">.</font>isAlive(),&nbsp;pool)

&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>for</strong></font>&nbsp;x&nbsp;<font color="#AA22FF"><strong>in</strong></font>&nbsp;<font color="#008000">xrange</font>(limit&nbsp;<font color="#666666">-</font>&nbsp;<font color="#008000">len</font>(pool)):
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>try</strong></font>:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;task&nbsp;<font color="#666666">=</font>&nbsp;tasks<font color="#666666">.</font>get(<font color="#008000">False</font>)
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>except</strong></font>&nbsp;QueueEmpty:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>pass</strong></font>
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>else</strong></font>:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>if</strong></font>&nbsp;<font color="#AA22FF"><strong>not</strong></font>&nbsp;<font color="#008000">isinstance</font>(task,&nbsp;(<font color="#008000">list</font>,&nbsp;<font color="#008000">tuple</font>)):
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;task&nbsp;<font color="#666666">=</font>&nbsp;[task]
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;t&nbsp;<font color="#666666">=</font>&nbsp;threading<font color="#666666">.</font>Thread(target<font color="#666666">=</font>callback,&nbsp;args<font color="#666666">=</font>task)
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;t<font color="#666666">.</font>start()
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;pool<font color="#666666">.</font>append(t)

&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>if</strong></font>&nbsp;<font color="#AA22FF"><strong>not</strong></font>&nbsp;pool:
&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;<font color="#008000"><strong>return</strong></font>

&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;&nbsp;time<font color="#666666">.</font>sleep(<font color="#666666">0.01</font>)
</code></blockquote><p><font color="gray">Colored with <a href="http://dumpz.org">dumpz.org</a></p>