модуль для распараллеливания работы

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import threading
import time
from Queue import Queue, Empty as QueueEmpty
def make_work(callback, tasks, limit, results=None):
"""
Run up to limit threads, do tasks.
Put results to results queue if it is specified.
"""
pool = []
if not isinstance(tasks, Queue):
new_tasks = Queue()
for x in tasks:
new_tasks.put(x)
tasks = new_tasks
# If callback result should be placed to the results queue
# wrap the callback to the function that will catch result and
# put it to the queue
if results:
old_callback = callback
def callback(*args):
results.put(old_callback(*args))
while True:
pool = filter(lambda x: x.isAlive(), pool)
for x in xrange(limit - len(pool)):
try:
task = tasks.get(False)
except QueueEmpty:
pass
else:
if not isinstance(task, (list, tuple)):
task = [task]
t = threading.Thread(target=callback, args=task)
t.start()
pool.append(t)
if not pool:
return
time.sleep(0.01)