Python
18 May 2009 HTML Text
 
модуль для распараллеливания работы
 
 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
import threading
import time
from Queue import Queue, Empty as QueueEmpty

def make_work(callback, tasks, limit, results=None):
    """
    Run up to limit threads, do tasks.

    Put results to results queue if it is specified.
    """

    pool = []

    if not isinstance(tasks, Queue):
        new_tasks = Queue()
        for x in tasks:
            new_tasks.put(x)
        tasks = new_tasks

    # If callback result should be placed to the results queue
    # wrap the callback to the function that will catch result and
    # put it to the queue
    if results:
        old_callback = callback
        def callback(*args):
            results.put(old_callback(*args))
    
    while True:
        pool = filter(lambda x: x.isAlive(), pool)

        for x in xrange(limit - len(pool)):
            try:
                task = tasks.get(False)
            except QueueEmpty:
                pass
            else:
                if not isinstance(task, (list, tuple)):
                    task = [task]
                t = threading.Thread(target=callback, args=task)
                t.start()
                pool.append(t)

        if not pool:
            return

        time.sleep(0.01)