import threading import time from Queue import Queue, Empty as QueueEmpty def make_work(callback, tasks, limit, results=None): """ Run up to limit threads, do tasks. Put results to results queue if it is specified. """ pool = [] if not isinstance(tasks, Queue): new_tasks = Queue() for x in tasks: new_tasks.put(x) tasks = new_tasks # If callback result should be placed to the results queue # wrap the callback to the function that will catch result and # put it to the queue if results: old_callback = callback def callback(*args): results.put(old_callback(*args)) while True: pool = filter(lambda x: x.isAlive(), pool) for x in xrange(limit - len(pool)): try: task = tasks.get(False) except QueueEmpty: pass else: if not isinstance(task, (list, tuple)): task = [task] t = threading.Thread(target=callback, args=task) t.start() pool.append(t) if not pool: return time.sleep(0.01)