import gevent
import socket
import urllib2
from Queue import Queue, Empty
import logging
import time
import threading
from urlparse import urlsplit
import urllib
import os

socket.setdefaulttimeout(5)

THREAD_LIMIT = 100

def worker(url, results):
    try:
        info = urlsplit(url)
        logging.debug('Fetching %s' % url)
        data = urllib.urlopen(url).read()
        tname = threading.currentThread().name
        results.put(('%s-%s' % (info.hostname, tname), data))
    except Exception, ex:
        logging.error('', exc_info=ex)


def clean_dumps():
    for fname in os.listdir('dumps'):
        os.unlink('dumps/%s' % fname)


def main():
    threads = []
    results = Queue()
    tasks = open('urls.txt').read().splitlines()
    for url in tasks:
        t = threading.Thread(target=worker, args=[url, results])
        t.start()
        threads.append(t)
    while True:
        try:
            res = results.get_nowait()
            logging.debug('Received result')
            open('dumps/' + res[0], 'w').write(res[1])
        except Empty:
            if not len([x for x in threads if x.isAlive()]):
                break
            time.sleep(0.1)
       

if __name__ == '__main__':
    logging.basicConfig(level=logging.DEBUG, format='%(threadName)s %(message)s')
    handler = logging.FileHandler('logs/session.log', 'w')
    logging.getLogger().addHandler(handler)
    clean_dumps()
    start = time.time()
    main()
    logging.debug('Work done in %.2f seconds' % (time.time() - start))
    logging.debug('Results: %d' % len(os.listdir('dumps')))