usr bin python coding utf-8 from optparse import OptionParser from gra

#!/usr/bin/python
# -*- coding: utf-8 -*-
from optparse import OptionParser
from grab import Grab
from grab.spider import Spider, Task
from grab.tools.logs import default_logging
from grab.spider.base import logger_verbose
import logging
import json

from spiders.market_categories import MarketCategorySpider

from config import spider_params, Session


def load_cookies_from(filename):
    cookies = {}
    try:
        with open(filename) as js:
            for cookie in json.loads(js.read()):
                cookies[cookie["name"]] = cookie["value"]
    except Exception:
        logging.debug('Cannot open file %s', filename)
        return {}
    return cookies


if __name__ == '__main__':
    default_logging(grab_log='/tmp/grab.log', level=10, mode='a', propagate_network_logger=True,
                    network_log='/tmp/grab.network.log')
    bot = MarketCategorySpider(**spider_params())
    grab_cookies = load_cookies_from('cookie.json')
    

    bot.setup_grab(timeout=100, connect_timeout=100,
                   debug=True,
                   encoding='gzip',
                   # log_dir='/tmp/responses',
                   # reuse_cookies=False,
                   # user_agent_file='user-agent.txt',
                   cookies=grab_cookies
    )
    try:
        bot.run()
    except KeyboardInterrupt:
        pass
    print bot.render_stats()