usr bin python coding utf-8 from optparse import OptionParser from gra

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#!/usr/bin/python
# -*- coding: utf-8 -*-
from optparse import OptionParser
from grab import Grab
from grab.spider import Spider, Task
from grab.tools.logs import default_logging
from grab.spider.base import logger_verbose
import logging
import json
from spiders.market_categories import MarketCategorySpider
from config import spider_params, Session
def load_cookies_from(filename):
cookies = {}
try:
with open(filename) as js:
for cookie in json.loads(js.read()):
cookies[cookie["name"]] = cookie["value"]
except Exception:
logging.debug('Cannot open file %s', filename)
return {}
return cookies
if __name__ == '__main__':
default_logging(grab_log='/tmp/grab.log', level=10, mode='a', propagate_network_logger=True,
network_log='/tmp/grab.network.log')
bot = MarketCategorySpider(**spider_params())
grab_cookies = load_cookies_from('cookie.json')
bot.setup_grab(timeout=100, connect_timeout=100,
debug=True,
encoding='gzip',
# log_dir='/tmp/responses',
# reuse_cookies=False,
# user_agent_file='user-agent.txt',
cookies=grab_cookies
)
try:
bot.run()
except KeyboardInterrupt:
pass
print bot.render_stats()