import sys import json from grab spider import Spider Task import logg

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
import sys
import json
from grab.spider import Spider, Task
import logging
class ExampleSpider(Spider):
def task_generator(self):
links = json.loads(open(sys.argv[1], "r").read())
for link in links:
url = 'http://localhost:8000%s' % link
yield Task('search', url=url, cookies={'protected': 'b327593e-adc1-4296-aee4-fd721fc64b67'})
def task_search(self, grab, task):
print("test")
logging.basicConfig(level=logging.DEBUG)
bot = ExampleSpider()
bot.run()