from grab import Grab from grab spider import Spider Task import loggi

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
from grab import Grab
from grab.spider import Spider, Task
import logging
class TestSpider(Spider):
def task_generator(self):
url = 'https://www.emex.ru/find?shkey.PGr=Original'\
'&QueryDetail=9091901164&shkey.SearchedDetailNum=9091901164'\
'&shkey.CustomGroupKey=-'
for x in xrange(10):
yield Task('page', url=url)
def task_page(self, grab, task):
print grab.response.code
if __name__ == '__main__':
logging.basicConfig(level=logging.DEBUG)
bot = TestSpider(thread_number=5)
bot.load_proxylist('/web/proxy.txt', 'text_file')
bot.run()