class ScrapeSomethingSpide Spider def task_generator self grab self cr

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
class ScrapeSomethingSpider(Spider):
def task_generator(self):
grab = self.create_grab_instance()
url = 'http://www.gofuckbiz.com/memberlist.php'
grab.setup(url=url)
yield Task('index', grab=grab)
def subroutine(self, grab):
grab.setup(url='http://localhost/subtask1')
yield Task(grab=grab)
yield self.subsubroutine(grab)
grab.setup(url='http://localhost/subtask2')
yield Task(grab=grab)
def subsubroutine(self, grab):
grab.setup(url='http://localhost/subsubtask1')
yield Task(grab=grab)
grab.setup(url='http://localhost/subsubtask2')
yield Task(grab=grab)
@inline_task
def task_index(self, grab, task):
grab.setup(url='http://localhost/task1')
yield Task(grab=grab)
yield self.subroutine(grab)
grab.setup(url='http://localhost/task2')
yield Task(grab=grab)
yield self.subroutine(grab)
grab.setup(url='http://localhost/task3')
yield Task(grab=grab)
grab.setup(url='http://localhost/task4')
yield Task(grab=grab)