usr bin env python from dja setup_django import go go __file__ import

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
#!/usr/bin/env python
from dja.setup_django import go
go(__file__, '/../..')
import time
import copy
from libpy.threads import Master, Slave
from domain.models import Domain
from dja.task.utils import build_storage
from dja.task.dump import Dump
from libpy.parser.google_bl import GoogleBL
from libpy.parser.google_pr import GooglePR
from libpy.parser.google_idx import GoogleIdx
from libpy.parser.google_trust import GoogleTrust
from libpy.parser.yahoo_bl import YahooBL
from libpy.parser.whois import Whois
tasks = (
#('google_bl', GoogleBL),
#('google_pr', GooglePR),
#('google_idx', GoogleIdx),
('yahoo_bl', YahooBL),
#('date_registered', Whois),
#('google_trust', GoogleTrust),
)
class MySlave(Slave):
def do_job(self, job):
for field, Parser in tasks:
resp = Parser(proxy=self.proxy).get(job.name, sleep=3)
if 'date_registered' == field:
setattr(job, field, resp.result and resp.result['date_registered'] or None)
else:
setattr(job, field, copy.copy(resp.result))
#setattr(job, '%s_dump' % field, copy.deepcopy(resp))
dump = Dump()
dump.message = resp.message
dump.head = resp.head
dump.body = 'adsf'#resp.body
dump.url = resp.url
setattr(job, '%s_dump' % field, dump)
self.message('field=[%s], result=[%s]' % (field, str(resp.result)))
del resp
print 'ok'
Domain.check_tasks.reset()
master = Master(build_storage(Domain, 'check'), MySlave,
slave_count=1, proxy_file='proxy.txt',
lock_file='var/domain_domain',
jobs_count=1)
master.run()