class Command LoggingBaseCommand help Find and run parsers def handle

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
class Command(LoggingBaseCommand):
help = 'Find and run parsers.'
def handle(self, *args, **options):
now = timezone.now()
self.log('Run parsers...')
parsers_qs = Parser.objects.filter(
status__in=(Parser.STATUS_OK, Parser.STATUS_WAIT_PARSE), is_active=True,
).only('source_site', 'name', 'period', 'last_parse_dt')
parsers = []
for t in parsers_qs:
if not t.status == Parser.STATUS_WAIT_PARSE:
np = t.get_next_parse_dt()
if np and np > now:
continue
parsers.append(t)
if not parsers:
self.log('End. Not found parsers to run.', double_br=True)
return
os.chdir(settings.BASE_DIR)
command_base = [sys.executable, 'manage.py', 'go_parse']
processes = []
for parser in parsers:
command = copy(command_base)
command.append(str(parser.pk))
self.log(u'Run parse #%s "%s"...' % (parser.pk, parser.get_full_name()))
if settings.DEBUG:
processes.append(subprocess.Popen(command))
else:
processes.append(subprocess.Popen(command, stderr=DEVNULL, stdout=DEVNULL))
for process in processes:
process.wait()
self.log('End.', double_br=True)