class Command LoggingBaseCommand args parser_id help Run parser def ha

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
class Command(LoggingBaseCommand):
args = '<parser_id>'
help = 'Run parser.'
def handle(self, *args, **options):
try:
parser_id = int(args[0])
except (IndexError, ValueError, TypeError):
self.log('Args error.', double_br=True)
return
parser = get_object_or_None(Parser, pk=parser_id)
if parser is None:
self.log('Parser #%s not found.' % parser_id, double_br=True)
return
if parser.status not in (Parser.STATUS_OK, Parser.STATUS_WAIT_PARSE):
self.log('Parser #%s has not OK or WAIT_PARSE status.' % parser_id, double_br=True)
return
spider = PARSERS[parser.source_site](parser=parser, thread_number=THREADS[parser.source_site])
parser.status = Parser.STATUS_PARSE
parser.save(update_fields=('status',))
self.log('Run parser #%s %s...' % (parser_id, spider.__class__.__name__))
try:
spider.run()
if hasattr(spider, 'render_stats'):
self.log(spider.render_stats())
except Exception, e:
open(os.path.join(settings.LOG_DIR, 'errors.log'), 'a').write(traceback.format_exc())
print traceback.format_exc()
self.log('parse error.')
parser.status = Parser.STATUS_ERROR
parser.save(update_fields=('status',))
raise e
parser.status = Parser.STATUS_OK
parser.last_parse_dt = timezone.now()
parser.save(update_fields=('status', 'last_parse_dt'))
self.log('End.', double_br=True)