class TransformerApplyer def __init__ self transformers verbose False

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
class TransformerApplyer():
def __init__(self, transformers, verbose=False):
self.transformers = transformers
self.verbose = verbose
def apply(self, collections):
for transformer in self.transformers:
if self.verbose: print 'Train:\t\t' + str(transformer.__class__)
transformer.train(collections)
if self.verbose: print 'Apply:\t\t' + str(transformer.__class__)
transformer.apply(collections)
if self.verbose: print 'Finished:\t' + str(transformer.__class__) + '\n'
class Transformer():
def train(self, collection):
raise NotImplementedError
def apply(self, collection):
raise NotImplementedError
class MultiThreadMapTransformApplyer():
def __init__(self, core=1):
self.core = core
def train(self, collection):
raise NotImplementedError
def apply(self, collection):
if __name__ == '__main__':
mp_applyer = Pool(self.core)
collection.documents = mp_applyer.map(self.map, collection)
@classmethod
def map(cls, item):
raise NotImplementedError