class RedisBucketFilter Redis filter with buckets Bucket is part of da

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
class RedisBucketFilter:
"""Redis filter with buckets
Bucket is part of data calculated as 3 first digit from murmur3 hash
from key.
>> mmh3.hash('B10E7oDEO6-', signed=False).to_bytes(4, byteorder='big').hex()
"""
def __init__(self, prefix, r=None):
self.__db = r
self.filter_prefix = str(prefix)
self.key_template = self.filter_prefix + ':{}:{}'
def _build_key(self, key):
bucket = str(mmh3.hash(key, signed=False))[:3]
return self.key_template.format(key, bucket)
def size(self):
"""Return amount of elements in all buckets"""
counter = 0
keys = self.__db.keys(self.filter_prefix + ':*')
for key in keys:
counter += self.__db.scard(key)
return counter
def inbucket(self, key):
"""Check if key already in bucked and add it if not.
Return True if key present False if not.
"""
k = self._build_key(key)
if self.__db.sismember(k, key):
return True
else:
self.__db.sadd(k, key)
return False
def test_redis_bucket_filter(redis_db): # noqa: F811
flt = RedisBucketFilter('test', redis_db)
key = flt._build_key('a')
assert key == 'test:a:100'
assert flt.inbucket('a') is False
assert flt.inbucket('a') is True
assert flt.inbucket('b') is False
assert flt.inbucket('c') is False
assert flt.size() == 3
flt.inbucket('d')
assert flt.size() == 4