hash functions comparison

 1
 2
 3
 4
 5
 6
 7
 8
 9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
import re
import matplotlib.pyplot as plt
from spooky import hash128, hash64, hash32
import mmh3
def hash_djb2(str):
hash = 5381
for c in str:
hash = ((hash << 5) + hash) + ord(c)
return hash
def hash_sdbm(str):
hash = 0
for c in str:
hash = ord(c) + (hash << 6) + (hash << 16) - hash
return hash
def hash_pjw(str):
h = 0
for c in str:
h = (h << 4) + ord(c)
g = h & 0xF0000000
if g != 0:
h ^= g >> 24
h ^= g
return h
emails = []
words = []
ip_addresses = []
filter = unicode('[0]')
for line in open("strings.txt", "r"):
line = unicode(line, encoding='utf-8')
line.__hash__()
parts = re.split("[\s]+", line)
first_part = parts[0]
second_part = parts[1]
if first_part == filter:
words.append(second_part.encode("UTF-8"))
counter = {}
output = open("output.txt", "w")
for word in words:
print>>output, word#.encode("UTF-8")
words.sort()
def count_hash_distribution(strings, table_size, hash_function):
hash_counter = [0] * table_size
for string in strings:
computed_hash = hash_function(string)
counter[string] = counter.get(computed_hash, 0) + 1
hash_counter[computed_hash % size] += 1
return hash_counter
for cnt in xrange(10, 11):
size = 2 ** cnt - 1
plt.figure()
x = range(0, size, 1)
y1 = count_hash_distribution(words, size, hash_djb2)
y2 = count_hash_distribution(words, size, hash_sdbm)
y3 = count_hash_distribution(words, size, hash_pjw)
y4 = count_hash_distribution(words, size, hash32)
y5 = count_hash_distribution(words, size, mmh3.hash)
#plt.plot(x, y1, 'r')
plt.plot(x, y2, 'g')
#plt.plot(x, y3, 'b')
plt.plot(x, y4, 'y')
plt.plot(x, y5, 'm')
plt.xlabel('hash slot')
plt.ylabel('words with this hash')
plt.title('distribution')
plt.show()