python 布隆过滤器

Posted 2021-03-08 眼镜儿

import hashlib
import redis


class SimpleHash(object):
    def __init__(self, cap, seed):
        self.cap = cap
        self.seed = seed

    def hash(self, value):
        ret = 0
        for i in range(len(value)):
            ret += self.seed * ret + ord(value[i])
        return (self.cap - 1) & ret


class BloomFilter(object):
    def __init__(self, redis_obj, block_num=1, key=‘bloomfilter‘):
        """
        :param block_num: one block_num for about 90,000,000; if u have more strings for filtering, increase it.
        """
        self.server = redis_obj
        self.bit_size = 1 << 31 # redis的string类型最大容量为512M,现在使用256M
        self.seeds = [5,7,11,1,3,31,37,61]
        self.key = key
        self.block_num = block_num
        self.hashfunc = []
        for seed in self.seeds:
            self.hashfunc.append(SimpleHash(self.bit_size, seed))

    def is_contains(self, str_input):
        if not str_input:
            return False

        md5 = hashlib.md5()
        md5.update(str_input.encode())
        str_input = md5.hexdigest()
        ret = True
        name = self.key + str(int(str_input[0:2], 16) % self.block_num)
        print(str_input[0:2])
        print(int(str_input[0:2], 16))
        print(int(str_input[0:2], 16) % self.block_num)
        for f in self.hashfunc:
            loc = f.hash(str_input)

            ret = ret & self.server.getbit(name, loc)
        return ret

    def insert(self, str_input):
        md5 = hashlib.md5()
        md5.update(str_input.encode())
        str_input = md5.hexdigest()
        name = self.key + str(int(str_input[0:2], 16) % self.block_num)
        for f in self.hashfunc:
            loc = f.hash(str_input)
            self.server.setbit(name, loc, int)

if __name__ == ‘__main__‘:
    """第一次运行时会显示 not exists!，之后再运行会显示 exists!"""
    redis = redis.from_url(‘redis://127.0.0.1‘)
    bf = BloomFilter(redis)
    for i in range(10):
        url = ‘http://www.baidu.com?bai=2132{}‘.format(i)

        if bf.is_contains(url):
            print(‘exists!‘)
        else:
            print(‘not exists‘)
            bf.insert(url)