From c17877025f8296b6040eee1593da88eee25014bf Mon Sep 17 00:00:00 2001 From: Tomas Vondra Date: Mon, 2 Nov 2020 23:55:28 +0100 Subject: [PATCH 6/8] use one-hash bloom variant one-hash tweaks tweak generation of primes --- src/backend/access/brin/brin_bloom.c | 179 ++++++++++++++++++++++++--- 1 file changed, 159 insertions(+), 20 deletions(-) diff --git a/src/backend/access/brin/brin_bloom.c b/src/backend/access/brin/brin_bloom.c index f7b405f76f..87a3d5598a 100644 --- a/src/backend/access/brin/brin_bloom.c +++ b/src/backend/access/brin/brin_bloom.c @@ -208,6 +208,9 @@ typedef struct BloomOptions #define BLOOM_DEFAULT_FALSE_POSITIVE_RATE 0.01 /* 1% fp rate */ #define BLOOM_DEFAULT_SORT_MODE true /* start in sort */ +/* With the minimum allowed false positive rate of 0.001, we need up to 10 hashes */ +#define BLOOM_MAX_NUM_PARTITIONS 10 + #define BloomGetNDistinctPerRange(opts) \ ((opts) && (((BloomOptions *) (opts))->nDistinctPerRange != 0) ? \ (((BloomOptions *) (opts))->nDistinctPerRange) : \ @@ -279,6 +282,7 @@ typedef struct BloomFilter uint8 nhashes; /* number of hash functions */ uint32 nbits; /* number of bits in the bitmap (size) */ uint32 nbits_set; /* number of bits set to 1 */ + uint32 partlens[BLOOM_MAX_NUM_PARTITIONS]; /* partition lengths */ /* data of the bloom filter (used both for sorted and hashed phase) */ char data[FLEXIBLE_ARRAY_MEMBER]; @@ -288,6 +292,127 @@ typedef struct BloomFilter static BloomFilter *bloom_switch_to_hashing(BloomFilter *filter); +/* + * generate_primes + * returns array of all primes less than limit + * + * WIP: very naive prime sieve; could be optimized using segmented ranges + */ +static uint32 * +generate_primes(int limit) +{ + /* upper bound of number of primes below limit */ + /* WIP: reference for this number */ + int numprimes = 1.26 * limit / log(limit); + + bool *is_composite = (bool *) palloc0(limit * sizeof(bool)); + uint32 *primes = (uint32 *) palloc0(numprimes * sizeof(uint32)); + + int maxfactor = floor(sqrt(limit)); + int factor = 2; /* first prime */ + + /* mark the sieve where the index is composite */ + while (factor < maxfactor) + { + for (int i = factor * factor; i < limit; i += factor) + is_composite[i] = true; + do { factor++; } while (is_composite[factor]); + } + + /* the unmarked numbers are prime, so copy over */ + for (int i = 2, j = 0; i < limit && j < numprimes; i++) + { + if (!is_composite[i]) + primes[j++] = i; + } + + /* there should still be some zeroes at the end, but make sure */ + primes[numprimes - 1] = 0; + + /* pretty large, so free it now (segmented ranges would make it smaller) */ + pfree(is_composite); + return primes; +} + +/* + * set_bloom_partitions + * Calculate k moduli for one-hashing bloom filter. + * + * Find consecutive primes whose sum is close to nbits and + * return the sum. Copy the primes to the filter to use as + * partition lengths. + * WIP: one-hashing bf paper ref somewhere + */ +static uint32 +set_bloom_partitions(int nbits, uint8 nhashes, uint32 *partlens) +{ + int min, diff, incr; + int pidx = 0; + int sum = 0; + + /* we want partitions roughly with this length */ + int target_partlen = nbits / nhashes; + uint32 *primes; + + /* + * Generate primes up to a maximum value, based on the target length. + * How much higher it needs to be is based on gaps between primes, + * as shown at: https://primes.utm.edu/notes/gaps.html + * + * The largest possible filter is 32kB (the largest page size), i.e. + * ~262kb. Assuming a single partition of this length, the maximum + * gap is 95. With more partitions the gaps would be smaller, so we + * just use this as an upper boundary, and multiply it with the max + * number of partitions. + * + * Note: This is an overkill, but the impact on CPU time is minimal, + * particularly with "regular" filter sizes. If needed, we can make + * this more efficient in the future. + */ + primes = generate_primes(target_partlen + BLOOM_MAX_NUM_PARTITIONS * 100); + + /* + * In our array of primes, find a sequence of length nhashes, whose + * last item is close to our target partition length. The end of the + * array will be filled with zeros, so we need to guard against that. + */ + while (primes[pidx + nhashes - 1] <= target_partlen && + primes[pidx + nhashes] > 0) + pidx++; + + for (int i = 0; i < nhashes; i++) + sum += primes[pidx + i]; + + /* + * Since all the primes are less than or equal the desired partition + * length, the sum is somewhat less than nbits. Increment the starting + * point until we find the sequence of primes whose sum is closest to + * nbits. It doesn't matter whether it's higher or lower. + */ + min = abs(nbits - sum); + for (;;) + { + incr = primes[pidx + nhashes] - primes[pidx]; + diff = abs(nbits - (sum + incr)); + if (diff >= min) + break; + + min = diff; + sum += incr; + pidx++; + } + + memcpy(partlens, &primes[pidx], nhashes * sizeof(uint32)); + + /* WIP: assuming it's not important to pfree primes */ + + /* + * The actual filter length will be the sum of the partition lengths + * rounded up to the nearest byte. + */ + return (uint32) ((sum + 7) / 8) * 8; +} + /* * bloom_init * Initialize the Bloom Filter, allocate all the memory. @@ -319,6 +444,7 @@ bloom_init(bool sort_mode, int ndistinct, double false_positive_rate) */ k = log(2.0) * m / ndistinct; k = (k - floor(k) >= 0.5) ? ceil(k) : floor(k); + k = Min(k, BLOOM_MAX_NUM_PARTITIONS); /* * When sort phase is enabled, start with a small filter which we grow @@ -338,7 +464,9 @@ bloom_init(bool sort_mode, int ndistinct, double false_positive_rate) filter->flags = 0; filter->nhashes = (int) k; - filter->nbits = m; + + /* calculate the partition lengths and adjust m to match */ + filter->nbits = set_bloom_partitions(m, k, filter->partlens); if (!sort_mode) filter->flags |= BLOOM_FLAG_PHASE_HASH; @@ -463,7 +591,7 @@ static BloomFilter * bloom_add_value(BloomFilter *filter, uint32 value, bool *updated) { int i; - uint32 h1, h2; + int part_boundary = 0; /* assume 'not updated' by default */ Assert(filter); @@ -532,17 +660,16 @@ bloom_add_value(BloomFilter *filter, uint32 value, bool *updated) /* we better be in the hashing phase */ Assert(BLOOM_IS_HASHED(filter)); - /* compute the hashes, used for the bloom filter */ - h1 = hash_uint32_extended(value, 0x71d924af) % filter->nbits; - h2 = hash_uint32_extended(value, 0xba48b314) % filter->nbits; - /* compute the requested number of hashes */ for (i = 0; i < filter->nhashes; i++) { - /* h1 + h2 + f(i) */ - uint32 h = (h1 + i * h2) % filter->nbits; - uint32 byte = (h / 8); - uint32 bit = (h % 8); + int partlen = filter->partlens[i]; + int bitloc = part_boundary + (value % partlen); + + int byte = (bitloc / 8); + int bit = (bitloc % 8); + + Assert(bitloc < filter->nbits); /* if the bit is not set, set it and remember we did that */ if (! (filter->data[byte] & (0x01 << bit))) @@ -552,6 +679,9 @@ bloom_add_value(BloomFilter *filter, uint32 value, bool *updated) if (updated) *updated = true; } + + /* next bit */ + part_boundary += partlen; } return filter; @@ -581,6 +711,7 @@ bloom_switch_to_hashing(BloomFilter *filter) newfilter->nhashes = filter->nhashes; newfilter->nbits = filter->nbits; + memcpy(newfilter->partlens, filter->partlens, filter->nhashes * sizeof(uint32)); newfilter->flags |= BLOOM_FLAG_PHASE_HASH; SET_VARSIZE(newfilter, len); @@ -605,7 +736,7 @@ static bool bloom_contains_value(BloomFilter *filter, uint32 value) { int i; - uint32 h1, h2; + int part_boundary = 0; Assert(filter); @@ -634,21 +765,23 @@ bloom_contains_value(BloomFilter *filter, uint32 value) /* now the regular hashing mode */ Assert(BLOOM_IS_HASHED(filter)); - /* calculate the two hashes */ - h1 = hash_uint32_extended(value, 0x71d924af) % filter->nbits; - h2 = hash_uint32_extended(value, 0xba48b314) % filter->nbits; - /* compute the requested number of hashes */ for (i = 0; i < filter->nhashes; i++) { - /* h1 + h2 + f(i) */ - uint32 h = (h1 + i * h2) % filter->nbits; - uint32 byte = (h / 8); - uint32 bit = (h % 8); + int partlen = filter->partlens[i]; + int bitloc = part_boundary + (value % partlen); + + int byte = (bitloc / 8); + int bit = (bitloc % 8); + + Assert(bitloc < filter->nbits); /* if the bit is not set, the value is not there */ if (! (filter->data[byte] & (0x01 << bit))) return false; + + /* next bit */ + part_boundary += partlen; } /* all hashes found in bloom filter */ @@ -1081,8 +1214,14 @@ brin_bloom_summary_out(PG_FUNCTION_ARGS) if (BLOOM_IS_HASHED(filter)) { - appendStringInfo(&str, "mode: hashed nhashes: %u nbits: %u nbits_set: %u", + appendStringInfo(&str, + "mode: hashed nhashes: %u nbits: %u nbits_set: %u partition lengths: [", filter->nhashes, filter->nbits, filter->nbits_set); + for (int i = 0; i < filter->nhashes - 1; i++) + { + appendStringInfo(&str, "%u, ", filter->partlens[i]); + } + appendStringInfo(&str, "%u]", filter->partlens[filter->nhashes - 1]); } else { -- 2.26.2