From 611bf40a82f0b0884160a8201515e7ba767082b9 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 5 Aug 2023 15:53:41 +0200 Subject: [PATCH 01/92] add comments --- CMakeLists.txt | 3 + include/pasta/block_tree/block_tree.hpp | 36 ++++ .../block_tree/construction/block_tree_fp.hpp | 188 ++++++++++++++++-- .../block_tree/utils/MersenneRabinKarp.hpp | 5 + 4 files changed, 215 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 9b0d497..db2ac71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -26,6 +26,9 @@ set(CMAKE_CXX_STANDARD_REQUIRED ON) project(pasta_block_tree) +# Generate compile_commands.json +set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "") + ## Build tests option(PASTA_BLOCK_TREE_BUILD_TESTS "Build blocktree's tests." OFF) diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index 3281cd3..951ee4d 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -38,6 +38,10 @@ namespace pasta { template class BlockTree { public: + /// @brief If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the tree will only contain levels 6 and below. bool CUT_FIRST_LEVELS = true; size_type tau_; size_type max_leaf_length_; @@ -526,25 +530,57 @@ template class BlockTree { return 0; } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. inline size_type leading_zeros(int32_t val) { return __builtin_clz(static_cast(val) | 1); } + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. inline size_type leading_zeros(int64_t val) { return __builtin_clzll(static_cast(val) | 1); } + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// void calculate_padding(int64_t &padding, int64_t text_length, int64_t &height, int64_t &blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. int64_t tmp_padding = this->s_ * this->tau_; int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) blk_size = tau_; + // While the tree does not cover the entire text, add a level while (tmp_padding < text_length) { tmp_padding *= this->tau_; blk_size *= this->tau_; h++; } + // once the tree has enough levels to cover the entire text, we set the + // tree's values height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. padding = tmp_padding - text_length; } diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 325d890..d44c54f 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -33,6 +33,28 @@ class BlockTreeFP : public BlockTree { public: size_type const_size = 0; size_type sigma_ = 0; + /// + /// @brief Prune this block and all of its children. + /// + /// This will set the pointers for each node that is pruned to PRUNED (== -2) + /// and will increase and decrease pointers for each new back block or pruned + /// back block respectively. + /// + /// @param[in] counter For each level (starting at the topmost level) and + /// block, contains the number of back blocks pointing to the block + /// @param[in] pointer For each level (starting at the topmost level) and + /// block, contains the earliest block index in which the content of this + /// @param[in] offset For each level (starting at the topmost level) and + /// block, contains the character offset at which this block's content can + /// @param[in] marked_tree For each level (starting at the topmost level) and + /// block, has a 1 if the block is internal, 0 if it is a back block. + /// @param[out] pruned_tree The resulting tree after pruning, with the same + /// content as marked_tree (as far as I can see this is unused) + /// @param[in] i The current level, 0 being the top level + /// @param[in] j The current block being pruned + /// @param[in] ranks Rank data structures for each bit vector in marked_tree. + /// @return true, if an this block was and stays internal, false otherwise + /// bool prune_block( std::vector> &counter, std::vector> &pointer, @@ -40,8 +62,9 @@ class BlockTreeFP : public BlockTree { std::vector &marked_tree, std::vector &pruned_tree, size_type i, size_type j, std::vector> &ranks) { - // string leaf children can always be pruned - // fully padded children don't exist and can be ignored/ sanity check + // String leaf children can always be pruned, + // since they are contained in the leaf string. + // Fully padded children don't exist and can be ignored/sanity check // assumes short circuit evaluation as compiler behaviour if (static_cast(i) >= marked_tree.size() || static_cast(j) >= marked_tree[i]->size()) { @@ -49,31 +72,40 @@ class BlockTreeFP : public BlockTree { } bool marked_children = false; - // we already incremented counters for unmarked blocks during a previous - // step and now only need to consider marked blocks + // We already incremented counters for back blocks during construction + // and now only need to consider internal blocks if ((*marked_tree[i])[j] == 1) { - // inverse postorder dfs - // prune_block returns true if a marked block stays marked false otherwise + // inverse postorder dfs. We handle children from back to front. + // We need the rank over the internal nodes of this level, + // because only the internal nodes generated children on the + // next level. size_type rank_blk = ranks[i].rank1(j); for (size_type k = this->tau_ - 1; k >= 0; k--) { marked_children |= prune_block(counter, pointer, offset, marked_tree, pruned_tree, i + 1, rank_blk * this->tau_ + k, ranks); } - // conditions to be pruned are no marked children, no pointers pointing to - // me and a former occurrence in S + // Conditions to be pruned are: + // - no internal children, + // - no pointers pointing to me + // - an earlier occurrence in the text if (!marked_children && counter[i][j] == 0 && pointer[i][j] != NO_FORMER_OCC) { - + // This block is no longer internal (*marked_tree[i])[j] = 0; + // Since this is a back block now, we need to increment the counters for + // the blocks this back block now points to counter[i][pointer[i][j]]++; if (offset[i][j] > 0) { counter[i][pointer[i][j] + 1]++; } + // We only remove children if we're not at the last level if (static_cast(i + 1) < counter.size()) { - // remove all of its children by decrementing counters and marking - // them as PRUNED + // Remove all of its children by decrementing counters + // and marking them as PRUNED for (size_type k = this->tau_ - 1; k >= 0; k--) { + // If this child node actually exists, + // decrement the counters of the blocks the child points to and set if (static_cast(rank_blk * this->tau_) + k < counter[i + 1].size()) { auto ptr_child = pointer[i + 1][rank_blk * this->tau_ + k]; @@ -81,16 +113,36 @@ class BlockTreeFP : public BlockTree { if (offset[i + 1][rank_blk * this->tau_ + k] > 0) { counter[i + 1][ptr_child + 1]--; } + // Set this child's pointer to PRUNED pointer[i + 1][rank_blk * this->tau_ + k] = PRUNED; } } } } } + // If this node has internal children, has other blocks pointing to itself + // or has no earlier occurrence, then it remains internal return marked_children || counter[i][j] > 0 || pointer[i][j] == NO_FORMER_OCC; }; + /// + /// @brief Prunes the block tree + /// + /// @param[in] counter For each level (starting at the topmost level) and + /// block, contains the number of back blocks pointing to the block + /// @param[in] pointer For each level (starting at the topmost level) and + /// block, contains the earliest block index in which the content of this + /// block can be found. + /// @param[in] offset For each level (starting at the topmost level) and + /// block, contains the character offset at which this block's content can + /// be found in the block the back-pointer points to. + /// @param[in] marked_tree For each level (starting at the topmost level) and + /// block, has a 1 if the block is internal, 0 if it is a back block. + /// @param[out] pruned_tree The resulting tree after pruning, with the same + /// content as marked_tree + /// @return 0 + /// int32_t pruning_extended(std::vector> &counter, std::vector> &pointer, std::vector> &offset, @@ -101,6 +153,7 @@ class BlockTreeFP : public BlockTree { ranks.push_back(pasta::RankSelect(*bv)); } auto &top_lvl = *marked_tree[0]; + /// Prune the blocks on the top level from back to front for (size_type j = top_lvl.size() - 1; j >= 0; j--) { prune_block(counter, pointer, offset, marked_tree, pruned_tree, 0, j, ranks); @@ -185,25 +238,45 @@ class BlockTreeFP : public BlockTree { int32_t init_extended(std::vector &text) { static constexpr uint128_t kPrime = 2305843009213693951ULL; + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size int64_t added_padding = 0; + /// The height of the tree int64_t tree_max_height = 0; + /// The size of the largest blocks (i.e. the top level blocks) int64_t max_blk_size = 0; + /// For each level (starting at the top) contains the text start indices of + /// each block on the level std::vector> blk_lvl; + /// For each level and each block, contains the first block index at which + /// the content of this block appears std::vector> pass1_pointer; + /// For each level and each block, contains the offset at which + /// the content of this block appears in its back-pointed block std::vector> pass1_offset; + /// For each level contains a bit vector containing a 1 for each block that + /// is internal and a 0 for each back block std::vector bv_marked; + /// For every level and block counts how many back blocks are pointing to + /// the block std::vector> counter; std::vector pass2_ones; + /// The block size for each level, starting at the top level std::vector block_size_lvl_temp; this->calculate_padding(added_padding, text.size(), tree_max_height, max_blk_size); auto is_padded = added_padding > 0 ? 1 : 0; + /// The current block size starting at the top level int64_t block_size = max_blk_size; + /// The text start indices of each block on the current level std::vector block_text_inx; for (uint64_t i = 0; i < text.size(); i += block_size) { block_text_inx.push_back(i); } + // if the blocks on the current level are already below the max leaf length, + // we may not divide them further. So the entire block tree just consists of + // the current level verbatim, no pointers if (block_size <= this->max_leaf_length_) { auto *bv = new pasta::BitVector(block_text_inx.size(), 1); this->block_tree_types_rs_.push_back( @@ -225,20 +298,33 @@ class BlockTreeFP : public BlockTree { } while (block_size > this->max_leaf_length_) { block_size_lvl_temp.push_back(block_size); + /// Marks whether a block should be internal or not auto *bv = new pasta::BitVector(block_text_inx.size(), false); + /// If left[i] == 1, then there is an earlier occurrence of + /// block[i]block[i+1] auto left = pasta::BitVector(block_text_inx.size(), false); + /// If right[i] == 1, then there is an earlier occurrence of + /// block[i-1]block[i] auto right = pasta::BitVector(block_text_inx.size(), false); auto pair_size = 2 * block_size; + // Check, whether the last block's end extends past the end of the text auto last_block_padded = static_cast(block_text_inx[block_text_inx.size() - 1] + block_size) != text.size() ? 1 : 0; + // map block pair hashes to the text index of their occurrences + // collecting duplicates in a vector TODO std::unordered_map, std::vector> pairs( 0); + // map block hashes to their *block* index, + // collecting duplicates in a vector TODO std::unordered_map, std::vector> blocks = std::unordered_map, std::vector>(); + // iterate through all blocks on the current level, skipping over the last + // block if it is padded for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { + // Hash the current block and insert it into the block hash map auto index = block_text_inx[i]; MersenneRabinKarp rk_block = MersenneRabinKarp(text, sigma_, index, @@ -247,14 +333,22 @@ class BlockTreeFP : public BlockTree { MersenneHash(text, rk_block.hash_, index, block_size); blocks[mh_block].push_back(i); } - std::vector pointers(block_text_inx.size(), -1); + // Back pointers, offsets and the incoming-pointer-counters which we need + // for pruning later + std::vector pointers(block_text_inx.size(), NO_FORMER_OCC); std::vector offsets(block_text_inx.size(), 0); std::vector counters(block_text_inx.size(), 0); + // If a block pairs are larger than the whole text, then there is + // nothing really to do on this level. There cannot be any back pointers if (static_cast(pair_size) > text.size()) { block_size /= this->tau_; + // Start indices of blocks for the next level std::vector new_blocks(0); for (uint64_t i = 0; i < block_text_inx.size(); i++) { + // All of the current blocks are internal (*bv)[i] = 1; + // We split all blocks on the current level into tau sub-blocks but + // only create blocks that start before the end of the text for (size_type j = 0; j < this->tau_; j++) { if (static_cast(block_text_inx[i] + (j * block_size)) < text.size()) { @@ -262,6 +356,7 @@ class BlockTreeFP : public BlockTree { } } } + /// There are no pointers, offsets or counters on this level std::vector p(block_text_inx.size(), -1); std::vector o(block_text_inx.size(), 0); std::vector c(block_text_inx.size(), 0); @@ -273,6 +368,8 @@ class BlockTreeFP : public BlockTree { counter.push_back(c); continue; } + // Iterate through the block pairs and add them to the pair hash table + // along with their index *if they are consecutive* for (uint64_t i = 0; i < block_text_inx.size() - 1; i++) { if (block_text_inx[i] + block_size == block_text_inx[i + 1] && static_cast(block_text_inx[i] + pair_size) <= @@ -286,14 +383,18 @@ class BlockTreeFP : public BlockTree { pairs[mh_pair].push_back(i); } } - // find pairs + // Find the occurrences of all block pairs' contents MersenneRabinKarp rk_pair_sw = MersenneRabinKarp(text, sigma_, 0, pair_size, kPrime); + // Hash each window in the text of the size of a block pair + // and see if it corresponds to an actual block pair for (uint64_t i = 0; i < text.size() - pair_size; i++) { MersenneHash mh_sw = MersenneHash(text, rk_pair_sw.hash_, i, pair_size); if (pairs.find(mh_sw) != pairs.end()) { + // If the current hash corresponds to a hashed block pair, + // we update for those pairs, that they have an earlier occurrence for (auto b : pairs[mh_sw]) { if (i != static_cast(block_text_inx[b])) { left[b] = 1; @@ -308,18 +409,28 @@ class BlockTreeFP : public BlockTree { auto new_block_size = block_size / this->tau_; std::vector new_blocks(0); for (uint64_t i = 0; i < block_text_inx.size(); i++) { + /// This is true <=> the current block is adjacent + /// to its predecessor and successor bool surrounded = (i > 0 && i < block_text_inx.size() - 1) && block_text_inx[i] + old_block_size == block_text_inx[i + 1] && block_text_inx[i - 1] + old_block_size == block_text_inx[i]; + /// marked <=> not internal (ONLY HERE IT SEEMS) bool marked = false; + // if the block is adjacent to its predecessor and successor, then it + // must have a previous occurrence both as a left and right part of a + // block pair in order to be a back block. + // Otherwise, only consecutive neighboring blocks need to be considered. if (surrounded) { marked = left[i] && right[i]; } else { marked = left[i] || right[i]; } + // If either the block is internal or extends past the text end (i.e. is + // padded), then we create tau child nodes for this block if (!(marked) || static_cast(block_text_inx[i] + old_block_size) >= text.size()) { + // This block is internal (*bv)[i] = 1; for (size_type j = 0; j < this->tau_; j++) { if (static_cast(block_text_inx[i] + @@ -332,33 +443,52 @@ class BlockTreeFP : public BlockTree { MersenneRabinKarp rk_first_occ = MersenneRabinKarp( text, sigma_, block_text_inx[0], block_size, kPrime); + // Identify the first occurrence for each block on this level for (int64_t i = 0; static_cast(i) < block_text_inx.size() - 1; i++) { + // This is true <=> + // This is not the last block, + // this block is adjacent to the next and + // the next block is internal + // We need this, because the hasher overlaps the next block as well. bool followed = (static_cast(i) < block_text_inx.size() - 1) && block_text_inx[i] + block_size == block_text_inx[i + 1] && (*bv)[i + 1] == 1; + // If this block is internal if ((*bv)[i] == 1) { + // If the hasher's current position is currently not at the current + // block index, move it there if (rk_first_occ.init_ != static_cast(block_text_inx[i])) { rk_first_occ.restart(block_text_inx[i]); } if (followed) { + // We iterate through every window that starts in this block + // and ends before the end of the text. + // j is the offset into the current block for (int64_t j = 0; j < block_size && static_cast(block_text_inx[i] + j + block_size) < text.size(); j++) { + // Hash the window and try to find an earlier occurrence MersenneHash mh_first_occ = MersenneHash( text, rk_first_occ.hash_, block_text_inx[i] + j, block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { - // b cant be i and if j>0 then b cant follow on i (j>0) -> b > - // i + 1 (a -> b <=> not a or b) + // The if the current block (b) were i, it would reference + // itself. If j > 0 then the occurrence overlaps the block i + // + 1. Therefore, in that case b must be a block *past* i+1 if (b > i && (j <= 0 || b > i + 1)) { + // If all is well, set the back pointer and offsets pointers[b] = i; offsets[b] = j; if ((*bv)[b] == 0) { + // If b is a back block, then we have another block + // pointing to i counters[i]++; if (j > 0) { + // if the offset is greater than 0, the copied area + // extends into the next block counters[i + 1]++; } } @@ -369,6 +499,7 @@ class BlockTreeFP : public BlockTree { rk_first_occ.next(); } } else { + // If the next block is not adjacent, we only hash once MersenneHash mh_first_occ = MersenneHash( text, rk_first_occ.hash_, block_text_inx[i], block_size); if (blocks.find(mh_first_occ) != blocks.end()) { @@ -383,6 +514,7 @@ class BlockTreeFP : public BlockTree { } } } + // Add the values calculated on this level pass1_pointer.push_back(pointers); pass1_offset.push_back(offsets); counter.push_back(counters); @@ -392,8 +524,11 @@ class BlockTreeFP : public BlockTree { block_size = new_block_size; bv_marked.push_back(bv); } + // By this point, the first pass is done and we have an unpruned block tree this->leaf_size = block_size; block_size *= this->tau_; + // Prune the tree. Doing so will replace the pointers of pruned nodes with + // PRUNED pruning_extended(counter, pass1_pointer, pass1_offset, bv_marked, bv_marked); @@ -412,6 +547,7 @@ class BlockTreeFP : public BlockTree { bool found_back_block = top_level.size() != static_cast(ones_per_lvl[0]) || bv_marked.size() == 1; + // If there is a back block on the first level, add its values to the tree if (found_back_block || !this->CUT_FIRST_LEVELS) { this->block_tree_types_.push_back(&top_level); this->block_tree_types_rs_.push_back( @@ -436,11 +572,14 @@ class BlockTreeFP : public BlockTree { } else { delete bv_marked[0]; } - for (uint64_t i = 1; i < bv_marked.size(); i++) { + for (uint64_t i = 1; i < bv_marked.size(); i++) { + // If the previous level is padded, we need to be careful, since the last + // block possibly does not generate exactly tau children size_type new_size = (ones_per_lvl[i - 1] - is_padded) * this->tau_; auto last_block_parent = blk_lvl[i - 1][blk_lvl[i - 1].size() - 1]; auto lvl_block_size = block_size_lvl_temp[i]; + // Determine the number of children the last block generated if (is_padded) { for (uint64_t j = 0; j < static_cast(this->tau_); j++) { if (last_block_parent + j * lvl_block_size < text.size()) { @@ -448,23 +587,32 @@ class BlockTreeFP : public BlockTree { } } } + // Check if we have found a back block on the current level found_back_block |= new_size != ones_per_lvl[i]; + // If there is a back block, we add this level's data to the tree if (found_back_block || !this->CUT_FIRST_LEVELS) { + // is_internal auto bit_vector = new pasta::BitVector(new_size, 0); auto &bv_ref = *bit_vector; auto p = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); auto o = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); auto &ptr = *p; auto &off = *o; + // Maps block index => number of pruned blocks before this block std::unordered_map blocks_skipped; auto &lvl_pass1 = *bv_marked[i]; + // Number of non-pruned blocks so far size_type c = 0; size_type c_u = 0; for (uint64_t j = 0; j < lvl_pass1.size(); j++) { blocks_skipped[j] = j - c; - if (pass1_pointer[i][j] != -2) { + // If the current block is not pruned, add it to the new tree + if (pass1_pointer[i][j] != PRUNED) { + // Add it to the is_internal bit vector bv_ref[c] = (bool)lvl_pass1[j]; + // If it is a back block, add its pointer and offset if (!lvl_pass1[j]) { + // We need to ignore the pruned blocks ptr[c_u] = pass1_pointer[i][j] - blocks_skipped[pass1_pointer[i][j]]; off[c_u] = pass1_offset[i][j]; @@ -473,6 +621,7 @@ class BlockTreeFP : public BlockTree { c++; } } + // Add the new data to the tree this->block_tree_types_.push_back(&bv_ref); this->block_tree_types_rs_.push_back( new pasta::RankSelect(bv_ref)); @@ -482,15 +631,20 @@ class BlockTreeFP : public BlockTree { this->block_tree_offsets_.push_back(o); this->block_size_lvl_.push_back(block_size_lvl_temp[i]); } else { + // Otherwise, we don't need the data from this level anymore delete bv_marked[i]; } } + // Construct the leaf string int64_t leaf_count = 0; auto &last_level = (*bv_marked[bv_marked.size() - 1]); for (uint64_t i = 0; i < last_level.size(); i++) { if (last_level[i] == 1) { + // For every leaf on the last level, we have tau leaf blocks leaf_count += this->tau_; + // Iterate through all characters in this child and add them to the leaf + // string for (uint64_t j = 0; j < static_cast(this->leaf_size * this->tau_); j++) { if (static_cast(blk_lvl[blk_lvl.size() - 1][i] + j) < diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 4d5f9c4..d7b49f4 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -28,11 +28,16 @@ template class MersenneRabinKarp { __extension__ typedef unsigned __int128 uint128_t; public: + /// The text being hashed std::vector const &text_; uint128_t sigma_; + /// The start index of the currently hashed window uint64_t init_; + /// The window size of this hasher uint64_t length_; + /// A large prime used for modulus operations uint128_t prime_; + /// The current hash value uint64_t hash_; uint128_t max_sigma_; From 196a414754e6298ea6806ee341a345275e733567 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 8 Aug 2023 19:00:54 +0200 Subject: [PATCH 02/92] new sequential implementation (without pruning for now) --- CMakePresets.json | 153 +++-- include/pasta/block_tree/block_tree.hpp | 15 +- .../block_tree/construction/block_tree_fp.hpp | 3 +- .../construction/block_tree_fp2_seq.hpp | 596 ++++++++++++++++++ .../block_tree/utils/MersenneRabinKarp.hpp | 5 + tests/CMakeLists.txt | 1 + tests/block_tree/block_tree_seq_test.cpp | 88 +++ 7 files changed, 792 insertions(+), 69 deletions(-) create mode 100644 include/pasta/block_tree/construction/block_tree_fp2_seq.hpp create mode 100644 tests/block_tree/block_tree_seq_test.cpp diff --git a/CMakePresets.json b/CMakePresets.json index 7282ac3..fc5dcda 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,65 +1,94 @@ { - "version": 2, - "cmakeMinimumRequired": { - "major": 3, - "minor": 20, - "patch": 0 + "version": 2, + "cmakeMinimumRequired": { + "major": 3, + "minor": 20, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default", + "description": "Default build options", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", + "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" + } }, - "configurePresets": [ - { - "name": "default", - "displayName": "Default", - "description": "Default build options", - "hidden": true, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", - "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" - } - }, - { - "name": "release", - "displayName": "Release", - "inherits": "default", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "relwithdeb", - "displayName": "ReleaseWithDebugInfo", - "inherits": "default", - "binaryDir": "${sourceDir}/build_with_debug_info", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo" - } - }, - { - "name": "debug", - "displayName": "Debug", - "inherits": "default", - "binaryDir": "${sourceDir}/debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - } - ], - "buildPresets": [ - { - "name": "release", - "configurePreset": "release" - }, - { - "name": "relwithdeb", - "configurePreset": "relwithdeb" - }, - { - "name": "debug", - "configurePreset": "debug" - } - ] + { + "name": "release", + "displayName": "Release", + "inherits": "default", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "relwithdeb", + "displayName": "ReleaseWithDebugInfo", + "inherits": "default", + "binaryDir": "${sourceDir}/build_with_debug_info", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + } + }, + { + "name": "debug", + "displayName": "Debug", + "inherits": "default", + "binaryDir": "${sourceDir}/debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "ninja-multi", + "displayName": "Ninja Multi-Config", + "description": "Default build using Ninja Multi-Config generator", + "generator": "Ninja Multi-Config", + "binaryDir": "${sourceDir}/build_multi", + "cacheVariables": { + "PASTA_BLOCK_TREE_BUILD_TESTS": "ON", + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", + "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" + } + } + ], + "buildPresets": [ + { + "name": "release", + "configurePreset": "release" + }, + { + "name": "relwithdeb", + "configurePreset": "relwithdeb" + }, + { + "name": "debug", + "configurePreset": "debug" + }, + { + "name": "release-multi", + "configurePreset": "ninja-multi", + "configuration": "Release" + }, + { + "name": "relwithdeb-multi", + "configurePreset": "ninja-multi", + "configuration": "RelWithDebInfo" + }, + { + "name": "debug-multi", + "configurePreset": "ninja-multi", + "configuration": "Debug" + } + ] } diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index 951ee4d..de69f2a 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -41,7 +41,8 @@ template class BlockTree { /// @brief If this is true, then the only levels of the tree start to be /// included starting at the first level that contains a back block /// - /// For example, if levels 0 to 5 do not contain any back blocks, then the tree will only contain levels 6 and below. + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. bool CUT_FIRST_LEVELS = true; size_type tau_; size_type max_leaf_length_; @@ -95,7 +96,7 @@ template class BlockTree { off = off % block_size; blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; } - return compressed_leaves_[blk_pointer * leaf_size + off]; + return decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; }; int64_t select(input_type c, size_type j) { @@ -405,12 +406,14 @@ template class BlockTree { void compress_leaves() { compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); for (size_t i = 0; i < this->leaves_.size(); ++i) { compress_map_[this->leaves_[i]] = 1; } - for (size_t i = 0, cur_val = 0; i < this->compress_map_.size(); ++i) { - size_t tmp = compress_map_[i]; - compress_map_[i] = cur_val; + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; cur_val += tmp; } @@ -683,7 +686,7 @@ template class BlockTree { return result; } - size_type map_unique_chars(std::vector &text) { + size_type map_unique_chars(const std::vector &text) { this->u_chars_ = 0; input_type i = 0; for (auto a : text) { diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index d44c54f..3d841b9 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -338,7 +338,7 @@ class BlockTreeFP : public BlockTree { std::vector pointers(block_text_inx.size(), NO_FORMER_OCC); std::vector offsets(block_text_inx.size(), 0); std::vector counters(block_text_inx.size(), 0); - // If a block pairs are larger than the whole text, then there is + // If a block pair is larger than the whole text, then there is // nothing really to do on this level. There cannot be any back pointers if (static_cast(pair_size) > text.size()) { block_size /= this->tau_; @@ -603,6 +603,7 @@ class BlockTreeFP : public BlockTree { auto &lvl_pass1 = *bv_marked[i]; // Number of non-pruned blocks so far size_type c = 0; + // size_type c_u = 0; for (uint64_t j = 0; j < lvl_pass1.size(); j++) { blocks_skipped[j] = j - c; diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp new file mode 100644 index 0000000..8f0779a --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -0,0 +1,596 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFP2 : public BlockTree { + + constexpr static size_type NO_EARLIER_OCC = -1; + constexpr static size_type PRUNED = -2; + + static constexpr size_t SIGMA = 256; + static constexpr uint128_t K_PRIME = 2305843009213693951ULL; + + using BitVector = pasta::BitVector; + // using Rank = pasta::FlatRank; + using Rank = pasta::RankSelect; + + template > + using HashMap = std::unordered_map; + + // using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; + // using RabinKarpHash = MersenneHash; + using RabinKarpHash = MersenneHash; + + template + using RabinKarpMap = HashMap; + + struct LevelData { + /// Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// The block from which a back block is copying + std::unique_ptr> pointers; + /// The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// The number of back blocks pointing to the block + std::unique_ptr> counters; + /// Block start indices + std::unique_ptr> block_starts; + /// The block size on this level + size_t block_size; + /// The index of the current level. First level is 0, second level is 1 etc. + size_t level_index; + /// The number of blocks on the current level + size_t num_blocks; + + inline LevelData(size_t level_index_, size_t block_size_, + size_t num_blocks_) + : is_internal(nullptr), is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), block_size(block_size_), + level_index(level_index_), num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool is_adjacent(size_t i) const { + assert(i < num_blocks - 1); + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + }; + + void construct(const std::vector &text) { + + const size_t text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData &top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (uint64_t i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + std::cout << "level " << level << ": " << std::endl; + auto begin = std::chrono::high_resolution_clock ::now(); + LevelData ¤t = levels.back(); + /* + const bool last_block_padded = + block_starts.back() + block_size == text_len; + */ + + scan_block_pairs(text, current, is_padded); + const auto scan_block_pair_time = + std::chrono::duration_cast( + std::chrono::high_resolution_clock ::now() - begin) + .count(); + begin = std::chrono::high_resolution_clock ::now(); + scan_blocks(text, current, is_padded); + const auto scan_block_time = + std::chrono::duration_cast( + std::chrono::high_resolution_clock ::now() - begin) + .count(); + begin = std::chrono::high_resolution_clock ::now(); + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(generate_next_level(text, current)); + } + const auto generate_time = + std::chrono::duration_cast( + std::chrono::high_resolution_clock ::now() - begin) + .count(); + + std::cout << "pair: " << scan_block_pair_time + << "ms, block: " << scan_block_time + << "ms, next: " << generate_time << "ms" << std::endl; + } + + make_tree(text, levels, padding); + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline size_t ceil_div(size_t x, size_t y) { return 1 + ((x - 1) / y); } + + /// Scan through the blocks pairwise in order to identify which blocks should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector &text, LevelData &level, + const bool is_padded) { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const size_t pair_size = 2 * block_size; + + if (num_blocks < 4) { + level.is_internal = std::make_unique(num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap> map(num_blocks - 1); + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + + { + RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); + for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + rk.restart((*level.block_starts)[i]); + RabinKarpHash hash = rk.current_hash(); + map[hash].push_back(i); + } + } + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); + for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + if (!level.is_adjacent(i)) { + continue; + } + scan_windows_in_block_pair(rk, map, markings, block_size); + } + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(num_blocks); + auto &is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_t i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } + level.is_internal_rank = std::make_unique(is_internal); + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData generate_next_level(const std::vector &text, + const LevelData &level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto &is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// @brief Scan through the windows starting in a block and mark them + /// accordingly if they represent the earliest occurrence of some block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// index. + /// @param markings A vector storing the marks on a block. Marks are 2-bit + /// integers. + /// If the MSB is set, that means that the content of the block and its + /// successor has an earlier occurrence. If the LSB being set means that + /// the content of the block and its predecessor has an earlier occurrence. + /// @param block_size The size of blocks on the current level. + static inline void scan_windows_in_block_pair( + RabinKarp &rk, RabinKarpMap> &map, + sdsl::int_vector<2> &markings, const size_t block_size) { + for (size_t offset = 0; offset < block_size; ++offset) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found_hash_ptr = map.find(current_hash); + if (found_hash_ptr == map.end()) { + continue; + } + auto &[block_pair_hash, block_indices] = *found_hash_ptr; + + // If the hash we found is just the first block pair in the hash map, + // then the first block pair has no earlier occurrence. + // So we want to skip that one + bool skip_first = current_hash.start_ == block_pair_hash.start_; + + // For all pairs with an earlier occurrence, + for (size_t i = skip_first; i < block_indices.size(); i++) { + auto block_index = block_indices[i]; + markings[block_index] = markings[block_index] | 0b10; + markings[block_index + 1] = markings[block_index + 1] | 0b01; + } + map.erase(found_hash_ptr); + rk.next(); + } + } + + /// @return The block start indices for the *next* level. + auto scan_blocks(const std::vector &s, LevelData &level_data, + const bool is_padded) { + const size_t block_size = level_data.block_size; + const size_t num_blocks = level_data.num_blocks; + const std::vector &block_starts = *level_data.block_starts; + + const BitVector &is_internal = *level_data.is_internal; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap>> links(num_blocks - 1); + for (size_t i = 0; i < num_blocks - is_padded; ++i) { + const RabinKarpHash hash = + RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) + .current_hash(); + auto entry = links.insert({hash, {false, {}}}); + auto &[found_hash, pair] = *entry.first; + auto &[was_handled, vec] = pair; + vec.emplace_back(i); + } + + // Hash every window and find the first occurrences for every block. + RabinKarp rk(s, SIGMA, 0, block_size, K_PRIME); + for (size_t current_block_index = 0; + current_block_index < num_blocks - is_padded; ++current_block_index) { + // We can skip this loop iteration if the current block is a back block + // Nothing is ever going to point to this anyway. + if (!is_internal[current_block_index]) { + continue; + } + + if (static_cast(rk.init_) != block_starts[current_block_index]) { + rk.restart(block_starts[current_block_index]); + } + + // This is true iff there exists a next block and it is not adjacent + const bool next_block_not_adjacent = + current_block_index < num_blocks - 1 && + !level_data.is_adjacent(current_block_index); + // If the next block is not adjacent, we just want to hash exactly this + // block. If it either is adjacent or we are at the end of the string, we + // take care not to hash windows that start beyond the end of the string + const size_t num_hashes = + next_block_not_adjacent + ? 1 + : block_size - saturating_sub(block_starts[current_block_index] + + block_size, + s.size()); + + scan_windows_in_block(rk, links, level_data, current_block_index, + num_hashes); + ++current_block_index; + } + } + + /// \brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// \param rk A Rabin-Karp hasher whose current state is at a block start. + /// \param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// \param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// \param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block( + RabinKarp &rk, + RabinKarpMap>> &links, + LevelData &level_data, const size_t current_block_index, + const size_t num_hashes) { + for (size_t offset = 0; offset < num_hashes; ++offset) { + const RabinKarpHash current_hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(current_hash); + // Only handle this hash if it does not exist and we have not handled it + // already + auto &[hash_is_handled, found_blocks] = found->second; + if (found == links.end() || hash_is_handled) { + continue; + } + const size_t num_found_blocks = found_blocks.size(); + for (size_t i = 1; i < num_found_blocks; ++i) { + int block_index = found_blocks[i]; + // link.source_block_index = current_block_index; + // link.offset = offset; + // There is only space for non-internal blocks in these vectors + // if (!is_internal[link.block_index]) { + // Get the index of the back block only considering back blocks + // const size_t back_block_index = + // is_internal_rank.rank0(link.block_index); + (*level_data.pointers)[block_index] = current_block_index; + (*level_data.offsets)[block_index] = offset; + //} + } + // We handled this hash, so we mark it as such + hash_is_handled = true; + // links.erase(found); + rk.next(); + } + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(std::vector text, std::vector &levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // TODO Maybe only create this if there is a back block + // Create first level + { + LevelData &top_level = levels.front(); + const size_t n = top_level.num_blocks; + const size_t num_internal = top_level.is_internal_rank->rank1(n); + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + } + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData &previous_level = levels[level_index - 1]; + size_type previous_level_num_blocks = previous_level.num_blocks; + LevelData &level = levels[level_index]; + size_type new_size = + (previous_level.is_internal_rank->rank1(previous_level_num_blocks) - + is_padded) * + this->tau_; + previous_level.is_internal.reset(); + previous_level.is_internal_rank.reset(); + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + for (uint64_t i = 0; i < static_cast(this->tau_); i++) { + // Count the number of blocks that still start before the text end + new_size += last_block_parent_start + i * block_size < text.size(); + } + } + previous_level.block_starts.reset(); + const size_t n = level.num_blocks; + // TODO This will break once pruning is implemented. + // Instead of using rank, the number of ones after pruning should be + // saved in a vec for example + const size_t num_internal = level.is_internal_rank->rank1(n); + auto is_internal = new BitVector(level.num_blocks); + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_non_pruned = 0; + size_t num_back_blocks = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + std::vector &num_blocks_skipped = *level.pointers; + for (size_t i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + num_blocks_skipped[i] = i - num_non_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + // If it is a back block, add its pointer and offset + const size_t offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - num_blocks_skipped[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + + // We don't need these anymore + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto &last_level = *this->block_tree_types_.back(); + std::vector &last_level_block_starts = + *levels.back().block_starts; + for (uint64_t i = 0; i < last_level.size(); i++) { + if (!last_level[i]) { + continue; + } + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (uint64_t j = 0; + j < static_cast(this->leaf_size * this->tau_); j++) { + if (static_cast(last_level_block_starts[i] + j) < + text.size()) { + this->leaves_.push_back(text[last_level_block_starts[i] + j]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Branch-less saturating subtraction. + /// + /// Source: http://locklessinc.com/articles/sat_arithmetic/ + /// + /// @param x The minuend. + /// @param y The subtrahend. + /// @return x - y. However, if the result would underflow, returns zero. + static inline constexpr auto saturating_sub(size_t x, size_t y) -> size_t { + size_t res = x - y; + res &= -(res <= x); + return res; + } + + // TODO prune + +public: + BlockTreeFP2(const std::vector &text, const size_t arity, + const size_t root_arity, const size_t max_leaf_length) { + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index d7b49f4..316b11e 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -20,6 +20,7 @@ #pragma once +#include "pasta/block_tree/utils/MersenneHash.hpp" #include namespace pasta { @@ -84,6 +85,10 @@ template class MersenneRabinKarp { // return (i >= prime_) ? i - prime_ : i; }; + inline MersenneHash current_hash() const { + return MersenneHash(text_, hash_, init_, length_); + } + void next() { if (text_.size() <= init_ + length_) { return; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eb76eb9..fcf2c25 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -35,6 +35,7 @@ include(CTest) find_package(GTest REQUIRED) pasta_block_tree_build_test(block_tree/block_tree_fp_test) +pasta_block_tree_build_test(block_tree/block_tree_seq_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_parallel_test) diff --git a/tests/block_tree/block_tree_seq_test.cpp b/tests/block_tree/block_tree_seq_test.cpp new file mode 100644 index 0000000..5e40224 --- /dev/null +++ b/tests/block_tree/block_tree_seq_test.cpp @@ -0,0 +1,88 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Florian Kurpicz + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#include +#include + +#include + +#include +#include + +class BlockTreeSeqTest : public ::testing::Test { + +protected: + std::vector text; + + pasta::BlockTreeFP2 *bt; + + void SetUp() override { + + std::random_device rd; + std::mt19937 gen(1); + std::uniform_int_distribution dist(0, 15); + + size_t const string_length = 100000; + text.resize(string_length); + for (size_t i = 0; i < text.size(); ++i) { + text[i] = dist(gen) + 10; + } + + auto ptr = + std::make_unique>(text, 4, 8, 4); + bt = ptr.release(); + bt->add_rank_support(); + } + + void TearDown() override { delete bt; } +}; + +TEST_F(BlockTreeSeqTest, access) { + for (size_t i = 0; i < text.size(); ++i) { + ASSERT_EQ(bt->access(i), text[i]) << "for index " << i; + } +} + +TEST_F(BlockTreeSeqTest, rank) { + std::array hist = {0}; + + for (size_t i = 0; i < text.size() - 1; ++i) { + ++hist[text[i]]; + ASSERT_EQ(bt->rank(text[i], i), hist[text[i]]); + } +} + +TEST_F(BlockTreeSeqTest, select) { + std::array hist = {0}; + + for (size_t i = 0; i < text.size() - 1; ++i) { + ++hist[text[i]]; + ASSERT_EQ(bt->select(text[i], hist[text[i]]), i); + } +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/******************************************************************************/ From 0e86066d7bd0329215993b80c0c0c88f2436a92f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 10 Aug 2023 22:02:14 +0200 Subject: [PATCH 03/92] remove branch in rabin karp --- .gitignore | 7 ++++- CMakeLists.txt | 28 +++++++++---------- .../construction/block_tree_fp2_seq.hpp | 25 ----------------- .../block_tree/utils/MersenneRabinKarp.hpp | 19 ++++--------- tests/block_tree/block_tree_fp_test.cpp | 19 +++++-------- 5 files changed, 32 insertions(+), 66 deletions(-) diff --git a/.gitignore b/.gitignore index 42afabf..92e0fa0 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ -/build \ No newline at end of file +/build* +/cmake-build-* +.cache +compile_commands.json +.idea/* +perf.data* diff --git a/CMakeLists.txt b/CMakeLists.txt index db2ac71..4849f70 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -33,10 +33,10 @@ set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "") option(PASTA_BLOCK_TREE_BUILD_TESTS "Build blocktree's tests." OFF) option(PASTA_BLOCK_TREE_BUILD_EXAMPLES - "Build blocktree's benchmarks." OFF) + "Build blocktree's benchmarks." OFF) # Optional test -if(PASTA_BLOCK_TREE_BUILD_TESTS) +if (PASTA_BLOCK_TREE_BUILD_TESTS) include(FetchContent) FetchContent_Declare( googletest @@ -48,13 +48,13 @@ if(PASTA_BLOCK_TREE_BUILD_TESTS) enable_testing() add_subdirectory(tests) include(GoogleTest) -endif() -if(PASTA_BLOCK_TREE_BUILD_EXAMPLES) - add_executable(example - examples/block_tree_construction.cpp) - target_link_libraries(example - pasta_block_tree) -endif() +endif () +if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) + add_executable(example + examples/block_tree_construction.cpp) + target_link_libraries(example + pasta_block_tree) +endif () set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) @@ -63,12 +63,12 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE - ${CMAKE_CURRENT_SOURCE_DIR}/include) + ${CMAKE_CURRENT_SOURCE_DIR}/include) target_link_libraries(pasta_block_tree INTERFACE - libsais - pasta_bit_vector - sdsl - tlx) + libsais + pasta_bit_vector + sdsl + tlx) ################################################################################ diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 8f0779a..ea7be6a 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -125,39 +125,14 @@ class BlockTreeFP2 : public BlockTree { // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { - std::cout << "level " << level << ": " << std::endl; - auto begin = std::chrono::high_resolution_clock ::now(); LevelData ¤t = levels.back(); - /* - const bool last_block_padded = - block_starts.back() + block_size == text_len; - */ - scan_block_pairs(text, current, is_padded); - const auto scan_block_pair_time = - std::chrono::duration_cast( - std::chrono::high_resolution_clock ::now() - begin) - .count(); - begin = std::chrono::high_resolution_clock ::now(); scan_blocks(text, current, is_padded); - const auto scan_block_time = - std::chrono::duration_cast( - std::chrono::high_resolution_clock ::now() - begin) - .count(); - begin = std::chrono::high_resolution_clock ::now(); // Generate the next level (if we're not at the last level) if (level < static_cast(tree_height) - 1) { levels.push_back(generate_next_level(text, current)); } - const auto generate_time = - std::chrono::duration_cast( - std::chrono::high_resolution_clock ::now() - begin) - .count(); - - std::cout << "pair: " << scan_block_pair_time - << "ms, block: " << scan_block_time - << "ms, next: " << generate_time << "ms" << std::endl; } make_tree(text, levels, padding); diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 316b11e..9a18e91 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -50,7 +50,7 @@ template class MersenneRabinKarp { uint128_t fp = 0; uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { - fp = mersenneModulo(fp * sigma); + fp = fp * sigma; fp = mersenneModulo(fp + text_[i]); } for (uint64_t i = 0; i < length_ - 1; i++) { @@ -67,16 +67,11 @@ template class MersenneRabinKarp { init_ = index; max_sigma_ = 1; uint128_t fp = 0; - uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { - fp = mersenneModulo(fp * sigma_); + fp = fp * sigma_; fp = mersenneModulo(fp + text_[i]); } - for (uint64_t i = 0; i < length_ - 1; i++) { - sigma_c = mersenneModulo(sigma_c * sigma_); - } hash_ = (uint64_t)(fp); - max_sigma_ = (uint64_t)(sigma_c); }; inline uint128_t mersenneModulo(uint128_t k) { @@ -97,13 +92,9 @@ template class MersenneRabinKarp { uint128_t fp = hash_; T out_char = text_[init_]; T in_char = text_[init_ + length_]; - uint128_t out_char_influence = out_char * max_sigma_; - out_char_influence = mersenneModulo(out_char_influence); - if (out_char_influence < hash_) { - fp -= out_char_influence; - } else { - fp = prime_ - (out_char_influence - fp); - } + const uint128_t out_char_influence = mersenneModulo(out_char * max_sigma_); + // Conditionally add the prime, of the out_char_influence is too large + fp += prime_ * (out_char_influence > hash_) - out_char_influence; fp *= sigma_; fp += in_char; fp = mersenneModulo(fp); diff --git a/tests/block_tree/block_tree_fp_test.cpp b/tests/block_tree/block_tree_fp_test.cpp index a703ab4..116e366 100644 --- a/tests/block_tree/block_tree_fp_test.cpp +++ b/tests/block_tree/block_tree_fp_test.cpp @@ -25,16 +25,14 @@ #include #include -#include class BlockTreeFPTest : public ::testing::Test { protected: - std::vector text; - pasta::BlockTreeFP* bt; - + pasta::BlockTreeFP *bt; + void SetUp() override { std::random_device rd; @@ -46,15 +44,12 @@ class BlockTreeFPTest : public ::testing::Test { for (size_t i = 0; i < text.size(); ++i) { text[i] = dist(gen); } - + bt = pasta::make_block_tree_fp(text, 2, 1); bt->add_rank_support(); } - void TearDown() override { - delete bt; - } - + void TearDown() override { delete bt; } }; TEST_F(BlockTreeFPTest, access) { @@ -81,9 +76,9 @@ TEST_F(BlockTreeFPTest, select) { } } -int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } /******************************************************************************/ From 86bc25ee1453c66728b0c697533d10c77462c58e Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 15 Aug 2023 18:59:20 +0200 Subject: [PATCH 04/92] add tentative pruning to new sequential impl --- CMakeLists.txt | 12 +- CMakePresets.json | 12 +- examples/build_bt.cpp | 96 ++++ .../block_tree/construction/block_tree_fp.hpp | 2 +- .../construction/block_tree_fp2_seq.hpp | 523 +++++++++++------- .../pasta/block_tree/utils/MersenneHash.hpp | 2 +- .../block_tree/utils/MersenneRabinKarp.hpp | 7 +- tests/CMakeLists.txt | 7 +- 8 files changed, 450 insertions(+), 211 deletions(-) create mode 100644 examples/build_bt.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 4849f70..b07d8d9 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -46,14 +46,20 @@ if (PASTA_BLOCK_TREE_BUILD_TESTS) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) enable_testing() - add_subdirectory(tests) include(GoogleTest) + add_subdirectory(tests) endif () if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) - add_executable(example + add_executable(block_tree_construction examples/block_tree_construction.cpp) - target_link_libraries(example + target_link_libraries(block_tree_construction pasta_block_tree) + + add_executable(build_bt + examples/build_bt.cpp) + target_link_libraries(build_bt + pasta_block_tree) + endif () set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE) diff --git a/CMakePresets.json b/CMakePresets.json index fc5dcda..d43766d 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -55,6 +55,7 @@ "binaryDir": "${sourceDir}/build_multi", "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_TESTS": "ON", + "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", @@ -90,5 +91,14 @@ "configurePreset": "ninja-multi", "configuration": "Debug" } - ] + ], + "testPresets": [ + { + "name": "default", + "configurePreset": "ninja-multi", + "output": {"outputOnFailure": true}, + "execution": {"noTestsAction": "error", "stopOnFailure": true}, + "configuration": "RelWithDebInfo" + } + ] } diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp new file mode 100644 index 0000000..05001f5 --- /dev/null +++ b/examples/build_bt.cpp @@ -0,0 +1,96 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#include +#include +#include +#include +#include + +// #include +#include + +int main(int argc, char **argv) { + + if (argc < 2) { + std::cerr << "Please input file" << std::endl; + exit(1); + } + + if (!std::filesystem::exists(argv[1])) { + std::cerr << "File " << argv[1] << " does not exist" << std::endl; + exit(1); + } + + if (argc < 3) { + std::cerr << "Please input tree arity (tau)" << std::endl; + exit(1); + } + + size_t arity = atoi(argv[2]); + + if (argc < 4) { + std::cerr << "Please input root arity (s)" << std::endl; + exit(1); + } + + size_t root_arity = atoi(argv[3]); + + if (argc < 5) { + std::cerr << "Please input max leaf length" << std::endl; + exit(1); + } + + size_t leaf_length = atoi(argv[4]); + + std::stringstream ss; + ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" + << leaf_length << "_new.bt"; + std::string out_path = ss.str(); + + std::cout << "building block tree with parameters:" + << "\narity: " << arity << "\nroot arity: " << root_arity + << "\nmax leaf length: " << leaf_length << "\nsaving to " + << out_path << std::endl; + + std::string input; + std::ifstream t(argv[1]); + std::stringstream buffer; + buffer << t.rdbuf(); + input = buffer.str(); + + std::vector text(input.begin(), input.end()); + + auto bt = std::make_unique>( + text, arity, root_arity, leaf_length); + // auto bt = + // pasta::make_block_tree_fp(text, arity, leaf_length); + + std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb" + << std::endl; + + // std::ofstream ot(out_path); + // bt->serialize(ot); + // ot.close(); + + return 0; +} + +/******************************************************************************/ diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 3d841b9..7560ac4 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -603,7 +603,7 @@ class BlockTreeFP : public BlockTree { auto &lvl_pass1 = *bv_marked[i]; // Number of non-pruned blocks so far size_type c = 0; - // + // size_type c_u = 0; for (uint64_t j = 0; j < lvl_pass1.size(); j++) { blocks_skipped[j] = j - c; diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index ea7be6a..6edcbbc 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -20,11 +20,13 @@ #pragma once +#include +#include + #include "pasta/bit_vector/bit_vector.hpp" #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include #include #include @@ -32,13 +34,13 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { -template +template class BlockTreeFP2 : public BlockTree { constexpr static size_type NO_EARLIER_OCC = -1; constexpr static size_type PRUNED = -2; - static constexpr size_t SIGMA = 256; + static constexpr size_type SIGMA = 256; static constexpr uint128_t K_PRIME = 2305843009213693951ULL; using BitVector = pasta::BitVector; @@ -49,10 +51,10 @@ class BlockTreeFP2 : public BlockTree { typename hash_type = std::hash> using HashMap = std::unordered_map; - // using RabinKarp = MersenneRabinKarp; - using RabinKarp = MersenneRabinKarp; - // using RabinKarpHash = MersenneHash; - using RabinKarpHash = MersenneHash; + using RabinKarp = MersenneRabinKarp; + // using RabinKarp = MersenneRabinKarp; + using RabinKarpHash = MersenneHash; + // using RabinKarpHash = MersenneHash; template using RabinKarpMap = HashMap; @@ -72,14 +74,14 @@ class BlockTreeFP2 : public BlockTree { /// Block start indices std::unique_ptr> block_starts; /// The block size on this level - size_t block_size; + size_type block_size; /// The index of the current level. First level is 0, second level is 1 etc. - size_t level_index; + size_type level_index; /// The number of blocks on the current level - size_t num_blocks; + size_type num_blocks; - inline LevelData(size_t level_index_, size_t block_size_, - size_t num_blocks_) + inline LevelData(size_type level_index_, size_type block_size_, + size_type num_blocks_) : is_internal(nullptr), is_internal_rank(nullptr), pointers(new std::vector()), offsets(new std::vector()), @@ -89,7 +91,7 @@ class BlockTreeFP2 : public BlockTree { /// @brief Checks whether a block is adjacent in the text /// to its successor on this level - [[nodiscard]] inline bool is_adjacent(size_t i) const { + [[nodiscard]] inline bool is_adjacent(size_type i) const { assert(i < num_blocks - 1); return (*block_starts)[i] + static_cast(block_size) == (*block_starts)[i + 1]; @@ -97,8 +99,7 @@ class BlockTreeFP2 : public BlockTree { }; void construct(const std::vector &text) { - - const size_t text_len = text.size(); + const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size int64_t padding; @@ -117,7 +118,7 @@ class BlockTreeFP2 : public BlockTree { levels.emplace_back(0, top_block_size, text_len / top_block_size); LevelData &top_level = levels.back(); top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); - for (uint64_t i = 0; i < text_len; i += top_level.block_size) { + for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); } top_level.block_size = top_block_size; @@ -131,17 +132,21 @@ class BlockTreeFP2 : public BlockTree { // Generate the next level (if we're not at the last level) if (level < static_cast(tree_height) - 1) { - levels.push_back(generate_next_level(text, current)); + levels.push_back(std::move(generate_next_level(text, current))); } } + prune(levels); make_tree(text, levels, padding); } /// @brief Returns the ceiling of x / y for x > 0; /// /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline size_t ceil_div(size_t x, size_t y) { return 1 + ((x - 1) / y); } + + inline size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } /// Scan through the blocks pairwise in order to identify which blocks should /// be replaced with back blocks. @@ -150,8 +155,9 @@ class BlockTreeFP2 : public BlockTree { /// @param level The data for the current level. /// /// @return The block start indices for the next level of the tree - void scan_block_pairs(const std::vector &text, LevelData &level, - const bool is_padded) { + __attribute__((noinline)) void + scan_block_pairs(const std::vector &text, LevelData &level, + const bool is_padded) { const size_t block_size = level.block_size; const size_t num_blocks = level.num_blocks; const size_t pair_size = 2 * block_size; @@ -164,7 +170,7 @@ class BlockTreeFP2 : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - RabinKarpMap> map(num_blocks - 1); + RabinKarpMap> map(num_blocks - 1); // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. @@ -191,7 +197,7 @@ class BlockTreeFP2 : public BlockTree { // previous occurrences. RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { - if (!level.is_adjacent(i)) { + if (!level.is_adjacent(i) | !level.is_adjacent(i + 1)) { continue; } scan_windows_in_block_pair(rk, map, markings, block_size); @@ -209,54 +215,13 @@ class BlockTreeFP2 : public BlockTree { level.is_internal_rank = std::make_unique(is_internal); } - /// @brief Generate the block size, number of block and block start indices - /// for the next level. - /// - /// This depends on the current level's block size, number of blocks and - /// is_internal bit vector. - /// - /// @param text The input text. - /// @param level The level data of the previous level. - /// @return The level data of the next level. - [[nodiscard]] LevelData generate_next_level(const std::vector &text, - const LevelData &level) const { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const auto &is_internal = *level.is_internal; - const size_t next_block_size = block_size / this->tau_; - - std::vector new_block_starts; - new_block_starts.reserve(num_blocks * this->tau_); - for (size_t i = 0; i < num_blocks; ++i) { - if (!is_internal[i]) { - continue; - } - - // We generate up to tau new blocks for each internal block, - // excluding blocks that start past the end of the text - const auto parent_block_start = (*level.block_starts)[i]; - for (size_t j = 0, current_block_start = parent_block_start; - j < static_cast(this->tau_) && - current_block_start < text.size(); - ++j, current_block_start += next_block_size) { - new_block_starts.push_back(current_block_start); - } - } - - LevelData next_level(level.level_index + 1, next_block_size, - new_block_starts.size()); - next_level.block_starts = - std::make_unique>(std::move(new_block_starts)); - return next_level; - } - /// @brief Scan through the windows starting in a block and mark them /// accordingly if they represent the earliest occurrence of some block hash. /// /// The supplied `RabinKarp` hasher must be at the start of the block. /// @param rk A Rabin-Karp hasher whose state is at the start of the block. /// @param map The map containing the hashes of block pairs mapped to their - /// index. + /// index. /// @param markings A vector storing the marks on a block. Marks are 2-bit /// integers. /// If the MSB is set, that means that the content of the block and its @@ -264,9 +229,9 @@ class BlockTreeFP2 : public BlockTree { /// the content of the block and its predecessor has an earlier occurrence. /// @param block_size The size of blocks on the current level. static inline void scan_windows_in_block_pair( - RabinKarp &rk, RabinKarpMap> &map, + RabinKarp &rk, RabinKarpMap> &map, sdsl::int_vector<2> &markings, const size_t block_size) { - for (size_t offset = 0; offset < block_size; ++offset) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { RabinKarpHash current_hash = rk.current_hash(); // Find the hash of the current window among the hashed block pairs. auto found_hash_ptr = map.find(current_hash); @@ -287,73 +252,59 @@ class BlockTreeFP2 : public BlockTree { markings[block_index + 1] = markings[block_index + 1] | 0b01; } map.erase(found_hash_ptr); - rk.next(); } } - /// @return The block start indices for the *next* level. - auto scan_blocks(const std::vector &s, LevelData &level_data, - const bool is_padded) { + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s + /// @param level_data + /// @param is_padded + /// @return + __attribute__((noinline)) auto scan_blocks(const std::vector &s, + LevelData &level_data, + const bool is_padded) { const size_t block_size = level_data.block_size; const size_t num_blocks = level_data.num_blocks; const std::vector &block_starts = *level_data.block_starts; - const BitVector &is_internal = *level_data.is_internal; - level_data.pointers = std::make_unique>(num_blocks, NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } // A map with hashed slices as keys, which map to a vector of links, // describing a link between a (potential) back block to their source block. // In addition to the vector, there is a boolean which denotes whether a // hash has already been processed - RabinKarpMap>> links(num_blocks - 1); + RabinKarpMap> links(num_blocks); for (size_t i = 0; i < num_blocks - is_padded; ++i) { const RabinKarpHash hash = RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) .current_hash(); - auto entry = links.insert({hash, {false, {}}}); - auto &[found_hash, pair] = *entry.first; - auto &[was_handled, vec] = pair; - vec.emplace_back(i); + links[hash].push_back(i); } // Hash every window and find the first occurrences for every block. - RabinKarp rk(s, SIGMA, 0, block_size, K_PRIME); + RabinKarp rk(s, SIGMA, block_starts[0], block_size, K_PRIME); for (size_t current_block_index = 0; - current_block_index < num_blocks - is_padded; ++current_block_index) { - // We can skip this loop iteration if the current block is a back block - // Nothing is ever going to point to this anyway. - if (!is_internal[current_block_index]) { - continue; - } + current_block_index < num_blocks - is_padded - 1; + ++current_block_index) { if (static_cast(rk.init_) != block_starts[current_block_index]) { rk.restart(block_starts[current_block_index]); } - // This is true iff there exists a next block and it is not adjacent - const bool next_block_not_adjacent = - current_block_index < num_blocks - 1 && - !level_data.is_adjacent(current_block_index); - // If the next block is not adjacent, we just want to hash exactly this - // block. If it either is adjacent or we are at the end of the string, we - // take care not to hash windows that start beyond the end of the string - const size_t num_hashes = - next_block_not_adjacent - ? 1 - : block_size - saturating_sub(block_starts[current_block_index] + - block_size, - s.size()); - - scan_windows_in_block(rk, links, level_data, current_block_index, - num_hashes); - ++current_block_index; + scan_windows_in_block(rk, links, level_data, current_block_index); } } - /// \brief Scans through block-sized windows starting inside one block and /// tries to find earlier occurrences of blocks. Non-internal blocks will /// have their respective m_source_blocks and m_offsets entries populated. @@ -364,40 +315,82 @@ class BlockTreeFP2 : public BlockTree { /// Rabin-Karp hasher is situated in only with respect to *internal blocks* /// on the current level, disregarding back blocks. /// \param num_hashes The number of times the Rabin-Karp hasher should hash. - static void scan_windows_in_block( - RabinKarp &rk, - RabinKarpMap>> &links, - LevelData &level_data, const size_t current_block_index, - const size_t num_hashes) { - for (size_t offset = 0; offset < num_hashes; ++offset) { + __attribute__((noinline)) static void scan_windows_in_block( + RabinKarp &rk, RabinKarpMap> &links, + LevelData &level_data, const size_t current_block_index) { + const BitVector &is_internal = *level_data.is_internal; + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { const RabinKarpHash current_hash = rk.current_hash(); // Find all blocks in the multimap that match our hash auto found = links.find(current_hash); - // Only handle this hash if it does not exist and we have not handled it - // already - auto &[hash_is_handled, found_blocks] = found->second; - if (found == links.end() || hash_is_handled) { + if (found == links.end()) { continue; } + const auto &[block_hash, found_blocks] = *found; const size_t num_found_blocks = found_blocks.size(); - for (size_t i = 1; i < num_found_blocks; ++i) { - int block_index = found_blocks[i]; - // link.source_block_index = current_block_index; - // link.offset = offset; - // There is only space for non-internal blocks in these vectors - // if (!is_internal[link.block_index]) { - // Get the index of the back block only considering back blocks - // const size_t back_block_index = - // is_internal_rank.rank0(link.block_index); + // In this case, we are hashing an actual block right now (not just an + // arbitrary window). As a result, the first block in the vector is the + // block we are currently hashing in + const size_t skip_first = + block_hash.start_ == current_hash.start_ ? 1 : 0; + for (size_t i = skip_first; i < num_found_blocks; ++i) { + const size_type block_index = found_blocks[i]; (*level_data.pointers)[block_index] = current_block_index; (*level_data.offsets)[block_index] = offset; - //} + // We increment the counter for the block that is being pointed to + // if the current block is actually a back block + // If the offset is greater than 0, + // then it also overlaps into the next block + const size_t is_internal_block = is_internal[block_index] == 0 ? 1 : 0; + (*level_data.counters)[current_block_index] += 1; + (*level_data.counters)[current_block_index + 1] += + is_internal_block & (offset > 0); + } + links.erase(found); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector &text, + const LevelData &level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto &is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); } - // We handled this hash, so we mark it as such - hash_is_handled = true; - // links.erase(found); - rk.next(); } + + LevelData next_level(level.level_index + 1, next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; } /// @@ -406,14 +399,25 @@ class BlockTreeFP2 : public BlockTree { /// @param[in] levels A vector containing data for each level, with the first /// entry corresponding to the topmost level. /// - void make_tree(std::vector text, std::vector &levels, - int64_t padding) { + void make_tree(const std::vector &text, + std::vector &levels, int64_t padding) { const bool is_padded = padding > 0; - // TODO Maybe only create this if there is a back block + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_type block = 0; block < levels[level].num_blocks; block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + // Create first level - { - LevelData &top_level = levels.front(); + bool found_back_block = + levels[0].is_internal->size() == new_num_internal[0] || + !this->CUT_FIRST_LEVELS; + LevelData &top_level = levels.front(); + if (found_back_block) { const size_t n = top_level.num_blocks; const size_t num_internal = top_level.is_internal_rank->rank1(n); auto pointers = new sdsl::int_vector<>(n - num_internal, 0); @@ -435,80 +439,29 @@ class BlockTreeFP2 : public BlockTree { this->block_tree_pointers_.push_back(pointers); this->block_tree_offsets_.push_back(offsets); this->block_size_lvl_.push_back(top_level.block_size); - - top_level.pointers.reset(); - top_level.offsets.reset(); - top_level.counters.reset(); } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + // Add level data to the tree for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData &previous_level = levels[level_index - 1]; - size_type previous_level_num_blocks = previous_level.num_blocks; LevelData &level = levels[level_index]; - size_type new_size = - (previous_level.is_internal_rank->rank1(previous_level_num_blocks) - - is_padded) * - this->tau_; - previous_level.is_internal.reset(); - previous_level.is_internal_rank.reset(); - // Determine the number of children the last block generated - if (is_padded) { - const size_type last_block_parent_start = - previous_level.block_starts->back(); - const size_type block_size = level.block_size; - for (uint64_t i = 0; i < static_cast(this->tau_); i++) { - // Count the number of blocks that still start before the text end - new_size += last_block_parent_start + i * block_size < text.size(); - } - } - previous_level.block_starts.reset(); - const size_t n = level.num_blocks; - // TODO This will break once pruning is implemented. - // Instead of using rank, the number of ones after pruning should be - // saved in a vec for example - const size_t num_internal = level.is_internal_rank->rank1(n); - auto is_internal = new BitVector(level.num_blocks); - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_non_pruned = 0; - size_t num_back_blocks = 0; - - // We will reuse the allocated memory of the pointers vector to store - // the number of pruned blocks before the block. - // The invariant is that all values up to i are overwritten while all - // values starting after i will still be valid pointers - std::vector &num_blocks_skipped = *level.pointers; - for (size_t i = 0; i < level.num_blocks; i++) { - const size_type ptr = (*level.pointers)[i]; - num_blocks_skipped[i] = i - num_non_pruned; - - // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { - continue; - } - - // Add it to the is_internal bit vector - const bool block_is_internal = (*level.is_internal)[i]; - (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; - - if (block_is_internal) { - continue; - } - // If it is a back block, add its pointer and offset - const size_t offset = (*level.offsets)[i]; + LevelData &previous_level = levels[level_index - 1]; - (*pointers)[num_back_blocks] = ptr - num_blocks_skipped[ptr]; - (*offsets)[num_back_blocks] = offset; - num_back_blocks++; + found_back_block |= levels[level_index].is_internal->size() == + new_num_internal[level_index]; + if (!found_back_block) { + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.is_internal.reset(); + previous_level.is_internal_rank.reset(); + continue; } - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(is_internal); - this->block_tree_types_rs_.push_back(new Rank(*is_internal)); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(level.block_size); + + make_tree_level(levels, new_num_internal, level_index, is_padded, + text.size()); // We don't need these anymore level.pointers.reset(); @@ -542,6 +495,176 @@ class BlockTreeFP2 : public BlockTree { this->compress_leaves(); } + /// @brief Generates a level and adds the relevant data to the block tree. + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector &levels, + std::vector &new_num_internal, + const size_t level_index, const bool is_padded, + const size_t text_len) { + LevelData &previous_level = levels[level_index - 1]; + LevelData &level = levels[level_index]; + + const size_type previous_level_num_blocks = previous_level.num_blocks; + size_type new_size = + (previous_level.is_internal_rank->rank1(previous_level_num_blocks) - + is_padded) * + this->tau_; + previous_level.is_internal.reset(); + previous_level.is_internal_rank.reset(); + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + for (uint64_t i = 0; i < static_cast(this->tau_); i++) { + // Count the number of blocks that still start before the text end + new_size += last_block_parent_start + i * block_size < text_len; + } + } + previous_level.block_starts.reset(); + const size_t n = level.num_blocks; + const size_t num_internal = new_num_internal[level_index]; + auto is_internal = new BitVector(level.num_blocks); + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_non_pruned = 0; + size_t num_back_blocks = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + std::vector &num_blocks_skipped = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + num_blocks_skipped[i] = i - num_non_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - num_blocks_skipped[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector &levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief + /// @param levels + /// @param level_index + /// @param block_index + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector &levels, const size_t level_index, + const size_t block_index) { + + LevelData &level = levels[level_index]; + BitVector &is_internal = *level.is_internal; + + // If we are at the leaf level, we can't prune anything + if (level_index == levels.size() - 1) { + return false; + } + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child_index = block_index * this->tau_; + + // the number of children before this block which are skipped on the next + // level. Since the children of back blocks are not included on the next + // level, we need to subtract the skipped blocks + const size_t missing_children = + block_index == 0 + ? 0 + : level.is_internal_rank->rank0(block_index) * this->tau_; + + bool has_internal_children = false; + + if (level_index < levels.size() - 1) { + const size_type last_child_index = + std::min(first_child_index + this->tau_ - 1, + levels[level_index + 1].num_blocks); + // Iterate through children in reverse + for (size_type child_index = last_child_index; + child_index >= first_child_index; --child_index) { + has_internal_children |= prune_block(levels, level_index + 1, + child_index - missing_children); + } + } + + if (has_internal_children) { + // If any of the children is internal, this block stays internal as well + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. We will make this block here into a back + // block and mark the children as pruned + LevelData &child_level = levels[level_index + 1]; + for (size_type child_index = first_child_index; + child_index < first_child_index + this->tau_; ++child_index) { + const size_type child_pointer = + (*child_level.pointers)[child_index - missing_children]; + const size_type child_offset = + (*child_level.offsets)[child_index - missing_children]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child_index - missing_children] = PRUNED; + } + + // This is now a back block + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + return false; + } + /// @brief Branch-less saturating subtraction. /// /// Source: http://locklessinc.com/articles/sat_arithmetic/ @@ -555,8 +678,6 @@ class BlockTreeFP2 : public BlockTree { return res; } - // TODO prune - public: BlockTreeFP2(const std::vector &text, const size_t arity, const size_t root_arity, const size_t max_leaf_length) { diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index f8fbb14..51a9460 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -45,7 +45,7 @@ template class MersenneHash { return false; } } - return true; + return hash_ == other.hash_; } }; diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 9a18e91..c7d5478 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -42,6 +42,12 @@ template class MersenneRabinKarp { uint64_t hash_; uint128_t max_sigma_; + /// @brief Construct a new Rabin Karp hasher. + /// @param text The text to hash. + /// @param sigma The alphabet size. + /// @param init The start index of the first hashed window in the text. + /// @param length The window size. + /// @param prime A large prime used for modulus operations. MersenneRabinKarp(std::vector const &text, uint64_t sigma, uint64_t init, uint64_t length, uint128_t prime) : text_(text), sigma_(sigma), init_(init), length_(length), @@ -65,7 +71,6 @@ template class MersenneRabinKarp { return; } init_ = index; - max_sigma_ = 1; uint128_t fp = 0; for (uint64_t i = init_; i < init_ + length_; i++) { fp = fp * sigma_; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index fcf2c25..77dba78 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,9 +26,10 @@ macro(pasta_block_tree_build_test TESTNAME) pasta_block_tree GTest::Main) include_directories(${TESTNAME_REPLACED} PRIVATE ${gtest_SOURCE_DIR}/include) - add_test( - NAME ${TESTNAME_REPLACED} - COMMAND ${TESTNAME_REPLACED} ${ARGN}) + #add_test( + # NAME ${TESTNAME_REPLACED} + # COMMAND ${TESTNAME_REPLACED} ${ARGN}) + gtest_add_tests(TARGET ${TESTNAME_REPLACED}) endmacro(pasta_block_tree_build_test) include(CTest) From 8179cc4105eb42643d9d4aa335dd18239f9c8b2f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 15 Aug 2023 20:20:57 +0200 Subject: [PATCH 05/92] improve pruning --- CMakePresets.json | 13 ++-- .../construction/block_tree_fp2_seq.hpp | 64 ++++++++----------- 2 files changed, 35 insertions(+), 42 deletions(-) diff --git a/CMakePresets.json b/CMakePresets.json index d43766d..edfaf46 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -58,7 +58,7 @@ "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" } } @@ -96,9 +96,14 @@ { "name": "default", "configurePreset": "ninja-multi", - "output": {"outputOnFailure": true}, - "execution": {"noTestsAction": "error", "stopOnFailure": true}, + "output": { + "outputOnFailure": true + }, + "execution": { + "noTestsAction": "error", + "stopOnFailure": true + }, "configuration": "RelWithDebInfo" } - ] + ] } diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 6edcbbc..95a6f57 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -405,7 +405,8 @@ class BlockTreeFP2 : public BlockTree { std::vector new_num_internal(levels.size(), 0); for (size_t level = 0; level < levels.size(); level++) { - for (size_type block = 0; block < levels[level].num_blocks; block++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { if ((*levels[level].is_internal)[block]) { new_num_internal[level]++; } @@ -414,12 +415,12 @@ class BlockTreeFP2 : public BlockTree { // Create first level bool found_back_block = - levels[0].is_internal->size() == new_num_internal[0] || + levels[0].is_internal->size() > new_num_internal[0] || !this->CUT_FIRST_LEVELS; LevelData &top_level = levels.front(); if (found_back_block) { const size_t n = top_level.num_blocks; - const size_t num_internal = top_level.is_internal_rank->rank1(n); + const size_t num_internal = new_num_internal[0]; auto pointers = new sdsl::int_vector<>(n - num_internal, 0); auto offsets = new sdsl::int_vector<>(n - num_internal, 0); size_t num_back_blocks = 0; @@ -448,15 +449,15 @@ class BlockTreeFP2 : public BlockTree { for (size_t level_index = 1; level_index < levels.size(); level_index++) { LevelData &level = levels[level_index]; LevelData &previous_level = levels[level_index - 1]; - found_back_block |= levels[level_index].is_internal->size() == new_num_internal[level_index]; if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); level.counters.reset(); - previous_level.is_internal.reset(); - previous_level.is_internal_rank.reset(); + previous_level.block_starts.reset(); continue; } @@ -464,9 +465,12 @@ class BlockTreeFP2 : public BlockTree { text.size()); // We don't need these anymore + level.is_internal.reset(); + level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); level.counters.reset(); + previous_level.block_starts.reset(); } this->leaf_size = levels.back().block_size / this->tau_; @@ -500,42 +504,34 @@ class BlockTreeFP2 : public BlockTree { /// @param level_index The index of the level to generate. /// @param is_padded Whether there is padding in the last block of the tree void make_tree_level(std::vector &levels, - std::vector &new_num_internal, + const std::vector &new_num_internal, const size_t level_index, const bool is_padded, const size_t text_len) { LevelData &previous_level = levels[level_index - 1]; LevelData &level = levels[level_index]; - const size_type previous_level_num_blocks = previous_level.num_blocks; size_type new_size = - (previous_level.is_internal_rank->rank1(previous_level_num_blocks) - - is_padded) * - this->tau_; - previous_level.is_internal.reset(); - previous_level.is_internal_rank.reset(); + (new_num_internal[level_index - 1] - is_padded) * this->tau_; // Determine the number of children the last block generated if (is_padded) { const size_type last_block_parent_start = previous_level.block_starts->back(); const size_type block_size = level.block_size; - for (uint64_t i = 0; i < static_cast(this->tau_); i++) { - // Count the number of blocks that still start before the text end - new_size += last_block_parent_start + i * block_size < text_len; - } + new_size += ceil_div(text_len - last_block_parent_start, block_size); } previous_level.block_starts.reset(); - const size_t n = level.num_blocks; - const size_t num_internal = new_num_internal[level_index]; - auto is_internal = new BitVector(level.num_blocks); - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_non_pruned = 0; - size_t num_back_blocks = 0; + const size_type num_internal = new_num_internal[level_index]; + auto is_internal = new BitVector(new_size); + auto pointers = new sdsl::int_vector<>(new_size - num_internal + 1, 0); + auto offsets = new sdsl::int_vector<>(new_size - num_internal + 1, 0); + size_type num_non_pruned = 0; + size_type num_back_blocks = 0; // We will reuse the allocated memory of the pointers vector to store // the number of pruned blocks before the block. // The invariant is that all values up to i are overwritten while all // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i std::vector &num_blocks_skipped = *level.pointers; for (size_type i = 0; i < level.num_blocks; i++) { const size_type ptr = (*level.pointers)[i]; @@ -554,6 +550,7 @@ class BlockTreeFP2 : public BlockTree { if (block_is_internal) { continue; } + // If it is a back block, add its pointer and offset const size_type offset = (*level.offsets)[i]; @@ -604,7 +601,7 @@ class BlockTreeFP2 : public BlockTree { const size_type first_child_index = block_index * this->tau_; - // the number of children before this block which are skipped on the next + // The number of children before this block which are skipped on the next // level. Since the children of back blocks are not included on the next // level, we need to subtract the skipped blocks const size_t missing_children = @@ -614,6 +611,9 @@ class BlockTreeFP2 : public BlockTree { bool has_internal_children = false; + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. if (level_index < levels.size() - 1) { const size_type last_child_index = std::min(first_child_index + this->tau_ - 1, @@ -655,6 +655,7 @@ class BlockTreeFP2 : public BlockTree { (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned (*child_level.pointers)[child_index - missing_children] = PRUNED; + (*child_level.is_internal)[child_index - missing_children] = false; } // This is now a back block @@ -665,19 +666,6 @@ class BlockTreeFP2 : public BlockTree { return false; } - /// @brief Branch-less saturating subtraction. - /// - /// Source: http://locklessinc.com/articles/sat_arithmetic/ - /// - /// @param x The minuend. - /// @param y The subtrahend. - /// @return x - y. However, if the result would underflow, returns zero. - static inline constexpr auto saturating_sub(size_t x, size_t y) -> size_t { - size_t res = x - y; - res &= -(res <= x); - return res; - } - public: BlockTreeFP2(const std::vector &text, const size_t arity, const size_t root_arity, const size_t max_leaf_length) { From f5bfbe3891f8450593109eb6d0ff77f949ede47d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 17 Aug 2023 14:27:29 +0200 Subject: [PATCH 06/92] fix pruning step --- .../construction/block_tree_fp2_seq.hpp | 76 ++++++++++++------- 1 file changed, 49 insertions(+), 27 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 95a6f57..86bb791 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -403,7 +403,8 @@ class BlockTreeFP2 : public BlockTree { std::vector &levels, int64_t padding) { const bool is_padded = padding > 0; - std::vector new_num_internal(levels.size(), 0); + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); for (size_t level = 0; level < levels.size(); level++) { for (size_t block = 0; block < levels[level].is_internal->size(); block++) { @@ -414,9 +415,9 @@ class BlockTreeFP2 : public BlockTree { } // Create first level - bool found_back_block = - levels[0].is_internal->size() > new_num_internal[0] || - !this->CUT_FIRST_LEVELS; + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; LevelData &top_level = levels.front(); if (found_back_block) { const size_t n = top_level.num_blocks; @@ -450,7 +451,7 @@ class BlockTreeFP2 : public BlockTree { LevelData &level = levels[level_index]; LevelData &previous_level = levels[level_index - 1]; found_back_block |= levels[level_index].is_internal->size() == - new_num_internal[level_index]; + static_cast(new_num_internal[level_index]); if (!found_back_block) { level.is_internal.reset(); level.is_internal_rank.reset(); @@ -500,11 +501,13 @@ class BlockTreeFP2 : public BlockTree { } /// @brief Generates a level and adds the relevant data to the block tree. + /// /// @param levels The vector of levels of the tree. - /// @param level_index The index of the level to generate. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. /// @param is_padded Whether there is padding in the last block of the tree void make_tree_level(std::vector &levels, - const std::vector &new_num_internal, + const std::vector &new_num_internal, const size_t level_index, const bool is_padded, const size_t text_len) { LevelData &previous_level = levels[level_index - 1]; @@ -521,11 +524,28 @@ class BlockTreeFP2 : public BlockTree { } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; - auto is_internal = new BitVector(new_size); - auto pointers = new sdsl::int_vector<>(new_size - num_internal + 1, 0); - auto offsets = new sdsl::int_vector<>(new_size - num_internal + 1, 0); + // 1 neue size hängt von num_internal auf prev_level ab + // 2 die tatsächlichen nodes hängen von num_internal auf level ab + + // Möglichkeiten: + // 1: Zu wenige interne Knoten auf prev_level + // idk wie das sein kann + // Dann wären ja zu viele Blöcke auf dem level davor + // 2: Zu viele interne Knoten auf last level + // Ma gucken + // + // num_internal zählt genau, wo ne 1 im bv steht + // da wurden die blöcke rausgekickt, die + // 1. Earlier Occ haben + // 2. Keine Pointer auf sich haben, + // 3. Keine Internal children haben. + // + auto *is_internal = new BitVector(new_size); + auto *pointers = new sdsl::int_vector<>(2 * new_size - num_internal, 0); + auto *offsets = new sdsl::int_vector<>(2 * new_size - num_internal, 0); size_type num_non_pruned = 0; size_type num_back_blocks = 0; + size_type num_pruned = 0; // We will reuse the allocated memory of the pointers vector to store // the number of pruned blocks before the block. @@ -539,6 +559,7 @@ class BlockTreeFP2 : public BlockTree { // If the current block is not pruned, add it to the new tree if (ptr == PRUNED) { + num_pruned++; continue; } @@ -558,6 +579,7 @@ class BlockTreeFP2 : public BlockTree { (*offsets)[num_back_blocks] = offset; num_back_blocks++; } + sdsl::util::bit_compress(*pointers); sdsl::util::bit_compress(*offsets); this->block_tree_types_.push_back(is_internal); @@ -585,14 +607,13 @@ class BlockTreeFP2 : public BlockTree { /// @return Whether this block is/stays internal after the pruning process bool prune_block(std::vector &levels, const size_t level_index, const size_t block_index) { - LevelData &level = levels[level_index]; BitVector &is_internal = *level.is_internal; // If we are at the leaf level, we can't prune anything - if (level_index == levels.size() - 1) { - return false; - } + // if (level_index == levels.size() - 1) { + // return false; + //} // If the current block is a back block already, there is nothing to prune if (!is_internal[block_index]) { @@ -643,19 +664,20 @@ class BlockTreeFP2 : public BlockTree { // Now we know that there is an earlier occurrence, // and nothing is pointing here. We will make this block here into a back // block and mark the children as pruned - LevelData &child_level = levels[level_index + 1]; - for (size_type child_index = first_child_index; - child_index < first_child_index + this->tau_; ++child_index) { - const size_type child_pointer = - (*child_level.pointers)[child_index - missing_children]; - const size_type child_offset = - (*child_level.offsets)[child_index - missing_children]; - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child_index - missing_children] = PRUNED; - (*child_level.is_internal)[child_index - missing_children] = false; + if (level_index < levels.size() - 1) { + LevelData &child_level = levels[level_index + 1]; + for (size_type child_index = first_child_index; + child_index < first_child_index + this->tau_; ++child_index) { + const size_type child_pointer = + (*child_level.pointers)[child_index - missing_children]; + const size_type child_offset = + (*child_level.offsets)[child_index - missing_children]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child_index - missing_children] = PRUNED; + } } // This is now a back block From aff155974b7d6ea747a7e70c1783598db60e06f4 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 21:15:51 +0200 Subject: [PATCH 07/92] fix bug in cosntruction of leaf string --- examples/build_bt.cpp | 28 +- .../construction/block_tree_fp2_seq.hpp | 381 ++++++++++-------- 2 files changed, 243 insertions(+), 166 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 05001f5..b8328f9 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -21,14 +21,16 @@ #include #include #include +#include +#include #include #include -// #include -#include - -int main(int argc, char **argv) { +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; +int main(int argc, char** argv) { if (argc < 2) { std::cerr << "Please input file" << std::endl; exit(1); @@ -78,14 +80,22 @@ int main(int argc, char **argv) { std::vector text(input.begin(), input.end()); - auto bt = std::make_unique>( - text, arity, root_arity, leaf_length); + TimePoint now = Clock::now(); + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length); + // auto bt = - // pasta::make_block_tree_fp(text, arity, leaf_length); + // pasta::make_block_tree_fp(text, arity, leaf_length); - std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb" - << std::endl; + auto elapsed = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" + << "Time: " << elapsed << "ms" << std::endl; // std::ofstream ot(out_path); // bt->serialize(ot); // ot.close(); diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 86bb791..4dfd37a 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -20,13 +20,13 @@ #pragma once -#include -#include - #include "pasta/bit_vector/bit_vector.hpp" #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include #include #include @@ -36,7 +36,6 @@ namespace pasta { template class BlockTreeFP2 : public BlockTree { - constexpr static size_type NO_EARLIER_OCC = -1; constexpr static size_type PRUNED = -2; @@ -47,7 +46,8 @@ class BlockTreeFP2 : public BlockTree { // using Rank = pasta::FlatRank; using Rank = pasta::RankSelect; - template > using HashMap = std::unordered_map; @@ -80,25 +80,35 @@ class BlockTreeFP2 : public BlockTree { /// The number of blocks on the current level size_type num_blocks; - inline LevelData(size_type level_index_, size_type block_size_, + inline LevelData(size_type level_index_, + size_type block_size_, size_type num_blocks_) - : is_internal(nullptr), is_internal_rank(nullptr), + : is_internal(nullptr), + is_internal_rank(nullptr), pointers(new std::vector()), offsets(new std::vector()), counters(new std::vector()), - block_starts(new std::vector()), block_size(block_size_), - level_index(level_index_), num_blocks(num_blocks_) {} + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} /// @brief Checks whether a block is adjacent in the text /// to its successor on this level - [[nodiscard]] inline bool is_adjacent(size_type i) const { - assert(i < num_blocks - 1); + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { return (*block_starts)[i] + static_cast(block_size) == (*block_starts)[i + 1]; } + + /// @brief Checks whether a block is adjacent in the text + /// to its predecessor on this level + [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { + return (*block_starts)[i - 1] + static_cast(block_size) == + (*block_starts)[i]; + } }; - void construct(const std::vector &text) { + void construct(const std::vector& text) { const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -116,7 +126,7 @@ class BlockTreeFP2 : public BlockTree { // Prepare the top level levels.emplace_back(0, top_block_size, text_len / top_block_size); - LevelData &top_level = levels.back(); + LevelData& top_level = levels.back(); top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); @@ -126,7 +136,7 @@ class BlockTreeFP2 : public BlockTree { // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { - LevelData ¤t = levels.back(); + LevelData& current = levels.back(); scan_block_pairs(text, current, is_padded); scan_blocks(text, current, is_padded); @@ -143,21 +153,20 @@ class BlockTreeFP2 : public BlockTree { /// @brief Returns the ceiling of x / y for x > 0; /// /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline size_t ceil_div(std::integral auto x, std::integral auto y) { return 1 + ((x - 1) / y); } - /// Scan through the blocks pairwise in order to identify which blocks should - /// be replaced with back blocks. + /// @briefScan through the blocks pairwise in order to identify which blocks should + /// be replaced with back blocks. /// /// @param text The input string. /// @param level The data for the current level. /// /// @return The block start indices for the next level of the tree - __attribute__((noinline)) void - scan_block_pairs(const std::vector &text, LevelData &level, - const bool is_padded) { + static void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { const size_t block_size = level.block_size; const size_t num_blocks = level.num_blocks; const size_t pair_size = 2 * block_size; @@ -183,7 +192,7 @@ class BlockTreeFP2 : public BlockTree { for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { // If the next block is not adjacent, we cannot hash the pair starting // at the current block - if (!level.is_adjacent(i)) { + if (!level.next_is_adjacent(i)) { continue; } // Move the hasher to the current block pair @@ -197,7 +206,7 @@ class BlockTreeFP2 : public BlockTree { // previous occurrences. RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { - if (!level.is_adjacent(i) | !level.is_adjacent(i + 1)) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; } scan_windows_in_block_pair(rk, map, markings, block_size); @@ -205,7 +214,7 @@ class BlockTreeFP2 : public BlockTree { // Generate the bit vector indicating which blocks are internal level.is_internal = std::make_unique(num_blocks); - auto &is_internal = *level.is_internal; + auto& is_internal = *level.is_internal; is_internal[0] = true; is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; for (size_t i = 0; i < num_blocks - 1; ++i) { @@ -216,7 +225,8 @@ class BlockTreeFP2 : public BlockTree { } /// @brief Scan through the windows starting in a block and mark them - /// accordingly if they represent the earliest occurrence of some block hash. + /// accordingly if they represent the earliest occurrence of some block + /// hash. /// /// The supplied `RabinKarp` hasher must be at the start of the block. /// @param rk A Rabin-Karp hasher whose state is at the start of the block. @@ -228,9 +238,11 @@ class BlockTreeFP2 : public BlockTree { /// successor has an earlier occurrence. If the LSB being set means that /// the content of the block and its predecessor has an earlier occurrence. /// @param block_size The size of blocks on the current level. - static inline void scan_windows_in_block_pair( - RabinKarp &rk, RabinKarpMap> &map, - sdsl::int_vector<2> &markings, const size_t block_size) { + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap>& map, + sdsl::int_vector<2>& markings, + const size_t block_size) { for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { RabinKarpHash current_hash = rk.current_hash(); // Find the hash of the current window among the hashed block pairs. @@ -238,7 +250,7 @@ class BlockTreeFP2 : public BlockTree { if (found_hash_ptr == map.end()) { continue; } - auto &[block_pair_hash, block_indices] = *found_hash_ptr; + auto& [block_pair_hash, block_indices] = *found_hash_ptr; // If the hash we found is just the first block pair in the hash map, // then the first block pair has no earlier occurrence. @@ -258,16 +270,16 @@ class BlockTreeFP2 : public BlockTree { /// @brief Determine the positions for each block's earliest occurrence if /// there is any. /// - /// @param s - /// @param level_data - /// @param is_padded - /// @return - __attribute__((noinline)) auto scan_blocks(const std::vector &s, - LevelData &level_data, - const bool is_padded) { + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + static void scan_blocks(const std::vector& s, + LevelData& level_data, + const bool is_padded) { const size_t block_size = level_data.block_size; const size_t num_blocks = level_data.num_blocks; - const std::vector &block_starts = *level_data.block_starts; + const std::vector& block_starts = *level_data.block_starts; level_data.pointers = std::make_unique>(num_blocks, NO_EARLIER_OCC); @@ -297,55 +309,60 @@ class BlockTreeFP2 : public BlockTree { for (size_t current_block_index = 0; current_block_index < num_blocks - is_padded - 1; ++current_block_index) { - if (static_cast(rk.init_) != block_starts[current_block_index]) { rk.restart(block_starts[current_block_index]); } - scan_windows_in_block(rk, links, level_data, current_block_index); + if (level_data.next_is_adjacent(current_block_index)) { + scan_windows_in_block(rk, links, level_data, current_block_index); + continue; + } } } - /// \brief Scans through block-sized windows starting inside one block and + /// @brief Scans through block-sized windows starting inside one block and /// tries to find earlier occurrences of blocks. Non-internal blocks will /// have their respective m_source_blocks and m_offsets entries populated. - /// \param rk A Rabin-Karp hasher whose current state is at a block start. - /// \param links A map whose keys are hashed blocks and the values + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values /// are all block indices of blocks matching the hash in ascending order. - /// \param current_block_internal_index The index of the block which the + /// @param current_block_internal_index The index of the block which the /// Rabin-Karp hasher is situated in only with respect to *internal blocks* /// on the current level, disregarding back blocks. - /// \param num_hashes The number of times the Rabin-Karp hasher should hash. - __attribute__((noinline)) static void scan_windows_in_block( - RabinKarp &rk, RabinKarpMap> &links, - LevelData &level_data, const size_t current_block_index) { - const BitVector &is_internal = *level_data.is_internal; + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap>& links, + LevelData& level_data, + const size_type current_block_index) { + const BitVector& is_internal = *level_data.is_internal; for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { - const RabinKarpHash current_hash = rk.current_hash(); + const RabinKarpHash hash = rk.current_hash(); // Find all blocks in the multimap that match our hash - auto found = links.find(current_hash); + auto found = links.find(hash); if (found == links.end()) { continue; } - const auto &[block_hash, found_blocks] = *found; + const auto& [block_hash, found_blocks] = *found; const size_t num_found_blocks = found_blocks.size(); // In this case, we are hashing an actual block right now (not just an // arbitrary window). As a result, the first block in the vector is the // block we are currently hashing in - const size_t skip_first = - block_hash.start_ == current_hash.start_ ? 1 : 0; - for (size_t i = skip_first; i < num_found_blocks; ++i) { + for (size_t i = 0; i < num_found_blocks; ++i) { const size_type block_index = found_blocks[i]; + if (block_index == current_block_index || + (offset > 0 && block_index == current_block_index + 1)) { + continue; + } (*level_data.pointers)[block_index] = current_block_index; (*level_data.offsets)[block_index] = offset; // We increment the counter for the block that is being pointed to // if the current block is actually a back block // If the offset is greater than 0, // then it also overlaps into the next block - const size_t is_internal_block = is_internal[block_index] == 0 ? 1 : 0; - (*level_data.counters)[current_block_index] += 1; + const bool is_back_block = !is_internal[block_index]; + (*level_data.counters)[current_block_index] += is_back_block; (*level_data.counters)[current_block_index + 1] += - is_internal_block & (offset > 0); + is_back_block && (offset > 0); } links.erase(found); } @@ -361,11 +378,11 @@ class BlockTreeFP2 : public BlockTree { /// @param level The level data of the previous level. /// @return The level data of the next level. [[nodiscard]] LevelData - generate_next_level(const std::vector &text, - const LevelData &level) const { + generate_next_level(const std::vector& text, + const LevelData& level) const { const size_t block_size = level.block_size; const size_t num_blocks = level.num_blocks; - const auto &is_internal = *level.is_internal; + const auto& is_internal = *level.is_internal; const size_t next_block_size = block_size / this->tau_; std::vector new_block_starts; @@ -386,7 +403,8 @@ class BlockTreeFP2 : public BlockTree { } } - LevelData next_level(level.level_index + 1, next_block_size, + LevelData next_level(level.level_index + 1, + next_block_size, new_block_starts.size()); next_level.block_starts = std::make_unique>(std::move(new_block_starts)); @@ -399,8 +417,9 @@ class BlockTreeFP2 : public BlockTree { /// @param[in] levels A vector containing data for each level, with the first /// entry corresponding to the topmost level. /// - void make_tree(const std::vector &text, - std::vector &levels, int64_t padding) { + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { const bool is_padded = padding > 0; // Count the current number of internal blocks per level @@ -418,7 +437,7 @@ class BlockTreeFP2 : public BlockTree { bool found_back_block = levels[0].is_internal->size() > static_cast(new_num_internal[0]) || !this->CUT_FIRST_LEVELS; - LevelData &top_level = levels.front(); + LevelData& top_level = levels.front(); if (found_back_block) { const size_t n = top_level.num_blocks; const size_t num_internal = new_num_internal[0]; @@ -448,10 +467,10 @@ class BlockTreeFP2 : public BlockTree { // Add level data to the tree for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData &level = levels[level_index]; - LevelData &previous_level = levels[level_index - 1]; - found_back_block |= levels[level_index].is_internal->size() == - static_cast(new_num_internal[level_index]); + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); if (!found_back_block) { level.is_internal.reset(); level.is_internal_rank.reset(); @@ -462,11 +481,16 @@ class BlockTreeFP2 : public BlockTree { continue; } - make_tree_level(levels, new_num_internal, level_index, is_padded, + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, text.size()); // We don't need these anymore - level.is_internal.reset(); + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); @@ -477,22 +501,21 @@ class BlockTreeFP2 : public BlockTree { this->leaf_size = levels.back().block_size / this->tau_; // Construct the leaf string int64_t leaf_count = 0; - auto &last_level = *this->block_tree_types_.back(); - std::vector &last_level_block_starts = - *levels.back().block_starts; - for (uint64_t i = 0; i < last_level.size(); i++) { - if (!last_level[i]) { + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { continue; } + const size_type block_start = last_block_starts[block]; // For every leaf on the last level, we have tau leaf blocks leaf_count += this->tau_; // Iterate through all characters in this child and // add them to the leaf string - for (uint64_t j = 0; - j < static_cast(this->leaf_size * this->tau_); j++) { - if (static_cast(last_level_block_starts[i] + j) < - text.size()) { - this->leaves_.push_back(text[last_level_block_starts[i] + j]); + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); } } } @@ -506,12 +529,13 @@ class BlockTreeFP2 : public BlockTree { /// @param level_index The index of the level to generate. This must be /// strictly greater than 0. /// @param is_padded Whether there is padding in the last block of the tree - void make_tree_level(std::vector &levels, - const std::vector &new_num_internal, - const size_t level_index, const bool is_padded, + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, const size_t text_len) { - LevelData &previous_level = levels[level_index - 1]; - LevelData &level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; size_type new_size = (new_num_internal[level_index - 1] - is_padded) * this->tau_; @@ -524,27 +548,17 @@ class BlockTreeFP2 : public BlockTree { } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; - // 1 neue size hängt von num_internal auf prev_level ab - // 2 die tatsächlichen nodes hängen von num_internal auf level ab - - // Möglichkeiten: - // 1: Zu wenige interne Knoten auf prev_level - // idk wie das sein kann - // Dann wären ja zu viele Blöcke auf dem level davor - // 2: Zu viele interne Knoten auf last level - // Ma gucken - // - // num_internal zählt genau, wo ne 1 im bv steht - // da wurden die blöcke rausgekickt, die - // 1. Earlier Occ haben - // 2. Keine Pointer auf sich haben, - // 3. Keine Internal children haben. - // - auto *is_internal = new BitVector(new_size); - auto *pointers = new sdsl::int_vector<>(2 * new_size - num_internal, 0); - auto *offsets = new sdsl::int_vector<>(2 * new_size - num_internal, 0); + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block size_type num_non_pruned = 0; + // Number of back blocks before the current block size_type num_back_blocks = 0; + // Number of pruned blocks before the current block size_type num_pruned = 0; // We will reuse the allocated memory of the pointers vector to store @@ -552,10 +566,10 @@ class BlockTreeFP2 : public BlockTree { // The invariant is that all values up to i are overwritten while all // values starting after i will still be valid pointers // This contains the number of pruned blocks before the block i - std::vector &num_blocks_skipped = *level.pointers; + std::vector& prefix_pruned_blocks = *level.pointers; for (size_type i = 0; i < level.num_blocks; i++) { const size_type ptr = (*level.pointers)[i]; - num_blocks_skipped[i] = i - num_non_pruned; + prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree if (ptr == PRUNED) { @@ -575,7 +589,7 @@ class BlockTreeFP2 : public BlockTree { // If it is a back block, add its pointer and offset const size_type offset = (*level.offsets)[i]; - (*pointers)[num_back_blocks] = ptr - num_blocks_skipped[ptr]; + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; (*offsets)[num_back_blocks] = offset; num_back_blocks++; } @@ -591,7 +605,7 @@ class BlockTreeFP2 : public BlockTree { /// @brief Prunes the tree of unnecessary nodes. /// @param levels The levels of the tre represented as a vector of levels. - void prune(std::vector &levels) { + void prune(std::vector& levels) { // We need to traverse the block tree in post order, // handling children from right to left. for (int block_index = levels[0].num_blocks - 1; block_index >= 0; @@ -600,35 +614,24 @@ class BlockTreeFP2 : public BlockTree { } } - /// @brief - /// @param levels - /// @param level_index - /// @param block_index + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. /// @return Whether this block is/stays internal after the pruning process - bool prune_block(std::vector &levels, const size_t level_index, + bool prune_block(std::vector& levels, + const size_t level_index, const size_t block_index) { - LevelData &level = levels[level_index]; - BitVector &is_internal = *level.is_internal; - - // If we are at the leaf level, we can't prune anything - // if (level_index == levels.size() - 1) { - // return false; - //} + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; // If the current block is a back block already, there is nothing to prune if (!is_internal[block_index]) { return false; } - const size_type first_child_index = block_index * this->tau_; - - // The number of children before this block which are skipped on the next - // level. Since the children of back blocks are not included on the next - // level, we need to subtract the skipped blocks - const size_t missing_children = - block_index == 0 - ? 0 - : level.is_internal_rank->rank0(block_index) * this->tau_; + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; bool has_internal_children = false; @@ -636,19 +639,17 @@ class BlockTreeFP2 : public BlockTree { // none of which can be pointed to. So only recurse, if we are not on the // last level. if (level_index < levels.size() - 1) { - const size_type last_child_index = - std::min(first_child_index + this->tau_ - 1, - levels[level_index + 1].num_blocks); + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); // Iterate through children in reverse - for (size_type child_index = last_child_index; - child_index >= first_child_index; --child_index) { - has_internal_children |= prune_block(levels, level_index + 1, - child_index - missing_children); + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); } } + // If any of the children is internal, this block stays internal as well if (has_internal_children) { - // If any of the children is internal, this block stays internal as well return true; } @@ -662,41 +663,107 @@ class BlockTreeFP2 : public BlockTree { } // Now we know that there is an earlier occurrence, - // and nothing is pointing here. We will make this block here into a back - // block and mark the children as pruned - if (level_index < levels.size() - 1) { - LevelData &child_level = levels[level_index + 1]; - for (size_type child_index = first_child_index; - child_index < first_child_index + this->tau_; ++child_index) { - const size_type child_pointer = - (*child_level.pointers)[child_index - missing_children]; - const size_type child_offset = - (*child_level.offsets)[child_index - missing_children]; - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child_index - missing_children] = PRUNED; - } - } - - // This is now a back block + // and nothing is pointing here. + // We will make this block here into a back block... is_internal[block_index] = false; (*level.counters)[pointer] += 1; (*level.counters)[pointer + 1] += offset > 0; + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + return false; } public: - BlockTreeFP2(const std::vector &text, const size_t arity, - const size_t root_arity, const size_t max_leaf_length) { + BlockTreeFP2(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length) { this->tau_ = arity; this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; this->map_unique_chars(text); construct(text); } + + ~BlockTreeFP2() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } }; } // namespace pasta From b31c3b3aa838252ef0748e40bae42894b3d95b8c Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 21:50:30 +0200 Subject: [PATCH 08/92] allow using mersenne modulo in rabin karp hasher --- examples/build_bt.cpp | 3 +- .../construction/block_tree_fp2_seq.hpp | 3 +- .../pasta/block_tree/utils/MersenneHash.hpp | 22 ++++++++---- .../block_tree/utils/MersenneRabinKarp.hpp | 35 ++++++++++++++----- 4 files changed, 46 insertions(+), 17 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index b8328f9..ac37498 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -81,6 +81,7 @@ int main(int argc, char** argv) { std::vector text(input.begin(), input.end()); TimePoint now = Clock::now(); + auto bt = std::make_unique>(text, arity, @@ -88,7 +89,7 @@ int main(int argc, char** argv) { leaf_length); // auto bt = - // pasta::make_block_tree_fp(text, arity, leaf_length); + // pasta::make_block_tree_fp(text, arity, leaf_length); auto elapsed = std::chrono::duration_cast(Clock::now() - now) diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 4dfd37a..69a553d 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -41,6 +41,7 @@ class BlockTreeFP2 : public BlockTree { static constexpr size_type SIGMA = 256; static constexpr uint128_t K_PRIME = 2305843009213693951ULL; + static constexpr uint8_t MERSENNE_EXPONENT = 61; using BitVector = pasta::BitVector; // using Rank = pasta::FlatRank; @@ -51,7 +52,7 @@ class BlockTreeFP2 : public BlockTree { typename hash_type = std::hash> using HashMap = std::unordered_map; - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; // using RabinKarp = MersenneRabinKarp; using RabinKarpHash = MersenneHash; // using RabinKarpHash = MersenneHash; diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 51a9460..4a71c5f 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -21,21 +21,28 @@ #pragma once #include +#include #include #include namespace pasta { -template class MersenneHash { +template +class MersenneHash { public: - std::vector const &text_; + std::vector const& text_; uint64_t hash_; uint32_t start_; uint32_t length_; - MersenneHash(std::vector const &text, size_t hash, uint64_t start, + MersenneHash(std::vector const& text, + size_t hash, + uint64_t start, uint64_t length) - : text_(text), hash_(hash), start_(start), length_(length){}; - bool operator==(const MersenneHash &other) const { + : text_(text), + hash_(hash), + start_(start), + length_(length){}; + bool operator==(const MersenneHash& other) const { // std::cout << start_ << " " << other.start_ << std::endl; if (length_ != other.length_) return false; @@ -52,8 +59,9 @@ template class MersenneHash { } // namespace pasta namespace std { -template struct hash> { - std::size_t operator()(const pasta::MersenneHash &hS) const { +template +struct hash> { + std::size_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; } }; diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index c7d5478..b9491ff 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -21,16 +21,26 @@ #pragma once #include "pasta/block_tree/utils/MersenneHash.hpp" + #include namespace pasta { -template class MersenneRabinKarp { +/// +/// @brief A Rabin-Karp rolling hasher. +/// +/// @tparam T The type of the characters in the text. +/// @tparam size_type The type to use for indexing etc. +/// @tparam mersenne_exponent If using a mersenne prime 2^p-1, then this should +/// be p. If this is 0, a normal modulus operation will be used +/// +template +class MersenneRabinKarp { __extension__ typedef unsigned __int128 uint128_t; public: /// The text being hashed - std::vector const &text_; + std::vector const& text_; uint128_t sigma_; /// The start index of the currently hashed window uint64_t init_; @@ -48,9 +58,15 @@ template class MersenneRabinKarp { /// @param init The start index of the first hashed window in the text. /// @param length The window size. /// @param prime A large prime used for modulus operations. - MersenneRabinKarp(std::vector const &text, uint64_t sigma, uint64_t init, - uint64_t length, uint128_t prime) - : text_(text), sigma_(sigma), init_(init), length_(length), + MersenneRabinKarp(std::vector const& text, + uint64_t sigma, + uint64_t init, + uint64_t length, + uint128_t prime) + : text_(text), + sigma_(sigma), + init_(init), + length_(length), prime_(prime) { max_sigma_ = 1; uint128_t fp = 0; @@ -80,9 +96,12 @@ template class MersenneRabinKarp { }; inline uint128_t mersenneModulo(uint128_t k) { - return k % prime_; - // uint128_t i = (k & prime_) + (k >> power_); - // return (i >= prime_) ? i - prime_ : i; + if constexpr (mersenne_exponent == 0) { + return k % prime_; + } else { + uint128_t i = (k & prime_) + (k >> mersenne_exponent); + return (i >= prime_) ? i - prime_ : i; + } }; inline MersenneHash current_hash() const { From 1b1c15c0c69e0f4840d5181a1e60d01027a85a26 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 22:09:59 +0200 Subject: [PATCH 09/92] make MersenneHash copy- and move-constructible and -assignable --- .gitmodules | 3 +++ extlib/robin-hood-hashing | 1 + include/pasta/block_tree/utils/MersenneHash.hpp | 16 ++++++++++++++-- 3 files changed, 18 insertions(+), 2 deletions(-) create mode 160000 extlib/robin-hood-hashing diff --git a/.gitmodules b/.gitmodules index 56edd9f..4ef2ce3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,3 +7,6 @@ [submodule "extlib/tlx"] path = extlib/tlx url = https://github.com/tlx/tlx +[submodule "extlib/robin-hood-hashing"] + path = extlib/robin-hood-hashing + url = https://github.com/martinus/robin-hood-hashing diff --git a/extlib/robin-hood-hashing b/extlib/robin-hood-hashing new file mode 160000 index 0000000..7697343 --- /dev/null +++ b/extlib/robin-hood-hashing @@ -0,0 +1 @@ +Subproject commit 7697343363af4cc3f42cab17be49e6af9ab181e2 diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 4a71c5f..15d5eba 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -22,6 +22,7 @@ #include #include +#include #include #include @@ -30,7 +31,7 @@ namespace pasta { template class MersenneHash { public: - std::vector const& text_; + std::reference_wrapper> text_; uint64_t hash_; uint32_t start_; uint32_t length_; @@ -42,18 +43,29 @@ class MersenneHash { hash_(hash), start_(start), length_(length){}; + + MersenneHash (const MersenneHash &other) = default; + MersenneHash (const MersenneHash &&other) = default; + + MersenneHash &operator=(MersenneHash &other) = default; + MersenneHash &operator=(MersenneHash &&other) = default; + bool operator==(const MersenneHash& other) const { // std::cout << start_ << " " << other.start_ << std::endl; if (length_ != other.length_) return false; + const std::vector &text = text_; + const std::vector &other_text = other.text_; + for (uint64_t i = 0; i < length_; i++) { - if (text_[start_ + i] != other.text_[other.start_ + i]) { + if (text[start_ + i] != other_text[other.start_ + i]) { return false; } } return hash_ == other.hash_; } + }; } // namespace pasta From 281ca73b744b28a977f8732d9957cebeb6fc04b8 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 22:10:42 +0200 Subject: [PATCH 10/92] use robin_hood::unordered_flat_map instead of std::unordered_map --- CMakeLists.txt | 4 +++- include/pasta/block_tree/construction/block_tree_fp2_seq.hpp | 3 ++- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b07d8d9..2bc1b1c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -66,6 +66,7 @@ set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE @@ -75,6 +76,7 @@ target_link_libraries(pasta_block_tree INTERFACE libsais pasta_bit_vector sdsl - tlx) + tlx + robin_hood) ################################################################################ diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 69a553d..4f4f59b 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -29,6 +29,7 @@ #include #include #include +#include __extension__ typedef unsigned __int128 uint128_t; @@ -50,7 +51,7 @@ class BlockTreeFP2 : public BlockTree { template > - using HashMap = std::unordered_map; + using HashMap = robin_hood::unordered_flat_map; using RabinKarp = MersenneRabinKarp; // using RabinKarp = MersenneRabinKarp; From daa2d4a8a5f29b413c852e21cb22e206ac0eda8d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 22:21:17 +0200 Subject: [PATCH 11/92] fix move constructor and assignment signatures --- include/pasta/block_tree/utils/MersenneHash.hpp | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 15d5eba..910add6 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -44,20 +44,20 @@ class MersenneHash { start_(start), length_(length){}; - MersenneHash (const MersenneHash &other) = default; - MersenneHash (const MersenneHash &&other) = default; + constexpr MersenneHash(const MersenneHash& other) = default; + constexpr MersenneHash(MersenneHash&& other) = default; - MersenneHash &operator=(MersenneHash &other) = default; - MersenneHash &operator=(MersenneHash &&other) = default; + MersenneHash& operator=(const MersenneHash& other) = default; + MersenneHash& operator=(MersenneHash&& other) = default; bool operator==(const MersenneHash& other) const { // std::cout << start_ << " " << other.start_ << std::endl; if (length_ != other.length_) return false; - const std::vector &text = text_; - const std::vector &other_text = other.text_; - + const std::vector& text = text_; + const std::vector& other_text = other.text_; + for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { return false; @@ -65,7 +65,6 @@ class MersenneHash { } return hash_ == other.hash_; } - }; } // namespace pasta From 1f59acbc001c8f74a87682988766e081eb5a5fa1 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 19 Aug 2023 23:12:27 +0200 Subject: [PATCH 12/92] switch to robin_hood::unordered_map --- include/pasta/block_tree/construction/block_tree_fp2_seq.hpp | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 4f4f59b..4cedb82 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -51,7 +51,7 @@ class BlockTreeFP2 : public BlockTree { template > - using HashMap = robin_hood::unordered_flat_map; + using HashMap = robin_hood::unordered_map; using RabinKarp = MersenneRabinKarp; // using RabinKarp = MersenneRabinKarp; From 4cdc0f8bd36df7c96e3f9d4e2bfe3e948e1263aa Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 4 Sep 2023 18:29:59 +0200 Subject: [PATCH 13/92] wip parallel implementation --- .gitmodules | 6 + CMakeLists.txt | 4 + examples/build_bt.cpp | 60 +- extlib/growt | 1 + extlib/parallel-hashmap | 1 + .../construction/block_tree_fp_par_phmap.hpp | 889 ++++++++++++++++++ .../pasta/block_tree/utils/MersenneHash.hpp | 10 +- 7 files changed, 952 insertions(+), 19 deletions(-) create mode 160000 extlib/growt create mode 160000 extlib/parallel-hashmap create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp diff --git a/.gitmodules b/.gitmodules index 4ef2ce3..6d57173 100644 --- a/.gitmodules +++ b/.gitmodules @@ -10,3 +10,9 @@ [submodule "extlib/robin-hood-hashing"] path = extlib/robin-hood-hashing url = https://github.com/martinus/robin-hood-hashing +[submodule "extlib/growt"] + path = extlib/growt + url = git@github.com:TooBiased/growt.git +[submodule "extlib/parallel-hashmap"] + path = extlib/parallel-hashmap + url = git@github.com:greg7mdp/parallel-hashmap.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bc1b1c..d34e278 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -78,5 +78,9 @@ target_link_libraries(pasta_block_tree INTERFACE sdsl tlx robin_hood) +target_include_directories(pasta_block_tree INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) +target_include_directories(pasta_block_tree INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parallel-hashmap/parallel_hashmap) ################################################################################ diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index ac37498..f5af22f 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,11 +18,15 @@ * ******************************************************************************/ +#include #include #include #include -#include -#include +//#include +//#include +#include +// #include +// #include #include #include @@ -72,34 +76,60 @@ int main(int argc, char** argv) { << "\nmax leaf length: " << leaf_length << "\nsaving to " << out_path << std::endl; - std::string input; - std::ifstream t(argv[1]); - std::stringstream buffer; - buffer << t.rdbuf(); - input = buffer.str(); - - std::vector text(input.begin(), input.end()); + std::vector text; + { + std::string input; + std::ifstream t(argv[1]); + std::stringstream buffer; + buffer << t.rdbuf(); + input = buffer.str(); + text = std::vector(input.begin(), input.end()); + } TimePoint now = Clock::now(); + /* auto bt = std::make_unique>(text, arity, root_arity, leaf_length); - - // auto bt = - // pasta::make_block_tree_fp(text, arity, leaf_length); - + */ + /* + auto bt = std::make_unique>(text, + arity, + leaf_length, + root_arity, + 256, + true, + true); + */ + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length, + 8); + /* + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length); +*/ auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" << "Time: " << elapsed << "ms" << std::endl; - // std::ofstream ot(out_path); + + std::ofstream ot(out_path); // bt->serialize(ot); - // ot.close(); + for (size_t i = 0; i < text.size(); ++i) { + ot << (char)bt->access(i); + } + ot.close(); return 0; } diff --git a/extlib/growt b/extlib/growt new file mode 160000 index 0000000..0c1148e --- /dev/null +++ b/extlib/growt @@ -0,0 +1 @@ +Subproject commit 0c1148ebcdfd4c04803be79706533ad09cc81d37 diff --git a/extlib/parallel-hashmap b/extlib/parallel-hashmap new file mode 160000 index 0000000..df7935a --- /dev/null +++ b/extlib/parallel-hashmap @@ -0,0 +1 @@ +Subproject commit df7935aca33afdae9218ea57f57a35dab3eec8fe diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp new file mode 100644 index 0000000..b882de0 --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -0,0 +1,889 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "data-structures/hash_table_mods.hpp" +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include +#include +#include +#include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +#define BT_NUM_THREADS 2 + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFPParPH : public BlockTree { + constexpr static size_type NO_EARLIER_OCC = -1; + constexpr static size_type PRUNED = -2; + + static constexpr size_type SIGMA = 256; + static constexpr uint128_t K_PRIME = 2305843009213693951ULL; + static constexpr uint8_t MERSENNE_EXPONENT = 61; + + using BitVector = pasta::BitVector; + using Rank = pasta::RankSelect; + + /// A concurrent hash map + template > + using HashMap = + phmap::parallel_flat_hash_map; + // robin_hood::unordered_map; + + /// A rabin karp hasher preconfigured for the current template parameters + using RabinKarp = MersenneRabinKarp; + /// A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// A hash map with rabin karp hashes as keys + template + using RabinKarpMap = HashMap; + + struct LevelData { + /// Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// The block from which a back block is copying + std::unique_ptr> pointers; + /// The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// The number of back blocks pointing to the block + std::unique_ptr> counters; + /// Block start indices + std::unique_ptr> block_starts; + /// The block size on this level + size_type block_size; + /// The index of the current level. First level is 0, second level is 1 etc. + size_type level_index; + /// The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + + /// @brief Checks whether a block is adjacent in the text + /// to its predecessor on this level + [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { + return (*block_starts)[i - 1] + static_cast(block_size) == + (*block_starts)[i]; + } + }; + + void construct(const std::vector& text) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + // std::cout << "level " << level << std::endl; + LevelData& current = levels.back(); + + TimePoint now = Clock::now(); + scan_block_pairs(text, current, is_padded); + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + scan_blocks(text, current, is_padded); + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + } + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms, blocks: " << (blocks_ns / 1'000'000) + << "ms, generate: " << (generate_ns / 1'000'000) << "ms" + << std::endl; + + prune(levels); + make_tree(text, levels, padding); + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + struct PairOccurrences { + /// The first block in the text in which the content appears + size_type first_occ_block; + + std::vector occurrences; + + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + inline void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @briefScan through the blocks pairwise in order to identify which blocks + /// should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + static void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const size_t pair_size = 2 * block_size; + + if (num_blocks < 4) { + level.is_internal = std::make_unique(num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap map(num_blocks); + + { + RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); + for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + rk.restart((*level.block_starts)[i]); + RabinKarpHash hash = rk.current_hash(); + auto ptr = map.find(hash); + if (ptr == map.end()) { + auto [insert_ptr, _] = map.insert({hash, PairOccurrences(i)}); + ptr = insert_ptr; + } + ptr->second.add_block(i); + } + } + + const size_t num_block_pairs = num_blocks - 1 - is_padded; + +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(num_block_pairs, pair_size, level, block_size, map, text) + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + { + const auto& block_starts = *level.block_starts; + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); + const size_t thread_id = omp_get_thread_num(); + + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + + for (auto it = map.begin(); it != map.end(); ++it) { + const PairOccurrences& pair_occs = it->second; + const std::vector& occs = pair_occs.occurrences; + const size_type first_block = occs.front(); + const bool has_prev_occ = pair_occs.first_occ_block < first_block; + const size_type skip = has_prev_occ ? 0 : 1; + for (size_t i = skip; i < occs.size(); i++) { + const size_type occ = occs[i]; + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + map.erase(it); + } + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(num_blocks); + auto& is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_t i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } + level.is_internal_rank = std::make_unique(is_internal); + } + + /// @brief Scan through the windows starting in a block and mark + /// them + /// accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// index. + /// @param markings A vector storing the marks on a block. Marks are 2-bit + /// integers. + /// If the MSB is set, that means that the content of the block and its + /// successor has an earlier occurrence. If the LSB being set means that + /// the content of the block and its predecessor has an earlier occurrence. + /// @param block_size The size of blocks on the current level. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap& map, + const size_t block_size, + const size_type current_block_index) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + continue; + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + struct BlockOccurrences { + bool handled; + std::vector occurrences; + BlockOccurrences() : handled(false), occurrences() {} + }; + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + static void scan_blocks(const std::vector& s, + LevelData& level_data, + const bool is_padded) { + const size_t block_size = level_data.block_size; + const size_t num_blocks = level_data.num_blocks; + const std::vector& block_starts = *level_data.block_starts; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap links(num_blocks); + for (size_t i = 0; i < num_blocks - is_padded; ++i) { + const RabinKarpHash hash = + RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) + .current_hash(); + links.insert({hash, BlockOccurrences()}); + auto& [found_hash, occs] = *links.find(hash); + occs.occurrences.push_back(i); + } + + const size_t num_total_iterations = num_blocks - is_padded - 1; + // #pragma omp parallel + { + const size_t thread_id = 0; // omp_get_thread_num(); + const size_t segment_size = + ceil_div(num_total_iterations, 1); // omp_get_num_threads()); + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size()) { + RabinKarp rk(s, SIGMA, block_starts[start], block_size, K_PRIME); + for (size_t current_block_index = start; current_block_index < end; + ++current_block_index) { + if (static_cast(rk.init_) != + block_starts[current_block_index]) { + rk.restart(block_starts[current_block_index]); + } + + if (level_data.next_is_adjacent(current_block_index)) { + scan_windows_in_block(rk, links, level_data, current_block_index); + continue; + } + } + } + } + } + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap& links, + LevelData& level_data, + const size_type current_block_index) { + const BitVector& is_internal = *level_data.is_internal; + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + auto& [block_hash, found_block_occs] = *found; + if (found_block_occs.handled) { + continue; + } + + const auto& found_blocks = found_block_occs.occurrences; + const size_t num_found_blocks = found_blocks.size(); + // In this case, we are hashing an actual block right now (not just an + // arbitrary window). As a result, the first block in the vector is the + // block we are currently hashing in + for (size_t i = 0; i < num_found_blocks; ++i) { + const size_type block_index = found_blocks[i]; + if (block_index == current_block_index || + (offset > 0 && block_index == current_block_index + 1)) { + continue; + } + (*level_data.pointers)[block_index] = current_block_index; + (*level_data.offsets)[block_index] = offset; + // We increment the counter for the block that is being pointed to + // if the current block is actually a back block + // If the offset is greater than 0, + // then it also overlaps into the next block + const bool is_back_block = !is_internal[block_index]; + (*level_data.counters)[current_block_index] += is_back_block; + (*level_data.counters)[current_block_index + 1] += + is_back_block && (offset > 0); + } + // TODO found may not be handled again! + found_block_occs.handled = true; + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParPH(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParPH() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } +}; + +} // namespace pasta + +#undef BT_NUM_THREADS \ No newline at end of file diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 910add6..378bd98 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -31,7 +31,7 @@ namespace pasta { template class MersenneHash { public: - std::reference_wrapper> text_; + const std::vector* text_; uint64_t hash_; uint32_t start_; uint32_t length_; @@ -39,11 +39,13 @@ class MersenneHash { size_t hash, uint64_t start, uint64_t length) - : text_(text), + : text_(&text), hash_(hash), start_(start), length_(length){}; + constexpr MersenneHash() : text_(nullptr), hash_(0), start_(0), length_(0){}; + constexpr MersenneHash(const MersenneHash& other) = default; constexpr MersenneHash(MersenneHash&& other) = default; @@ -55,8 +57,8 @@ class MersenneHash { if (length_ != other.length_) return false; - const std::vector& text = text_; - const std::vector& other_text = other.text_; + const std::vector& text = *text_; + const std::vector& other_text = *other.text_; for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { From e9e6f6c711c2326f3c61960d7cdc1b96bfb71bdd Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 6 Sep 2023 16:15:29 +0200 Subject: [PATCH 14/92] parallelize pair scan step --- examples/build_bt.cpp | 50 ++--- .../construction/block_tree_fp_par_phmap.hpp | 173 ++++++++++++------ 2 files changed, 145 insertions(+), 78 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index f5af22f..1a4aac0 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,11 +22,11 @@ #include #include #include -//#include -//#include +// #include +// #include #include -// #include -// #include +// #include +// #include #include #include @@ -89,27 +89,29 @@ int main(int argc, char** argv) { TimePoint now = Clock::now(); /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length); + */ /* - auto bt = std::make_unique>(text, - arity, - leaf_length, - root_arity, - 256, - true, - true); + auto bt = std::make_unique>(text, + arity, + leaf_length, + root_arity, + 256, + true, + true); */ + auto bt = std::make_unique>(text, arity, root_arity, leaf_length, 8); + /* auto bt = std::make_unique>(text, @@ -124,12 +126,18 @@ int main(int argc, char** argv) { std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" << "Time: " << elapsed << "ms" << std::endl; - std::ofstream ot(out_path); - // bt->serialize(ot); + // std::ofstream ot(out_path); + // bt->serialize(ot); + #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { - ot << (char)bt->access(i); + const auto c = bt->access(i); + if (c != text[i]) { + std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] + << "\nActual: " << c << std::endl; + exit(1); + } } - ot.close(); + // ot.close(); return 0; } diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp index b882de0..885bfac 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -26,7 +26,9 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include #include +#include #include #include #include @@ -38,7 +40,7 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; -#define BT_NUM_THREADS 2 +#define BT_NUM_THREADS 8 __extension__ typedef unsigned __int128 uint128_t; @@ -61,7 +63,7 @@ class BlockTreeFPParPH : public BlockTree { typename value_type, typename hash_type = std::hash> using HashMap = - phmap::parallel_flat_hash_map; + phmap::parallel_node_hash_map; // robin_hood::unordered_map; /// A rabin karp hasher preconfigured for the current template parameters @@ -73,6 +75,14 @@ class BlockTreeFPParPH : public BlockTree { template using RabinKarpMap = HashMap; +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + +private: + /// @brief Contains data about a block tree level under construction struct LevelData { /// Contains a 1 for each internal block (= block with children) /// and a 0 for each block that has a back pointer @@ -179,8 +189,12 @@ class BlockTreeFPParPH : public BlockTree { } std::cout << "pairs: " << (pairs_ns / 1'000'000) - << "ms, blocks: " << (blocks_ns / 1'000'000) - << "ms, generate: " << (generate_ns / 1'000'000) << "ms" + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\ngenerate: " << (generate_ns / 1'000'000) << "ms" << std::endl; prune(levels); @@ -195,10 +209,15 @@ class BlockTreeFPParPH : public BlockTree { } struct PairOccurrences { - /// The first block in the text in which the content appears + /// @brief The first block in the text in which the content appears size_type first_occ_block; - - std::vector occurrences; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; inline explicit PairOccurrences(size_type first_occ_block_) : first_occ_block(first_occ_block_), @@ -224,55 +243,66 @@ class BlockTreeFPParPH : public BlockTree { /// @param level The data for the current level. /// /// @return The block start indices for the next level of the tree - static void scan_block_pairs(const std::vector& text, - LevelData& level, - const bool is_padded) { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const size_t pair_size = 2 * block_size; - - if (num_blocks < 4) { - level.is_internal = std::make_unique(num_blocks, true); + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); level.is_internal_rank = std::make_unique(*level.is_internal); return; } // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - RabinKarpMap map(num_blocks); + RabinKarpMap map(level.num_blocks); + TimePoint now = Clock::now(); + +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level, map, text, now, is_padded) { - RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); - for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_blocks = level.num_blocks; + const size_t num_block_pairs = num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; +#pragma omp for + for (size_t i = 0; i < num_block_pairs; ++i) { // If the next block is not adjacent, we cannot hash the pair starting // at the current block if (!level.next_is_adjacent(i)) { continue; } // Move the hasher to the current block pair - rk.restart((*level.block_starts)[i]); + RabinKarp rk(text, SIGMA, block_starts[i], pair_size, K_PRIME); RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry auto ptr = map.find(hash); if (ptr == map.end()) { auto [insert_ptr, _] = map.insert({hash, PairOccurrences(i)}); ptr = insert_ptr; } ptr->second.add_block(i); + ptr->second.update(i); + } +#pragma omp barrier +#pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); } - } - - const size_t num_block_pairs = num_blocks - 1 - is_padded; -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(num_block_pairs, pair_size, level, block_size, map, text) - // Hash every window and determine for all block pairs whether they have - // previous occurrences. - { - const auto& block_starts = *level.block_starts; + // Hash every window and determine for all block pairs whether they have + // previous occurrences. size_t segment_size = std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); const size_t thread_id = omp_get_thread_num(); + // Start and end index of the const auto start = thread_id * segment_size; const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); @@ -288,36 +318,49 @@ class BlockTreeFPParPH : public BlockTree { } } + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. // The MSB is 1 iff the block and its successor have a prior occurrence. // The LSB is 1 iff the block and its predecessor have a prior occurrence. - sdsl::int_vector<2> markings(num_blocks, 0); - + sdsl::int_vector<2> markings(level.num_blocks, 0); for (auto it = map.begin(); it != map.end(); ++it) { const PairOccurrences& pair_occs = it->second; - const std::vector& occs = pair_occs.occurrences; - const size_type first_block = occs.front(); - const bool has_prev_occ = pair_occs.first_occ_block < first_block; - const size_type skip = has_prev_occ ? 0 : 1; - for (size_t i = skip; i < occs.size(); i++) { - const size_type occ = occs[i]; - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; + const auto& block_indices = pair_occs.occurrences; + for (auto list_it = block_indices.cbegin(); + list_it != block_indices.cend(); + ++list_it) { + const size_type occ = *list_it; + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } } map.erase(it); } + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); // Generate the bit vector indicating which blocks are internal - level.is_internal = std::make_unique(num_blocks); + level.is_internal = std::make_unique(level.num_blocks); + auto& is_internal = *level.is_internal; is_internal[0] = true; - is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; - for (size_t i = 0; i < num_blocks - 1; ++i) { + is_internal[level.num_blocks - 1] = markings[level.num_blocks - 1] != 0b01; + for (size_type i = 0; i < level.num_blocks - 1; ++i) { const bool block_is_internal = markings[i] != 0b11; is_internal[i] = block_is_internal; } - level.is_internal_rank = std::make_unique(is_internal); + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + level.is_internal_rank = std::make_unique(*level.is_internal); } /// @brief Scan through the windows starting in a block and mark @@ -328,13 +371,8 @@ class BlockTreeFPParPH : public BlockTree { /// The supplied `RabinKarp` hasher must be at the start of the block. /// @param rk A Rabin-Karp hasher whose state is at the start of the block. /// @param map The map containing the hashes of block pairs mapped to their - /// index. - /// @param markings A vector storing the marks on a block. Marks are 2-bit - /// integers. - /// If the MSB is set, that means that the content of the block and its - /// successor has an earlier occurrence. If the LSB being set means that - /// the content of the block and its predecessor has an earlier occurrence. - /// @param block_size The size of blocks on the current level. + /// block indexes at which they occur. + /// @param current_block_index The index of the block being currently hashed. static inline void scan_windows_in_block_pair(RabinKarp& rk, RabinKarpMap& map, @@ -353,9 +391,29 @@ class BlockTreeFPParPH : public BlockTree { } struct BlockOccurrences { - bool handled; + size_type first_occ_block; + size_type first_occ_offset; std::vector occurrences; - BlockOccurrences() : handled(false), occurrences() {} + BlockOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + first_occ_offset(0), + occurrences() {} + + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + inline void update(size_type block_index, size_type block_offset) { + if (block_index < first_occ_block || + (first_occ_block == block_index && block_offset < first_occ_offset)) { + first_occ_block = block_index; + first_occ_offset = block_offset; + } + } }; /// @brief Determine the positions for each block's earliest occurrence if @@ -392,7 +450,7 @@ class BlockTreeFPParPH : public BlockTree { const RabinKarpHash hash = RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) .current_hash(); - links.insert({hash, BlockOccurrences()}); + links.insert({hash, BlockOccurrences(i)}); auto& [found_hash, occs] = *links.find(hash); occs.occurrences.push_back(i); } @@ -448,9 +506,9 @@ class BlockTreeFPParPH : public BlockTree { continue; } auto& [block_hash, found_block_occs] = *found; - if (found_block_occs.handled) { - continue; - } + + // found_block_occs.update(current_block_index, offset); + // continue; const auto& found_blocks = found_block_occs.occurrences; const size_t num_found_blocks = found_blocks.size(); @@ -474,8 +532,9 @@ class BlockTreeFPParPH : public BlockTree { (*level_data.counters)[current_block_index + 1] += is_back_block && (offset > 0); } + links.erase(found); // TODO found may not be handled again! - found_block_occs.handled = true; + // found_block_occs.handled = true; } } From 88480ba0c29dd168a9c23b86f1e10308b78dbc01 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 7 Sep 2023 19:57:31 +0200 Subject: [PATCH 15/92] parallelize part of the second step --- examples/build_bt.cpp | 6 +- .../construction/block_tree_fp_par_phmap.hpp | 224 +++++++++++------- 2 files changed, 147 insertions(+), 83 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 1a4aac0..cf13d70 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -126,9 +126,9 @@ int main(int argc, char** argv) { std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" << "Time: " << elapsed << "ms" << std::endl; - // std::ofstream ot(out_path); - // bt->serialize(ot); - #pragma omp parallel for +// std::ofstream ot(out_path); +// bt->serialize(ot); +#pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp index 885bfac..b833211 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -81,6 +81,10 @@ class BlockTreeFPParPH : public BlockTree { size_t bp_markings_ns = 0; size_t bp_bitvec_ns = 0; + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + private: /// @brief Contains data about a block tree level under construction struct LevelData { @@ -187,18 +191,29 @@ class BlockTreeFPParPH : public BlockTree { Clock::now() - now) .count(); } + TimePoint now = Clock::now(); + prune(levels); + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + make_tree(text, levels, padding); + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); std::cout << "pairs: " << (pairs_ns / 1'000'000) << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) << "ms,\nblocks: " << (blocks_ns / 1'000'000) - << "ms,\ngenerate: " << (generate_ns / 1'000'000) << "ms" - << std::endl; - - prune(levels); - make_tree(text, levels, padding); + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) + << "ms,\nprune: " << (prune_ns / 1'000'000) + << "ms,\nmake: " << (make_ns / 1'000'000) << "ms" << std::endl; } /// @brief Returns the ceiling of x / y for x > 0; @@ -302,7 +317,7 @@ class BlockTreeFPParPH : public BlockTree { std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); const size_t thread_id = omp_get_thread_num(); - // Start and end index of the + // Start and end index of the current thread's segment const auto start = thread_id * segment_size; const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); @@ -330,11 +345,7 @@ class BlockTreeFPParPH : public BlockTree { sdsl::int_vector<2> markings(level.num_blocks, 0); for (auto it = map.begin(); it != map.end(); ++it) { const PairOccurrences& pair_occs = it->second; - const auto& block_indices = pair_occs.occurrences; - for (auto list_it = block_indices.cbegin(); - list_it != block_indices.cend(); - ++list_it) { - const size_type occ = *list_it; + for (const size_type occ : pair_occs.occurrences) { if (pair_occs.first_occ_block < occ) { markings[occ] = markings[occ] | 0b10; markings[occ + 1] = markings[occ + 1] | 0b01; @@ -391,14 +402,36 @@ class BlockTreeFPParPH : public BlockTree { } struct BlockOccurrences { - size_type first_occ_block; - size_type first_occ_offset; - std::vector occurrences; + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + BlockOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - first_occ_offset(0), + : first_occ({first_occ_block_, 0}), occurrences() {} + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + inline void add_block(size_type block_index) { occurrences.push_back(block_index); } @@ -408,11 +441,12 @@ class BlockTreeFPParPH : public BlockTree { /// @param block_index The block index of an occurrence /// @param block_index The offset of that occurrence inline void update(size_type block_index, size_type block_offset) { - if (block_index < first_occ_block || - (first_occ_block == block_index && block_offset < first_occ_offset)) { - first_occ_block = block_index; - first_occ_offset = block_offset; + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { } + //||(first_occ_block == block_index && block_offset < first_occ_offset)) { } }; @@ -423,12 +457,10 @@ class BlockTreeFPParPH : public BlockTree { /// @param level_data The data for the current level /// @param is_padded true, iff the last block of the level extends past the /// end of the text - static void scan_blocks(const std::vector& s, - LevelData& level_data, - const bool is_padded) { - const size_t block_size = level_data.block_size; + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded) { const size_t num_blocks = level_data.num_blocks; - const std::vector& block_starts = *level_data.block_starts; level_data.pointers = std::make_unique>(num_blocks, NO_EARLIER_OCC); @@ -446,41 +478,100 @@ class BlockTreeFPParPH : public BlockTree { // In addition to the vector, there is a boolean which denotes whether a // hash has already been processed RabinKarpMap links(num_blocks); - for (size_t i = 0; i < num_blocks - is_padded; ++i) { - const RabinKarpHash hash = - RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) - .current_hash(); - links.insert({hash, BlockOccurrences(i)}); - auto& [found_hash, occs] = *links.find(hash); - occs.occurrences.push_back(i); - } - const size_t num_total_iterations = num_blocks - is_padded - 1; - // #pragma omp parallel + TimePoint now = Clock::now(); + +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level_data, text, links, now, is_padded, std::cout) { - const size_t thread_id = 0; // omp_get_thread_num(); + const std::vector& block_starts = *level_data.block_starts; +#pragma omp single + for (size_type i = 0; i < level_data.num_blocks - is_padded; ++i) { + const RabinKarp rk(text, + SIGMA, + block_starts[i], + level_data.block_size, + K_PRIME); + const RabinKarpHash hash = rk.current_hash(); +#pragma omp critical + { + auto ptr = links.find(hash); + if (ptr == links.end()) { + auto [insert_ptr, _] = links.emplace(hash, BlockOccurrences(i)); + ptr = insert_ptr; + } + + ptr->second.add_block(i); + ptr->second.update(i, 0); + } + } +#pragma omp barrier + +#pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + + const size_t thread_id = omp_get_thread_num(); const size_t segment_size = - ceil_div(num_total_iterations, 1); // omp_get_num_threads()); + ceil_div(num_total_iterations, omp_get_num_threads()); const size_t start = thread_id * segment_size; const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); - // Hash every window and find the first occurrences for every block. - if (start < block_starts.size()) { - RabinKarp rk(s, SIGMA, block_starts[start], block_size, K_PRIME); - for (size_t current_block_index = start; current_block_index < end; - ++current_block_index) { - if (static_cast(rk.init_) != - block_starts[current_block_index]) { - rk.restart(block_starts[current_block_index]); - } - if (level_data.next_is_adjacent(current_block_index)) { - scan_windows_in_block(rk, links, level_data, current_block_index); + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + SIGMA, + block_starts[start], + level_data.block_size, + K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { continue; } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); } } +#pragma omp barrier } + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + for (auto it = links.cbegin(); it != links.cend(); ++it) { + // The occurrences of all blocks with a given hash + const BlockOccurrences& occs = it->second; + for (const size_type occ : occs.occurrences) { + auto first_occ = occs.first_occ.load(); + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + } + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); } /// @brief Scans through block-sized windows starting inside one block and /// tries to find earlier occurrences of blocks. Non-internal blocks will @@ -496,7 +587,6 @@ class BlockTreeFPParPH : public BlockTree { RabinKarpMap& links, LevelData& level_data, const size_type current_block_index) { - const BitVector& is_internal = *level_data.is_internal; for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { const RabinKarpHash hash = rk.current_hash(); @@ -505,36 +595,10 @@ class BlockTreeFPParPH : public BlockTree { if (found == links.end()) { continue; } - auto& [block_hash, found_block_occs] = *found; - - // found_block_occs.update(current_block_index, offset); - // continue; - - const auto& found_blocks = found_block_occs.occurrences; - const size_t num_found_blocks = found_blocks.size(); - // In this case, we are hashing an actual block right now (not just an - // arbitrary window). As a result, the first block in the vector is the - // block we are currently hashing in - for (size_t i = 0; i < num_found_blocks; ++i) { - const size_type block_index = found_blocks[i]; - if (block_index == current_block_index || - (offset > 0 && block_index == current_block_index + 1)) { - continue; - } - (*level_data.pointers)[block_index] = current_block_index; - (*level_data.offsets)[block_index] = offset; - // We increment the counter for the block that is being pointed to - // if the current block is actually a back block - // If the offset is greater than 0, - // then it also overlaps into the next block - const bool is_back_block = !is_internal[block_index]; - (*level_data.counters)[current_block_index] += is_back_block; - (*level_data.counters)[current_block_index + 1] += - is_back_block && (offset > 0); - } - links.erase(found); - // TODO found may not be handled again! - // found_block_occs.handled = true; + // + // TODO This must be thread safe + found->second.update(current_block_index, offset); + continue; } } @@ -941,7 +1005,7 @@ class BlockTreeFPParPH : public BlockTree { } return true; } -}; +}; // namespace pasta } // namespace pasta From 5cd8f07a6733a107c5418395f0c07cf11801bb19 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 26 Sep 2023 00:47:40 +0200 Subject: [PATCH 16/92] inital version of sharded hash map algorithm --- .gitmodules | 9 + CMakeLists.txt | 14 +- CMakePresets.json | 4 +- extlib/Jiffy | 1 + extlib/waitfree-mpsc-queue | 1 + .../block_tree_fp_par_sharded.hpp | 1143 +++++++++++++++++ .../pasta/block_tree/utils/MersenneHash.hpp | 1 + .../block_tree/utils/mpsc_queue/jiffy.hpp | 87 ++ .../block_tree/utils/mpsc_queue/mpscq.hpp | 43 + .../block_tree/utils/mpsc_queue/queue.hpp | 32 + .../utils/mpsc_queue/stupid_queue.hpp | 73 ++ .../pasta/block_tree/utils/sharded_map.hpp | 312 +++++ 12 files changed, 1717 insertions(+), 3 deletions(-) create mode 160000 extlib/Jiffy create mode 160000 extlib/waitfree-mpsc-queue create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp create mode 100644 include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp create mode 100644 include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp create mode 100644 include/pasta/block_tree/utils/mpsc_queue/queue.hpp create mode 100644 include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp create mode 100644 include/pasta/block_tree/utils/sharded_map.hpp diff --git a/.gitmodules b/.gitmodules index 6d57173..7421660 100644 --- a/.gitmodules +++ b/.gitmodules @@ -16,3 +16,12 @@ [submodule "extlib/parallel-hashmap"] path = extlib/parallel-hashmap url = git@github.com:greg7mdp/parallel-hashmap.git +[submodule "extlib/waitfree-mpsc-queue"] + path = extlib/waitfree-mpsc-queue + url = https://github.com/dbittman/waitfree-mpsc-queue +[submodule "extlib/Jiffy"] + path = extlib/Jiffy + url = https://github.com/DolevAdas/Jiffy +[submodule "fastwfc"] + path = extlib/fastwfc/fast-wait-free-queue + url = https://github.com/chaoran/fast-wait-free-queue diff --git a/CMakeLists.txt b/CMakeLists.txt index d34e278..f50490d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -68,6 +68,16 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) +add_library(waitfree-mpsc-queue + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) +target_include_directories(waitfree-mpsc-queue PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue) + + +add_library(jiffy INTERFACE) +target_include_directories(jiffy INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy) + add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -77,7 +87,9 @@ target_link_libraries(pasta_block_tree INTERFACE pasta_bit_vector sdsl tlx - robin_hood) + robin_hood + waitfree-mpsc-queue + jiffy) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE diff --git a/CMakePresets.json b/CMakePresets.json index edfaf46..10d2973 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -16,7 +16,7 @@ "cacheVariables": { "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" } }, @@ -59,7 +59,7 @@ "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address -static-libasan" } } ], diff --git a/extlib/Jiffy b/extlib/Jiffy new file mode 160000 index 0000000..82cb6fb --- /dev/null +++ b/extlib/Jiffy @@ -0,0 +1 @@ +Subproject commit 82cb6fbff6b6ff28b6cfc1e2c1231e85570d03bb diff --git a/extlib/waitfree-mpsc-queue b/extlib/waitfree-mpsc-queue new file mode 160000 index 0000000..020ba22 --- /dev/null +++ b/extlib/waitfree-mpsc-queue @@ -0,0 +1 @@ +Subproject commit 020ba2262c48b24828bed98e5cc63c5529ca24ce diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp new file mode 100644 index 0000000..17b17bc --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -0,0 +1,1143 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "data-structures/hash_table_mods.hpp" +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" +#include "pasta/block_tree/utils/mpsc_queue/queue.hpp" +#include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" +#include "pasta/block_tree/utils/sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BT_NUM_THREADS 8 +#define BT_FILL_THRESHOLD 0.5 +#define BT_QUEUE_CAPACITY 1024 + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template typename queue_type = StupidQueue> +class BlockTreeFPParSharded : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 61; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A concurrent queue for communication between threads in the + /// sharded hash map. + template + requires MpscQueue, elem_type> + using Queue = queue_type; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + robin_hood::unordered_node_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + ShardedMap; + +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + size_type block_size; + /// @brief The index of the current level. First level is 0, second level is + /// 1 etc. + size_type level_index; + /// @brief The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + inline void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + inline void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + inline void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + std::cout << "level " << level << std::endl; + LevelData& current = levels.back(); + + TimePoint now = Clock::now(); + scan_block_pairs(text, current, is_padded); + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + scan_blocks(text, current, is_padded); + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + } + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; + prune(levels); + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; + make_tree(text, levels, padding); + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + + TimePoint now = Clock::now(); + std::atomic_size_t num_threads_finished = 0; + +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level, map, text, now, is_padded, std::cout, num_threads_finished) + { + const size_t thread_id = omp_get_thread_num(); + const size_t num_threads = omp_get_num_threads(); + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + RabinKarp rk(text, SIGMA, block_starts[i], pair_size, PRIME); + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + map.insert(hash, i); + if (map.should_handle_queue(thread_id)) { + map.handle_queue(thread_id); + } + } + num_threads_finished.fetch_add(1); + // Threads might be done before the others with the loop. So essentially, + // we need all threads to wait for the others to finish the loop and + // handle thread events that might come in from the other threads + do { + map.handle_queue(thread_id); + } while (num_threads_finished.load() < num_threads); +#pragma omp barrier +#pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the given + /// map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); + TimePoint now = Clock::now(); + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + // Generate the bit vector indicating which blocks are internal + + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + continue; + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + + TimePoint now = Clock::now(); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_threads_finished = 0; +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_threads_finished, \ + std::cout) + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + const size_t block_size = level_data.block_size; + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + // FIXME This causes issues when run in parallel + // In make_tree, we get an error when deallocating vectors in LevelData + // Seems like in this case, the algorithm fails to identify some earlier + // occurrences for non-internal blocks, leading to writes to offsets[-1] + // etc. later on. + for (size_t i = start; i < end; ++i) { + const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); + RabinKarpHash hash = rk.current_hash(); + links.insert(hash, {i, 0}); + // The thread checks whether it should handle the inserts in its queue + if (links.should_handle_queue(thread_id)) { + links.handle_queue(thread_id); + } + } + num_threads_finished.fetch_add(1); + // Threads might be done with the loop before others. So essentially, + // we need all threads to wait for the others to finish the loop and + // handle thread events that might come in from the other threads + do { + links.handle_queue(thread_id); + } while (num_threads_finished.load() < num_threads); +#pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + // links.print_map_loads(); + } + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks will + /// have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + found->second.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + assert(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + assert(child_pointer >= 0); + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParSharded() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + } + return true; + } +}; // namespace pasta + +} // namespace pasta + +#undef BT_NUM_THREADS diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 378bd98..8773cc5 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -24,6 +24,7 @@ #include #include #include +#include #include namespace pasta { diff --git a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp new file mode 100644 index 0000000..7b83304 --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp @@ -0,0 +1,87 @@ +#pragma once + +#include "queue.hpp" + +#include +#include +#include +#include +#include +#include + +namespace pasta { + +namespace jiffy { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wreorder" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wparentheses" +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#pragma GCC diagnostic pop +} // namespace jiffy + +template +class JiffyQueue { + std::unique_ptr> queue_; + std::atomic_size_t size_; + size_t capacity_; + bool space_available_; + std::condition_variable enqueue_cv_; + std::mutex enqueue_mtx_; + +public: + explicit JiffyQueue(size_t capacity, size_t) + : queue_(std::make_unique>(capacity)), + size_(0), + capacity_(capacity), + space_available_(capacity > 0), + enqueue_cv_(), + enqueue_mtx_() {} + + JiffyQueue(JiffyQueue&& other) noexcept + : queue_(std::move(other.queue_)), + size_(other.size()), + capacity_(other.capacity_), + space_available_(other.space_available_), + enqueue_cv_(), + enqueue_mtx_() {} + + ~JiffyQueue() { + while (size() > 0) { + dequeue().release(); + } + }; + + bool enqueue(std::unique_ptr&& elem) { + std::unique_lock lock(enqueue_mtx_); + enqueue_cv_.wait(lock, [this] { return this->space_available_; }); + const size_t size = size_.load(); + space_available_ = size + 1 < capacity_; + assert(size < capacity_); + size_.fetch_add(1); + queue_->enqueue(*elem.release()); + return true; + } + + std::unique_ptr dequeue() { + T* t = new T; + size_.fetch_sub(1); + queue_->dequeue(*t); + this->space_available_ = true; + enqueue_cv_.notify_one(); + return std::unique_ptr(static_cast(t)); + } + + [[nodiscard]] __attribute_noinline__ size_t size() const { + return size_.load(); + } + + [[nodiscard]] __attribute_noinline__ size_t capacity() const { + return capacity_; + } +}; + +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp new file mode 100644 index 0000000..a787389 --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "queue.hpp" + +namespace pasta { + +namespace mpscq { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#pragma GCC diagnostic pop +} // namespace mpscq + +template +class Mpscq { + mpscq::mpscq* queue_; + +public: + Mpscq(size_t capacity, size_t) + : queue_(mpscq::mpscq_create(NULL, capacity)) {} + + ~Mpscq() { + mpscq::mpscq_destroy(queue_); + } + + bool enqueue(std::unique_ptr&& elem) { + return mpscq::mpscq_enqueue(queue_, elem.release()); + } + + std::unique_ptr dequeue() { + return std::unique_ptr(static_cast(mpscq::mpscq_dequeue(queue_))); + } + + size_t size() const { + return mpscq::mpscq_count(queue_); + } + + size_t capacity() const { + return mpscq::mpscq_capacity(queue_); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/mpsc_queue/queue.hpp b/include/pasta/block_tree/utils/mpsc_queue/queue.hpp new file mode 100644 index 0000000..8af6650 --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/queue.hpp @@ -0,0 +1,32 @@ +#pragma once + +#include +#include +#include + +namespace pasta { + +namespace internal { +using Capacity = size_t; +using ThreadCount = size_t; +} // namespace internal + +template +// Should have a constructor that allows the construction with a given capacity +// and thread count, should the queue need it +concept MpscQueue = + std::constructible_from && + requires(Queue q, std::unique_ptr&& e) { + // enqueue should enqueue a value and return true, if the element was + // enqueued (i.e. there was space) + { q.enqueue(std::move(e)) } -> std::convertible_to; // aa + // dequeue dequeue the oldest value and return it + { q.dequeue() } -> std::convertible_to>; + // size should return the current number of elements + { q.size() } -> std::convertible_to; + // capacity should return the maximum number of elements + // this queue can hold + { q.capacity() } -> std::convertible_to; + }; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp new file mode 100644 index 0000000..36a21e5 --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp @@ -0,0 +1,73 @@ +#pragma once + +#include "queue.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// @brief A simple queue that uses a std::list as the underlying data. Not even +/// lock-free. Probably bad, but should work. +/// @tparam T The type of the elements in the queue. +template +class StupidQueue { + std::list queue_; + size_t capacity_; + std::condition_variable enqueue_cv_; + std::mutex enqueue_mtx_; + +public: + explicit StupidQueue(size_t capacity, size_t) + : queue_(), + capacity_(capacity), + enqueue_cv_(), + enqueue_mtx_() {} + + StupidQueue(StupidQueue&& other) noexcept + : queue_(std::move(other.queue_)), + capacity_(other.capacity_), + enqueue_cv_(), + enqueue_mtx_() {} + + bool enqueue(std::unique_ptr&& elem) { + std::unique_lock lock(enqueue_mtx_); + const bool can_insert = + enqueue_cv_.wait_for(lock, std::chrono::milliseconds(10), [this] { + return size() < capacity_; + }); + if (!can_insert) { + return false; + } + assert(size() < capacity_); + queue_.push_back(*elem.release()); + return true; + } + + std::unique_ptr dequeue() { + assert(size() > 0); + T t = queue_.front(); + std::unique_ptr v = std::make_unique(std::move(t)); + { + std::lock_guard lock(enqueue_mtx_); + queue_.pop_front(); + } + enqueue_cv_.notify_one(); + return v; + } + + [[nodiscard]] __attribute_noinline__ size_t size() const { + return queue_.size(); + } + + [[nodiscard]] __attribute_noinline__ size_t capacity() const { + return capacity_; + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp new file mode 100644 index 0000000..1f60f69 --- /dev/null +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -0,0 +1,312 @@ +#pragma once + +#include "pasta/block_tree/utils/mpsc_queue/mpscq.hpp" +#include "pasta/block_tree/utils/mpsc_queue/queue.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// @brief Represents an update function which given a value, +/// updates a value in the map. +/// +/// @tparam Fn The type of the update function. +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// @tparam InputV The type that the update function accepts. This is not +/// required to be the same as the map's value type. +/// +template +concept UpdateFunction = requires(K k, V& v_lv, Fn::InputValue in_v_rv) { + typename Fn::InputValue; + // Updates a pre-existing value in the map. + // Arguments are the key, the value in the map, + // and the input value used to update the value in + // the map + { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; + // Initialize a value from an input value + // Arguments are the key, and the value used to + // initialize the value in the map. This returns the + // value to be inserted into the map + { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; +}; + +/// +/// @brief An update function which on update just overwrites the value. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Overwrite { + using InputValue = V; + + inline static void update(K&, V& value, V&& input_value) { + value = input_value; + } + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// +/// @brief An update function which upon update does nothing besides +/// inserting the value if it doesn't exist. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Keep { + using InputValue = V; + inline static void update(K&, V&, V&&) {} + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// @brief A hash map that must be used by multiple threads, each thread having +/// only having write access to a certain segment of the input space. +/// @tparam K The type of the keys in the hash map. +/// @tparam V The type of the values in the hash map. +/// @tparam SeqHashMapType The type of the hash map used internally. +/// This should be compatible with std::unordered_map. +/// @tparam QueueType The type of queue used for communication between threads. +/// @tparam UpdateFn The update function deciding how to insert or update values +/// in the map. +template typename SeqHashMapType = + std::unordered_map, + template typename QueueType = JiffyQueue, + UpdateFunction UpdateFn = Overwrite> + requires MpscQueue>, + std::pair> && + std::movable +class ShardedMap { + /// The sequential backing hash map type + using SeqHashMap = SeqHashMapType; + /// The sequential hash map's hasher + using Hasher = SeqHashMap::hasher; + /// The type used for updates + using InputValue = UpdateFn::InputValue; + /// The task queue used for communication between threads + using Queue = QueueType>; + + /// @brief A value between 0 and 1, determining to which extent + /// each thread's queue should be filled, before the thread is signaled to + /// handle its queued operations. + /// + /// If this value is 0.25, then the thread's value in threshold_met_ + /// is set to true, signaling that the thread should handle its requests in + /// task_queue_ + const double fill_threshold_; + /// @brief The number of threads operating on this map. + const size_t thread_count_; + /// @brief The capacity of each task_queue_. + const size_t queue_capacity_; + /// @brief Contains a hash map for each thread + std::vector map_; + /// @brief Contains a boolean for each thread, that is true, + /// iff the fill_threshold is met. + std::vector threshold_met_; + /// @brief Contains a task queue for each thread, holding insert + /// operations for each thread. + std::vector task_queue_; + + std::vector queue_empty_space_; + std::vector queue_cvs_; + + [[nodiscard]] bool threshold_exceeded(const size_t thread_id) const { + return static_cast(task_queue_[thread_id].size()) / + static_cast(task_queue_[thread_id].capacity()) >= + fill_threshold_; + } + + /// @brief Inserts or updates a new value in the map, depending on whether + /// @param k The key to insert or update a value for. + /// @param in_value The value with which to insert or update. + /// @param thread_id + inline void + insert_or_update_direct(K& k, InputValue&& in_value, const size_t thread_id) { + assert(thread_id == static_cast(omp_get_thread_num())); + auto res = map_[thread_id].find(k); + if (res == map_[thread_id].end()) { + // If the value does not exist, insert it + V initial = UpdateFn::init(k, std::move(in_value)); + K key = k; + map_[thread_id].emplace(key, std::move(initial)); + } else { + // Otherwise, update it. + V& val = res->second; + UpdateFn::update(k, val, std::move(in_value)); + } + } + +public: + // + /// @brief Creates a new sharded map. + /// + /// @param fill_threshold The fill percentage (between 0 and 1) above which + /// a thread is signaled to handle its own tasks. + /// @param thread_count The exact number of threads working on this map. + /// @param queue_capacity The maximum amount of tasks allowed in each queue. + /// + ShardedMap(double fill_threshold, size_t thread_count, size_t queue_capacity) + : fill_threshold_(fill_threshold), + thread_count_(thread_count), + queue_capacity_(queue_capacity), + map_(), + threshold_met_(), + task_queue_() { + assert(0 <= fill_threshold && fill_threshold <= 1); + threshold_met_.resize(thread_count, false); + map_.reserve(thread_count); + task_queue_.reserve(thread_count); + for (size_t i = 0; i < thread_count; i++) { + map_.emplace_back(); + task_queue_.emplace_back(queue_capacity, thread_count); + } + } + + /// @brief Waits for another thread to handle its queue, + /// this thread handling its own queue in the meantime. + /// @param current_thread_id The current thread's id. + /// @param target_thread_id The id of the thread to wait for. + void busy_wait(size_t current_thread_id, size_t target_thread_id) { + while (task_queue_[target_thread_id].size() == + task_queue_[target_thread_id].capacity()) { + handle_queue(current_thread_id); + std::this_thread::yield(); + } + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes around + /// to handle its queue using the handle_queue method. + /// + /// @param pair The key-value pair to insert or update. + void insert(std::pair&& pair) { + const size_t current_thread_id = omp_get_thread_num(); + const size_t hash = Hasher{}(pair.first); + const size_t target_thread_id = hash % thread_count_; + + // Otherwise enqueue the new value in the target thread + Queue& q = task_queue_[target_thread_id]; + while (q.size() == q.capacity()) { + handle_queue(current_thread_id); + // busy_wait(current_thread_id, target_thread_id); + } + while (!q.enqueue( + std::make_unique>(std::move(pair)))) { + handle_queue(current_thread_id); + }; + + // If the fill threshold is exceeded, mark it as such + threshold_met_[target_thread_id] = threshold_exceeded(target_thread_id); + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param key The key of the value to insert. + /// @param value The value to associate with the key. + inline void insert(K& key, InputValue value) { + insert(std::pair(key, value)); + } + + /// @brief Determines whether this thread should handle its queue. + /// + /// This translates to whether this thread's queue's fill level exceeds the + /// fill threshold. + /// @param current_thread_id The current thread's id. + /// @return `true` iff the fill threshold is exceeded. + bool should_handle_queue(const size_t current_thread_id) { + return threshold_met_[current_thread_id]; + } + + /// @brief Handles this thread's queue, inserting or updating all values in + /// its queue. + /// @param current_thread_id The current thread's id. + void handle_queue(const size_t current_thread_id) { + assert(current_thread_id == static_cast(omp_get_thread_num())); + if (task_queue_[current_thread_id].size() == 0) { + return; + } + Queue& q = task_queue_[current_thread_id]; + while (q.size() > 0) { + std::unique_ptr> pair = q.dequeue(); + assert(pair != nullptr); + insert_or_update_direct(pair->first, + std::move(pair->second), + current_thread_id); + // std::cout << "Handling queue: " << current_thread_id << std::endl; + } + threshold_met_[current_thread_id] = false; + } + + /// @brief Returns the number of key-value pairs in the map. + /// + /// Note, that this method calculates the size for each map separately and + /// is therefore not O(1). + /// @return The number of key-value pairs in the map. + [[nodiscard]] size_t size() const { + size_t size = 0; + for (const SeqHashMap& map : map_) { + size += map.size(); + } + return size; + } + + /// @brief Runs a method for each value in the map. + /// + /// The given function must take const references to a key and a value + /// respectively. + /// @param f The function or lambda to run for each value. + void for_each(std::invocable auto f) { + for (const SeqHashMap& map : map_) { + for (const auto& [k, v] : map) { + f(k, v); + } + } + } + + SeqHashMap::iterator end() { + return map_.back().end(); + } + + SeqHashMap::iterator find(const K& key) { + const size_t hash = Hasher{}(key); + const size_t target_thread_id = hash % thread_count_; + typename SeqHashMap::iterator it = map_[target_thread_id].find(key); + if (it == map_[target_thread_id].end()) { + return end(); + } + return it; + } + + void print_map_loads() { + for (size_t i = 0; i < map_.size(); ++i) { + std::cout << "Map " << i << " load: " << map_[i].size() << std::endl; + } + } + +}; // namespace pasta + +} // namespace pasta \ No newline at end of file From 741d71e80f0def016ee5f46b0d643c3e60826a26 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 26 Sep 2023 14:11:46 +0200 Subject: [PATCH 17/92] add handle to sharded map --- .gitmodules | 4 + CMakeLists.txt | 6 +- examples/build_bt.cpp | 74 ++++--- extlib/Jiffy-1 | 1 + .../block_tree_fp_par_sharded.hpp | 26 +-- .../block_tree/utils/mpsc_queue/jiffy.hpp | 41 +++- include/pasta/block_tree/utils/semaphore.hpp | 8 + .../pasta/block_tree/utils/sharded_map.hpp | 191 +++++++++--------- 8 files changed, 199 insertions(+), 152 deletions(-) create mode 160000 extlib/Jiffy-1 create mode 100644 include/pasta/block_tree/utils/semaphore.hpp diff --git a/.gitmodules b/.gitmodules index 7421660..4e0ff89 100644 --- a/.gitmodules +++ b/.gitmodules @@ -25,3 +25,7 @@ [submodule "fastwfc"] path = extlib/fastwfc/fast-wait-free-queue url = https://github.com/chaoran/fast-wait-free-queue +[submodule "extlib/Jiffy-1"] + path = extlib/Jiffy-1 + url = https://github.com/quininer/Jiffy-1 + branch = fix-atomic diff --git a/CMakeLists.txt b/CMakeLists.txt index f50490d..b43ca18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -77,6 +77,9 @@ target_include_directories(waitfree-mpsc-queue PUBLIC add_library(jiffy INTERFACE) target_include_directories(jiffy INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy) +add_library(jiffy1 INTERFACE) +target_include_directories(jiffy1 INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy-1) add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE @@ -89,7 +92,8 @@ target_link_libraries(pasta_block_tree INTERFACE tlx robin_hood waitfree-mpsc-queue - jiffy) + #jiffy + jiffy1) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index cf13d70..0dd6cfd 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,11 +22,14 @@ #include #include #include -// #include -// #include -#include -// #include -// #include +// #include +// #include +// #include +// #include +// #include +#include +// #include +// #include #include #include @@ -35,6 +38,7 @@ using TimePoint = Clock::time_point; using Duration = Clock::duration; int main(int argc, char** argv) { + using namespace pasta; if (argc < 2) { std::cerr << "Please input file" << std::endl; exit(1); @@ -66,6 +70,13 @@ int main(int argc, char** argv) { size_t leaf_length = atoi(argv[4]); + JiffyQueue, std::pair>> q(100, 8); + + auto p = + std::make_unique, std::pair>>( + std::make_pair(MersenneHash(), std::make_pair(0, 0))); + q.enqueue(std::move(p)); + std::stringstream ss; ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" << leaf_length << "_new.bt"; @@ -89,35 +100,40 @@ int main(int argc, char** argv) { TimePoint now = Clock::now(); /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ - /* - auto bt = std::make_unique>(text, - arity, - leaf_length, - root_arity, - 256, - true, - true); - */ + auto bt = std::make_unique>(text, + arity, + root_arity, + leaf_length); + */ + /* + std::unique_ptr> bt( + pasta::make_block_tree_lpf_parallel(text, + arity, + leaf_length, + true, + 8)); + */ + /* + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length); + */ + auto bt = std::make_unique>( + text, + arity, + root_arity, + leaf_length, + 8); + /* auto bt = std::make_unique>(text, arity, root_arity, leaf_length, 8); - - /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length); */ auto elapsed = std::chrono::duration_cast(Clock::now() - now) @@ -126,8 +142,8 @@ int main(int argc, char** argv) { std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" << "Time: " << elapsed << "ms" << std::endl; -// std::ofstream ot(out_path); -// bt->serialize(ot); + // std::ofstream ot(out_path); + // bt->serialize(ot); #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); diff --git a/extlib/Jiffy-1 b/extlib/Jiffy-1 new file mode 160000 index 0000000..4121cd2 --- /dev/null +++ b/extlib/Jiffy-1 @@ -0,0 +1 @@ +Subproject commit 4121cd27ecf62889e1cf22e62da3a72cbd0f8e7d diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp index 17b17bc..658906d 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -435,6 +435,7 @@ class BlockTreeFPParSharded : public BlockTree { shared(level, map, text, now, is_padded, std::cout, num_threads_finished) { const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); const size_t num_threads = omp_get_num_threads(); const size_t block_size = level.block_size; const size_t pair_size = 2 * block_size; @@ -462,9 +463,9 @@ class BlockTreeFPParSharded : public BlockTree { RabinKarpHash hash = rk.current_hash(); // Try to find the hash in the map, insert a new entry if it doesn't // exist, and add the current block to the entry - map.insert(hash, i); - if (map.should_handle_queue(thread_id)) { - map.handle_queue(thread_id); + shard.insert(hash, i); + if (shard.should_handle_queue()) { + shard.handle_queue(); } } num_threads_finished.fetch_add(1); @@ -472,7 +473,7 @@ class BlockTreeFPParSharded : public BlockTree { // we need all threads to wait for the others to finish the loop and // handle thread events that might come in from the other threads do { - map.handle_queue(thread_id); + shard.handle_queue(); } while (num_threads_finished.load() < num_threads); #pragma omp barrier #pragma omp single @@ -603,16 +604,11 @@ class BlockTreeFPParSharded : public BlockTree { // The number of threads finished with hashing blocks std::atomic_size_t num_threads_finished = 0; #pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level_data, \ - text, \ - links, \ - now, \ - is_padded, \ - num_threads_finished, \ - std::cout) + shared(level_data, text, links, now, is_padded, num_threads_finished) { const size_t num_threads = omp_get_num_threads(); const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); const size_t block_size = level_data.block_size; const std::vector& block_starts = *level_data.block_starts; // Number of total iterations the for loop should do @@ -633,10 +629,10 @@ class BlockTreeFPParSharded : public BlockTree { for (size_t i = start; i < end; ++i) { const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); RabinKarpHash hash = rk.current_hash(); - links.insert(hash, {i, 0}); + shard.insert(hash, {i, 0}); // The thread checks whether it should handle the inserts in its queue - if (links.should_handle_queue(thread_id)) { - links.handle_queue(thread_id); + if (shard.should_handle_queue()) { + shard.handle_queue(); } } num_threads_finished.fetch_add(1); @@ -644,7 +640,7 @@ class BlockTreeFPParSharded : public BlockTree { // we need all threads to wait for the others to finish the loop and // handle thread events that might come in from the other threads do { - links.handle_queue(thread_id); + shard.handle_queue(); } while (num_threads_finished.load() < num_threads); #pragma omp single { diff --git a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp index 7b83304..5ce86ae 100644 --- a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp +++ b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp @@ -31,6 +31,8 @@ class JiffyQueue { bool space_available_; std::condition_variable enqueue_cv_; std::mutex enqueue_mtx_; + std::condition_variable dequeue_cv_; + std::mutex dequeue_mtx_; public: explicit JiffyQueue(size_t capacity, size_t) @@ -39,7 +41,9 @@ class JiffyQueue { capacity_(capacity), space_available_(capacity > 0), enqueue_cv_(), - enqueue_mtx_() {} + enqueue_mtx_(), + dequeue_cv_(), + dequeue_mtx_() {} JiffyQueue(JiffyQueue&& other) noexcept : queue_(std::move(other.queue_)), @@ -47,7 +51,9 @@ class JiffyQueue { capacity_(other.capacity_), space_available_(other.space_available_), enqueue_cv_(), - enqueue_mtx_() {} + enqueue_mtx_(), + dequeue_cv_(), + dequeue_mtx_() {} ~JiffyQueue() { while (size() > 0) { @@ -57,20 +63,37 @@ class JiffyQueue { bool enqueue(std::unique_ptr&& elem) { std::unique_lock lock(enqueue_mtx_); - enqueue_cv_.wait(lock, [this] { return this->space_available_; }); - const size_t size = size_.load(); - space_available_ = size + 1 < capacity_; - assert(size < capacity_); + const bool can_insert = + enqueue_cv_.wait_for(lock, std::chrono::milliseconds(1), [this] { + return size() < capacity_; + }); + if (!can_insert) { + return false; + } + assert(size() < capacity_); size_.fetch_add(1); queue_->enqueue(*elem.release()); + dequeue_cv_.notify_one(); return true; } std::unique_ptr dequeue() { + std::unique_lock lock(enqueue_mtx_); + const bool can_dequeue = + dequeue_cv_.wait_for(lock, std::chrono::milliseconds(10), [this] { + return size() > 0; + }); + if (!can_dequeue) { + return std::unique_ptr(nullptr); + } + assert(size() > 0); + const auto size = size_.fetch_sub(1) - 1; T* t = new T; - size_.fetch_sub(1); - queue_->dequeue(*t); - this->space_available_ = true; + const bool queue_was_not_empty = queue_->dequeue(*t); + if (!queue_was_not_empty) { + std::cout << "Size is " << size << std::endl; + assert(queue_was_not_empty); + } enqueue_cv_.notify_one(); return std::unique_ptr(static_cast(t)); } diff --git a/include/pasta/block_tree/utils/semaphore.hpp b/include/pasta/block_tree/utils/semaphore.hpp new file mode 100644 index 0000000..ead4559 --- /dev/null +++ b/include/pasta/block_tree/utils/semaphore.hpp @@ -0,0 +1,8 @@ +// +// Created by skadic on 26.09.23. +// + +#ifndef PASTA_BLOCK_TREE_SEMAPHORE_HPP +#define PASTA_BLOCK_TREE_SEMAPHORE_HPP + +#endif //PASTA_BLOCK_TREE_SEMAPHORE_HPP diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp index 1f60f69..51e9db2 100644 --- a/include/pasta/block_tree/utils/sharded_map.hpp +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -118,40 +118,17 @@ class ShardedMap { std::vector map_; /// @brief Contains a boolean for each thread, that is true, /// iff the fill_threshold is met. - std::vector threshold_met_; + std::vector threshold_met_; /// @brief Contains a task queue for each thread, holding insert /// operations for each thread. std::vector task_queue_; - std::vector queue_empty_space_; - std::vector queue_cvs_; - [[nodiscard]] bool threshold_exceeded(const size_t thread_id) const { return static_cast(task_queue_[thread_id].size()) / static_cast(task_queue_[thread_id].capacity()) >= fill_threshold_; } - /// @brief Inserts or updates a new value in the map, depending on whether - /// @param k The key to insert or update a value for. - /// @param in_value The value with which to insert or update. - /// @param thread_id - inline void - insert_or_update_direct(K& k, InputValue&& in_value, const size_t thread_id) { - assert(thread_id == static_cast(omp_get_thread_num())); - auto res = map_[thread_id].find(k); - if (res == map_[thread_id].end()) { - // If the value does not exist, insert it - V initial = UpdateFn::init(k, std::move(in_value)); - K key = k; - map_[thread_id].emplace(key, std::move(initial)); - } else { - // Otherwise, update it. - V& val = res->second; - UpdateFn::update(k, val, std::move(in_value)); - } - } - public: // /// @brief Creates a new sharded map. @@ -178,87 +155,104 @@ class ShardedMap { } } - /// @brief Waits for another thread to handle its queue, - /// this thread handling its own queue in the meantime. - /// @param current_thread_id The current thread's id. - /// @param target_thread_id The id of the thread to wait for. - void busy_wait(size_t current_thread_id, size_t target_thread_id) { - while (task_queue_[target_thread_id].size() == - task_queue_[target_thread_id].capacity()) { - handle_queue(current_thread_id); - std::this_thread::yield(); - } - } + class Shard { + ShardedMap& sharded_map_; + const size_t thread_id_; + SeqHashMap& map_; + char& threshold_met_; + Queue& task_queue_; - /// @brief Inserts or updates a new value in the map. - /// - /// If the value is inserted into the current thread's map, - /// it is inserted immediately. If not, then it is added to that thread's - /// queue. It will only be inserted into the map, once the thread comes around - /// to handle its queue using the handle_queue method. - /// - /// @param pair The key-value pair to insert or update. - void insert(std::pair&& pair) { - const size_t current_thread_id = omp_get_thread_num(); - const size_t hash = Hasher{}(pair.first); - const size_t target_thread_id = hash % thread_count_; + public: + Shard(ShardedMap& sharded_map, size_t thread_id) + : sharded_map_(sharded_map), + thread_id_(thread_id), + map_(sharded_map_.map_[thread_id]), + threshold_met_(sharded_map_.threshold_met_[thread_id]), + task_queue_(sharded_map_.task_queue_[thread_id]) {} - // Otherwise enqueue the new value in the target thread - Queue& q = task_queue_[target_thread_id]; - while (q.size() == q.capacity()) { - handle_queue(current_thread_id); - // busy_wait(current_thread_id, target_thread_id); + /// @brief Inserts or updates a new value in the map, depending on whether + /// @param k The key to insert or update a value for. + /// @param in_value The value with which to insert or update. + inline void insert_or_update_direct(K& k, InputValue&& in_value) { + auto res = map_.find(k); + if (res == map_.end()) { + // If the value does not exist, insert it + V initial = UpdateFn::init(k, std::move(in_value)); + K key = k; + map_.emplace(key, std::move(initial)); + } else { + // Otherwise, update it. + V& val = res->second; + UpdateFn::update(k, val, std::move(in_value)); + } } - while (!q.enqueue( - std::make_unique>(std::move(pair)))) { - handle_queue(current_thread_id); - }; - // If the fill threshold is exceeded, mark it as such - threshold_met_[target_thread_id] = threshold_exceeded(target_thread_id); - } + /// @brief Handles this thread's queue, inserting or updating all values in + /// its queue. + void handle_queue() { + if (task_queue_.size() == 0) { + return; + } + while (task_queue_.size() > 0) { + std::unique_ptr> pair = task_queue_.dequeue(); + assert(pair != nullptr); + insert_or_update_direct(pair->first, std::move(pair->second)); + } + threshold_met_ = false; + } - /// @brief Inserts or updates a new value in the map. - /// - /// If the value is inserted into the current thread's map, - /// it is inserted immediately. If not, then it is added to that thread's - /// queue. It will only be inserted into the map, once the thread comes - /// around to handle its queue using the handle_queue method. - /// - /// @param key The key of the value to insert. - /// @param value The value to associate with the key. - inline void insert(K& key, InputValue value) { - insert(std::pair(key, value)); - } + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param pair The key-value pair to insert or update. + void insert(std::pair&& pair) { + const size_t hash = Hasher{}(pair.first); + const size_t target_thread_id = hash % sharded_map_.thread_count_; - /// @brief Determines whether this thread should handle its queue. - /// - /// This translates to whether this thread's queue's fill level exceeds the - /// fill threshold. - /// @param current_thread_id The current thread's id. - /// @return `true` iff the fill threshold is exceeded. - bool should_handle_queue(const size_t current_thread_id) { - return threshold_met_[current_thread_id]; - } + // Otherwise enqueue the new value in the target thread + Queue& q = sharded_map_.task_queue_[target_thread_id]; + while (q.size() == q.capacity()) { + handle_queue(); + } + while (!q.enqueue( + std::make_unique>(std::move(pair)))) { + handle_queue(); + }; + + // If the fill threshold is exceeded, mark it as such + sharded_map_.threshold_met_[target_thread_id] = + sharded_map_.threshold_exceeded(target_thread_id); + } - /// @brief Handles this thread's queue, inserting or updating all values in - /// its queue. - /// @param current_thread_id The current thread's id. - void handle_queue(const size_t current_thread_id) { - assert(current_thread_id == static_cast(omp_get_thread_num())); - if (task_queue_[current_thread_id].size() == 0) { - return; + /// @brief Determines whether this thread should handle its queue. + /// + /// This translates to whether this thread's queue's fill level exceeds the + /// fill threshold. + /// @return `true` iff the fill threshold is exceeded. + [[nodiscard]] bool should_handle_queue() const { + return threshold_met_; } - Queue& q = task_queue_[current_thread_id]; - while (q.size() > 0) { - std::unique_ptr> pair = q.dequeue(); - assert(pair != nullptr); - insert_or_update_direct(pair->first, - std::move(pair->second), - current_thread_id); - // std::cout << "Handling queue: " << current_thread_id << std::endl; + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param key The key of the value to insert. + /// @param value The value to associate with the key. + inline void insert(K& key, InputValue value) { + insert(std::pair(key, value)); } - threshold_met_[current_thread_id] = false; + }; + + Shard get_shard(const size_t thread_id) { + return Shard(*this, thread_id); } /// @brief Returns the number of key-value pairs in the map. @@ -294,8 +288,9 @@ class ShardedMap { SeqHashMap::iterator find(const K& key) { const size_t hash = Hasher{}(key); const size_t target_thread_id = hash % thread_count_; - typename SeqHashMap::iterator it = map_[target_thread_id].find(key); - if (it == map_[target_thread_id].end()) { + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(key); + if (it == map.end()) { return end(); } return it; From 95f86ea94ac0cc235c183c0e5f2da0a68d9f6cd7 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 26 Sep 2023 15:16:17 +0200 Subject: [PATCH 18/92] use http for submodules --- .gitmodules | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.gitmodules b/.gitmodules index 4e0ff89..cb8f73f 100644 --- a/.gitmodules +++ b/.gitmodules @@ -12,10 +12,10 @@ url = https://github.com/martinus/robin-hood-hashing [submodule "extlib/growt"] path = extlib/growt - url = git@github.com:TooBiased/growt.git + url = https://github.com/TooBiased/growt [submodule "extlib/parallel-hashmap"] path = extlib/parallel-hashmap - url = git@github.com:greg7mdp/parallel-hashmap.git + url = https://github.com/greg7mdp/parallel-hashmap [submodule "extlib/waitfree-mpsc-queue"] path = extlib/waitfree-mpsc-queue url = https://github.com/dbittman/waitfree-mpsc-queue From 9d2ceda8869731fa523e8ecd2689fc961373818e Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 27 Sep 2023 15:15:27 +0200 Subject: [PATCH 19/92] use local sdsl --- .gitignore | 1 + .gitmodules | 3 +++ CMakeLists.txt | 3 ++- CMakePresets.json | 19 +++++++++++++++++++ extlib/sdsl-lite | 1 + 5 files changed, 26 insertions(+), 1 deletion(-) create mode 160000 extlib/sdsl-lite diff --git a/.gitignore b/.gitignore index 92e0fa0..73d76ee 100644 --- a/.gitignore +++ b/.gitignore @@ -4,3 +4,4 @@ compile_commands.json .idea/* perf.data* +.pdf diff --git a/.gitmodules b/.gitmodules index cb8f73f..576d401 100644 --- a/.gitmodules +++ b/.gitmodules @@ -29,3 +29,6 @@ path = extlib/Jiffy-1 url = https://github.com/quininer/Jiffy-1 branch = fix-atomic +[submodule "extlib/sdsl-lite"] + path = extlib/sdsl-lite + url = https://github.com/Skadic/sdsl-lite diff --git a/CMakeLists.txt b/CMakeLists.txt index b43ca18..857a514 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,7 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite) add_library(waitfree-mpsc-queue ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) @@ -88,10 +89,10 @@ target_include_directories(pasta_block_tree INTERFACE target_link_libraries(pasta_block_tree INTERFACE libsais pasta_bit_vector - sdsl tlx robin_hood waitfree-mpsc-queue + sdsl #jiffy jiffy1) target_include_directories(pasta_block_tree INTERFACE diff --git a/CMakePresets.json b/CMakePresets.json index 10d2973..c7b43da 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -61,6 +61,20 @@ "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address -static-libasan" } + }, + { + "name": "lax", + "displayName": "Lax Ninja Multi-Config", + "description": "Use Ninja Multi-Config generator with lax warnings", + "generator": "Ninja Multi-Config", + "binaryDir": "${sourceDir}/build_lax", + "cacheVariables": { + "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", + "CMAKE_CXX_FLAGS": "-fopenmp -w -march=native -fdiagnostics-color=always", + "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address -static-libasan" + } } ], "buildPresets": [ @@ -90,6 +104,11 @@ "name": "debug-multi", "configurePreset": "ninja-multi", "configuration": "Debug" + }, + { + "name": "release-lax", + "configurePreset": "lax", + "configuration": "Release" } ], "testPresets": [ diff --git a/extlib/sdsl-lite b/extlib/sdsl-lite new file mode 160000 index 0000000..513f9eb --- /dev/null +++ b/extlib/sdsl-lite @@ -0,0 +1 @@ +Subproject commit 513f9ebe87ee9d3cfe8dbed5133d639767e1722c From 45d4dc93695d224192dbe35f731f0baaf2d9f814 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 27 Sep 2023 15:42:38 +0200 Subject: [PATCH 20/92] add lax relwithdeb preset --- CMakePresets.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/CMakePresets.json b/CMakePresets.json index c7b43da..533cc69 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -109,6 +109,11 @@ "name": "release-lax", "configurePreset": "lax", "configuration": "Release" + }, + { + "name": "relwithdeb-lax", + "configurePreset": "lax", + "configuration": "RelWithDebInfo" } ], "testPresets": [ From ef67de6dd45f1e2cbf4fd0fa5cfd480900286345 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 28 Sep 2023 20:46:35 +0200 Subject: [PATCH 21/92] move conceps to their own file --- examples/build_bt.cpp | 161 ------------------ .../block_tree/construction/block_tree_fp.hpp | 21 ++- .../construction/block_tree_fp_par_phmap.hpp | 91 ++++++---- .../block_tree_fp_par_sharded.hpp | 15 +- .../{mpsc_queue/queue.hpp => concepts.hpp} | 25 ++- .../block_tree/utils/mpsc_queue/jiffy.hpp | 2 - .../block_tree/utils/mpsc_queue/mpscq.hpp | 2 +- .../utils/mpsc_queue/stupid_queue.hpp | 2 - include/pasta/block_tree/utils/semaphore.hpp | 8 - .../pasta/block_tree/utils/sharded_map.hpp | 30 +--- 10 files changed, 105 insertions(+), 252 deletions(-) delete mode 100644 examples/build_bt.cpp rename include/pasta/block_tree/utils/{mpsc_queue/queue.hpp => concepts.hpp} (51%) delete mode 100644 include/pasta/block_tree/utils/semaphore.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp deleted file mode 100644 index 0dd6cfd..0000000 --- a/examples/build_bt.cpp +++ /dev/null @@ -1,161 +0,0 @@ -/******************************************************************************* - * This file is part of pasta::block_tree - * - * Copyright (C) 2023 Etienne Palanga - * - * pasta::block_tree is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * pasta::block_tree is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with pasta::block_tree. If not, see . - * - ******************************************************************************/ - -#include -#include -#include -#include -// #include -// #include -// #include -// #include -// #include -#include -// #include -// #include -#include -#include - -using Clock = std::chrono::high_resolution_clock; -using TimePoint = Clock::time_point; -using Duration = Clock::duration; - -int main(int argc, char** argv) { - using namespace pasta; - if (argc < 2) { - std::cerr << "Please input file" << std::endl; - exit(1); - } - - if (!std::filesystem::exists(argv[1])) { - std::cerr << "File " << argv[1] << " does not exist" << std::endl; - exit(1); - } - - if (argc < 3) { - std::cerr << "Please input tree arity (tau)" << std::endl; - exit(1); - } - - size_t arity = atoi(argv[2]); - - if (argc < 4) { - std::cerr << "Please input root arity (s)" << std::endl; - exit(1); - } - - size_t root_arity = atoi(argv[3]); - - if (argc < 5) { - std::cerr << "Please input max leaf length" << std::endl; - exit(1); - } - - size_t leaf_length = atoi(argv[4]); - - JiffyQueue, std::pair>> q(100, 8); - - auto p = - std::make_unique, std::pair>>( - std::make_pair(MersenneHash(), std::make_pair(0, 0))); - q.enqueue(std::move(p)); - - std::stringstream ss; - ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" - << leaf_length << "_new.bt"; - std::string out_path = ss.str(); - - std::cout << "building block tree with parameters:" - << "\narity: " << arity << "\nroot arity: " << root_arity - << "\nmax leaf length: " << leaf_length << "\nsaving to " - << out_path << std::endl; - - std::vector text; - { - std::string input; - std::ifstream t(argv[1]); - std::stringstream buffer; - buffer << t.rdbuf(); - input = buffer.str(); - text = std::vector(input.begin(), input.end()); - } - - TimePoint now = Clock::now(); - - /* - auto bt = std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ - - /* - std::unique_ptr> bt( - pasta::make_block_tree_lpf_parallel(text, - arity, - leaf_length, - true, - 8)); - */ - /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ - auto bt = std::make_unique>( - text, - arity, - root_arity, - leaf_length, - 8); - /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length, - 8); -*/ - auto elapsed = - std::chrono::duration_cast(Clock::now() - now) - .count(); - - std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" - << "Time: " << elapsed << "ms" << std::endl; - - // std::ofstream ot(out_path); - // bt->serialize(ot); -#pragma omp parallel for - for (size_t i = 0; i < text.size(); ++i) { - const auto c = bt->access(i); - if (c != text[i]) { - std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] - << "\nActual: " << c << std::endl; - exit(1); - } - } - // ot.close(); - - return 0; -} - -/******************************************************************************/ diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 7560ac4..9b8a4ae 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -24,10 +24,15 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include + __extension__ typedef unsigned __int128 uint128_t; namespace pasta { + template> + using HashMap = robin_hood::unordered_map; + template class BlockTreeFP : public BlockTree { public: @@ -315,12 +320,12 @@ class BlockTreeFP : public BlockTree { : 0; // map block pair hashes to the text index of their occurrences // collecting duplicates in a vector TODO - std::unordered_map, std::vector> pairs( + HashMap, std::vector> pairs( 0); // map block hashes to their *block* index, // collecting duplicates in a vector TODO - std::unordered_map, std::vector> blocks = - std::unordered_map, std::vector>(); + HashMap, std::vector> blocks = + HashMap, std::vector>(); // iterate through all blocks on the current level, skipping over the last // block if it is padded for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { @@ -599,7 +604,7 @@ class BlockTreeFP : public BlockTree { auto &ptr = *p; auto &off = *o; // Maps block index => number of pruned blocks before this block - std::unordered_map blocks_skipped; + HashMap blocks_skipped; auto &lvl_pass1 = *bv_marked[i]; // Number of non-pruned blocks so far size_type c = 0; @@ -715,10 +720,10 @@ class BlockTreeFP : public BlockTree { block_size) != text.size() ? 1 : 0; - std::unordered_map, std::vector> pairs( + HashMap, std::vector> pairs( 0); - std::unordered_map, std::vector> blocks = - std::unordered_map, std::vector>(); + HashMap, std::vector> blocks = + HashMap, std::vector>(); for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { auto index = block_text_inx[i]; MersenneRabinKarp rk_block = @@ -918,7 +923,7 @@ class BlockTreeFP : public BlockTree { auto offset = std::vector(); size_type pointer_saved = 0; size_type pointer_skipped = 0; - std::unordered_map blocks_skipped; + HashMap blocks_skipped; size_type skip = 0; size_type replace = 0; for (uint64_t j = 0; j < bv_pass_1[pass1_i - 1]->size(); j++) { diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp index b833211..722e16b 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -59,12 +59,25 @@ class BlockTreeFPParPH : public BlockTree { using Rank = pasta::RankSelect; /// A concurrent hash map + /*template , + size_t num_submaps = 6, + typename mutex_type = phmap::NullMutex> + using HashMap = phmap::parallel_flat_hash_map< + key_type, + value_type, + hash_type, + phmap::priv::hash_default_eq, + phmap::priv::Allocator< + typename phmap::priv::Pair>, + num_submaps, + mutex_type>;*/ + template > - using HashMap = - phmap::parallel_node_hash_map; - // robin_hood::unordered_map; + using HashMap = robin_hood::unordered_node_map; /// A rabin karp hasher preconfigured for the current template parameters using RabinKarp = MersenneRabinKarp; @@ -72,8 +85,12 @@ class BlockTreeFPParPH : public BlockTree { using RabinKarpHash = MersenneHash; /// A hash map with rabin karp hashes as keys - template - using RabinKarpMap = HashMap; + template + using RabinKarpMap = HashMap>; public: size_t bp_hash_pairs_ns = 0; @@ -193,15 +210,6 @@ class BlockTreeFPParPH : public BlockTree { } TimePoint now = Clock::now(); - prune(levels); - size_t prune_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); - make_tree(text, levels, padding); - size_t make_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); std::cout << "pairs: " << (pairs_ns / 1'000'000) << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) @@ -211,9 +219,20 @@ class BlockTreeFPParPH : public BlockTree { << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) - << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) - << "ms,\nprune: " << (prune_ns / 1'000'000) - << "ms,\nmake: " << (make_ns / 1'000'000) << "ms" << std::endl; + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; + prune(levels); + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; + make_tree(text, levels, padding); + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; } /// @brief Returns the ceiling of x / y for x > 0; @@ -281,7 +300,7 @@ class BlockTreeFPParPH : public BlockTree { const size_t num_blocks = level.num_blocks; const size_t num_block_pairs = num_blocks - 1 - is_padded; const auto& block_starts = *level.block_starts; -#pragma omp for +#pragma omp single for (size_t i = 0; i < num_block_pairs; ++i) { // If the next block is not adjacent, we cannot hash the pair starting // at the current block @@ -414,25 +433,32 @@ class BlockTreeFPParPH : public BlockTree { offset(first_occ_offset_) {} }; + // The block index and offset of the first occurrence of this block's + // content std::atomic first_occ; /// @brief A list of block indices in which the content of the hashed block /// occurs std::list occurrences; + std::mutex list_mutex; BlockOccurrences(size_type first_occ_block_) : first_occ({first_occ_block_, 0}), - occurrences() {} + occurrences(), + list_mutex() {} BlockOccurrences(const BlockOccurrences& other) : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} + occurrences(other.occurrences), + list_mutex() {} BlockOccurrences(BlockOccurrences&& other) : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} + occurrences(std::move(other.occurrences)), + list_mutex() {} inline void add_block(size_type block_index) { + const std::lock_guard lock(list_mutex); occurrences.push_back(block_index); } @@ -493,17 +519,14 @@ class BlockTreeFPParPH : public BlockTree { level_data.block_size, K_PRIME); const RabinKarpHash hash = rk.current_hash(); -#pragma omp critical - { - auto ptr = links.find(hash); - if (ptr == links.end()) { - auto [insert_ptr, _] = links.emplace(hash, BlockOccurrences(i)); - ptr = insert_ptr; - } - - ptr->second.add_block(i); - ptr->second.update(i, 0); + auto ptr = links.find(hash); + if (ptr == links.end()) { + auto [insert_ptr, _] = links.emplace(hash, BlockOccurrences(i)); + ptr = insert_ptr; } + + ptr->second.add_block(i); + ptr->second.update(i, 0); } #pragma omp barrier @@ -555,8 +578,8 @@ class BlockTreeFPParPH : public BlockTree { for (auto it = links.cbegin(); it != links.cend(); ++it) { // The occurrences of all blocks with a given hash const BlockOccurrences& occs = it->second; + auto first_occ = occs.first_occ.load(); for (const size_type occ : occs.occurrences) { - auto first_occ = occs.first_occ.load(); if (occ == first_occ.block || (first_occ.offset > 0 && occ == first_occ.block + 1)) { continue; @@ -595,8 +618,6 @@ class BlockTreeFPParPH : public BlockTree { if (found == links.end()) { continue; } - // - // TODO This must be thread safe found->second.update(current_block_index, offset); continue; } @@ -1009,4 +1030,4 @@ class BlockTreeFPParPH : public BlockTree { } // namespace pasta -#undef BT_NUM_THREADS \ No newline at end of file +#undef BT_NUM_THREADS diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp index 658906d..97eabf8 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -58,7 +58,7 @@ namespace pasta { template typename queue_type = StupidQueue> -class BlockTreeFPParSharded : public BlockTree { +class BlockTreeFPParShardedSync : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -566,6 +566,7 @@ class BlockTreeFPParSharded : public BlockTree { // Find the hash of the current window among the hashed block pairs. auto found = map.find(current_hash); if (found == map.end()) { + // TODO count how often this actually happens continue; } PairOccurrences& occurrences = found->second; @@ -1054,11 +1055,11 @@ class BlockTreeFPParSharded : public BlockTree { } public: - BlockTreeFPParSharded(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads) { + BlockTreeFPParShardedSync(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { const auto old = omp_get_max_threads(); const auto old_dynamic = omp_get_dynamic(); omp_set_dynamic(0); @@ -1072,7 +1073,7 @@ class BlockTreeFPParSharded : public BlockTree { omp_set_num_threads(old); } - ~BlockTreeFPParSharded() { + ~BlockTreeFPParShardedSync() { for (auto& rank : this->block_tree_types_rs_) { delete rank; } diff --git a/include/pasta/block_tree/utils/mpsc_queue/queue.hpp b/include/pasta/block_tree/utils/concepts.hpp similarity index 51% rename from include/pasta/block_tree/utils/mpsc_queue/queue.hpp rename to include/pasta/block_tree/utils/concepts.hpp index 8af6650..8fbac04 100644 --- a/include/pasta/block_tree/utils/mpsc_queue/queue.hpp +++ b/include/pasta/block_tree/utils/concepts.hpp @@ -1,4 +1,3 @@ -#pragma once #include #include @@ -6,6 +5,30 @@ namespace pasta { +/// @brief Represents an update function which given a value, +/// updates a value in the map. +/// +/// @tparam Fn The type of the update function. +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// @tparam InputV The type that the update function accepts. This is not +/// required to be the same as the map's value type. +/// +template +concept UpdateFunction = requires(K k, V& v_lv, Fn::InputValue in_v_rv) { + typename Fn::InputValue; + // Updates a pre-existing value in the map. + // Arguments are the key, the value in the map, + // and the input value used to update the value in + // the map + { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; + // Initialize a value from an input value + // Arguments are the key, and the value used to + // initialize the value in the map. This returns the + // value to be inserted into the map + { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; +}; + namespace internal { using Capacity = size_t; using ThreadCount = size_t; diff --git a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp index 5ce86ae..15ce0ff 100644 --- a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp +++ b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp @@ -1,7 +1,5 @@ #pragma once -#include "queue.hpp" - #include #include #include diff --git a/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp index a787389..2068f75 100644 --- a/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp +++ b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp @@ -1,6 +1,6 @@ #pragma once -#include "queue.hpp" +#include "../concepts.hpp" namespace pasta { diff --git a/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp index 36a21e5..bd0ca5c 100644 --- a/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp +++ b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp @@ -1,7 +1,5 @@ #pragma once -#include "queue.hpp" - #include #include #include diff --git a/include/pasta/block_tree/utils/semaphore.hpp b/include/pasta/block_tree/utils/semaphore.hpp deleted file mode 100644 index ead4559..0000000 --- a/include/pasta/block_tree/utils/semaphore.hpp +++ /dev/null @@ -1,8 +0,0 @@ -// -// Created by skadic on 26.09.23. -// - -#ifndef PASTA_BLOCK_TREE_SEMAPHORE_HPP -#define PASTA_BLOCK_TREE_SEMAPHORE_HPP - -#endif //PASTA_BLOCK_TREE_SEMAPHORE_HPP diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp index 51e9db2..45f8237 100644 --- a/include/pasta/block_tree/utils/sharded_map.hpp +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -1,43 +1,19 @@ #pragma once -#include "pasta/block_tree/utils/mpsc_queue/mpscq.hpp" -#include "pasta/block_tree/utils/mpsc_queue/queue.hpp" - #include #include #include #include #include +#include +#include +#include #include #include #include namespace pasta { -/// @brief Represents an update function which given a value, -/// updates a value in the map. -/// -/// @tparam Fn The type of the update function. -/// @tparam K The key type saved in the hash map. -/// @tparam V The value type saved in the hash map. -/// @tparam InputV The type that the update function accepts. This is not -/// required to be the same as the map's value type. -/// -template -concept UpdateFunction = requires(K k, V& v_lv, Fn::InputValue in_v_rv) { - typename Fn::InputValue; - // Updates a pre-existing value in the map. - // Arguments are the key, the value in the map, - // and the input value used to update the value in - // the map - { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; - // Initialize a value from an input value - // Arguments are the key, and the value used to - // initialize the value in the map. This returns the - // value to be inserted into the map - { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; -}; - /// /// @brief An update function which on update just overwrites the value. /// From e626e95712e46132409f4ae7072e5add28b24c3d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 5 Oct 2023 18:07:56 +0200 Subject: [PATCH 22/92] synchronized sharded map wip --- examples/build_bt.cpp | 162 +++ .../block_tree_fp_par_sync_sharded.hpp | 1183 +++++++++++++++++ .../block_tree/utils/sync_sharded_map.hpp | 335 +++++ 3 files changed, 1680 insertions(+) create mode 100644 examples/build_bt.cpp create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp create mode 100644 include/pasta/block_tree/utils/sync_sharded_map.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp new file mode 100644 index 0000000..1512134 --- /dev/null +++ b/examples/build_bt.cpp @@ -0,0 +1,162 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#include +#include +#include +#include +// #include +// #include +// #include +// #include +// #include +#include +// #include +// #include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +int main(int argc, char** argv) { + using namespace pasta; + if (argc < 2) { + std::cerr << "Please input file" << std::endl; + exit(1); + } + + if (!std::filesystem::exists(argv[1])) { + std::cerr << "File " << argv[1] << " does not exist" << std::endl; + exit(1); + } + + if (argc < 3) { + std::cerr << "Please input tree arity (tau)" << std::endl; + exit(1); + } + + size_t arity = atoi(argv[2]); + + if (argc < 4) { + std::cerr << "Please input root arity (s)" << std::endl; + exit(1); + } + + size_t root_arity = atoi(argv[3]); + + if (argc < 5) { + std::cerr << "Please input max leaf length" << std::endl; + exit(1); + } + + size_t leaf_length = atoi(argv[4]); + + JiffyQueue, std::pair>> q(100, 8); + + auto p = + std::make_unique, std::pair>>( + std::make_pair(MersenneHash(), std::make_pair(0, 0))); + q.enqueue(std::move(p)); + + std::stringstream ss; + ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" + << leaf_length << "_new.bt"; + std::string out_path = ss.str(); + + std::cout << "building block tree with parameters:" + << "\narity: " << arity << "\nroot arity: " << root_arity + << "\nmax leaf length: " << leaf_length << "\nsaving to " + << out_path << std::endl; + + std::vector text; + { + std::string input; + std::ifstream t(argv[1]); + std::stringstream buffer; + buffer << t.rdbuf(); + input = buffer.str(); + text = std::vector(input.begin(), input.end()); + } + + TimePoint now = Clock::now(); + + /* + auto bt = std::make_unique>(text, + arity, + root_arity, + leaf_length); + */ + + /* + std::unique_ptr> bt( + pasta::make_block_tree_lpf_parallel(text, + arity, + leaf_length, + true, + 8)); + */ + /* + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length); + */ + auto bt = + std::make_unique>( + text, + arity, + root_arity, + leaf_length, + 8); + /* + auto bt = + std::make_unique>(text, + arity, + root_arity, + leaf_length, + 8); +*/ + auto elapsed = + std::chrono::duration_cast(Clock::now() - now) + .count(); + + std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" + << "Time: " << elapsed << "ms" << std::endl; + + // std::ofstream ot(out_path); + // bt->serialize(ot); +#pragma omp parallel for + for (size_t i = 0; i < text.size(); ++i) { + const auto c = bt->access(i); + if (c != text[i]) { + std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] + << "\nActual: " << c << std::endl; + exit(1); + } + } + // ot.close(); + + return 0; +} + +/******************************************************************************/ diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp new file mode 100644 index 0000000..3d9fa5d --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -0,0 +1,1183 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" +#include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BT_NUM_THREADS 4 +#define BT_FILL_THRESHOLD 0.5 +#define BT_QUEUE_CAPACITY 1024 + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template +class BlockTreeFPParShardedSync : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 61; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + robin_hood::unordered_node_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + SyncShardedMap; + +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + size_type block_size; + /// @brief The index of the current level. First level is 0, second level is + /// 1 etc. + size_type level_index; + /// @brief The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + inline void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + inline void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + inline void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + std::cout << "level " << level << std::endl; + LevelData& current = levels.back(); + + TimePoint now = Clock::now(); + scan_block_pairs(text, current, is_padded); + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + scan_blocks(text, current, is_padded); + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + } + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; + prune(levels); + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; + make_tree(text, levels, padding); + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + + TimePoint now = Clock::now(); + std::atomic_size_t num_threads_done = 0; + std::atomic_size_t insert_ops = 0; + std::atomic_bool last_thread_done = false; + std::mutex m; + std::condition_variable cv; + auto& barrier = map.barrier(); + +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + std::cout, \ + num_threads_done, \ + last_thread_done, \ + insert_ops, \ + m, \ + cv, \ + barrier) + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + RabinKarp rk(text, SIGMA, block_starts[i], pair_size, PRIME); + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + shard.insert(hash, i, cv); + insert_ops.fetch_add(1, std::memory_order_acq_rel); + } + const size_t thread_order = + num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_thread_done.store(true, std::memory_order_release); + } + barrier.arrive_and_drop(); + + while (!last_thread_done.load(std::memory_order::acquire)) { + shard.handle_queue(); + } + +#pragma omp barrier +#pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + + std::cout << "Pairs: " << std::endl; + map.print_map_loads(); + map.print_ins_upd(); + std::cout << "Size: " << map.size() << std::endl; + std::cout << "Insert Ops: " << insert_ops.load() << std::endl; + // assert(map.size() == (insert_ops.load() - map.num_updates_.load())); + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given + /// map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); + TimePoint now = Clock::now(); + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + // Generate the bit vector indicating which blocks are internal + + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + // TODO count how often this actually happens + continue; + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + + TimePoint now = Clock::now(); + + std::atomic_size_t insert_ops = 0; + // The number of threads finished with hashing blocks + std::atomic_size_t num_threads_done = 0; + std::atomic_bool last_thread_done = false; + std::mutex m; + std::condition_variable cv; + auto& barrier = links.barrier(); +#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_threads_done, \ + last_thread_done, \ + insert_ops, \ + cv, \ + m, \ + barrier) + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = level_data.block_size; + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + for (size_t i = start; i < end; ++i) { + const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}, cv); + insert_ops.fetch_add(1, std::memory_order_acq_rel); + } + const size_t thread_order = + num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_thread_done.store(true, std::memory_order_release); + cv.notify_all(); + } + barrier.arrive_and_drop(); + + while (!last_thread_done.load(std::memory_order::acquire)) { + shard.handle_queue(); + } +#pragma omp barrier +#pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + // links.print_map_loads(); + } + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "Pairs: " << std::endl; + links.print_map_loads(); + links.print_ins_upd(); + std::cout << "Size: " << links.size() << std::endl; + std::cout << "Insert Ops: " << insert_ops.load() << std::endl; + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + found->second.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + assert(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + assert(child_pointer >= 0); + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParShardedSync(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParShardedSync() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + } + return true; + } +}; // namespace pasta + +} // namespace pasta + +#undef BT_NUM_THREADS diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp new file mode 100644 index 0000000..6d2e6f6 --- /dev/null +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -0,0 +1,335 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// +/// @brief An update function which on update just overwrites the value. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Overwrite { + using InputValue = V; + + inline static void update(K&, V& value, V&& input_value) { + value = input_value; + } + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// +/// @brief An update function which upon update does nothing besides +/// inserting the value if it doesn't exist. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Keep { + using InputValue = V; + inline static void update(K&, V&, V&&) {} + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// @brief A hash map that must be used by multiple threads, each thread having +/// only having write access to a certain segment of the input space. +/// @tparam K The type of the keys in the hash map. +/// @tparam V The type of the values in the hash map. +/// @tparam SeqHashMapType The type of the hash map used internally. +/// This should be compatible with std::unordered_map. +/// @tparam UpdateFn The update function deciding how to insert or update values +/// in the map. +template typename SeqHashMapType = + std::unordered_map, + UpdateFunction UpdateFn = Overwrite> + requires std::movable +class SyncShardedMap { + /// The sequential backing hash map type + using SeqHashMap = SeqHashMapType; + /// The sequential hash map's hasher + using Hasher = SeqHashMap::hasher; + /// The type used for updates + using InputValue = UpdateFn::InputValue; + + using StoredValue = std::pair; + + using mem = std::memory_order; + + /// @brief A value between 0 and 1, determining to which extent + /// each thread's queue should be filled, before the thread is signaled to + /// handle its queued operations. + /// + /// If this value is 0.25, then the thread's value in threshold_met_ + /// is set to true, signaling that the thread should handle its requests + /// in task_queue_ + const double fill_threshold_; + /// @brief The number of threads operating on this map. + const size_t thread_count_; + /// @brief Contains a hash map for each thread + std::vector map_; + /// @brief Contains a task queue for each thread, holding insert + /// operations for each thread. + std::vector> task_queue_; + /// @brief Contains the number of tasks in each thread's queue. + std::span task_count_; + /// @brief Contains the number of threads currently handling their queues. + /// This is used 1. signal to other threads that they should handle their + /// queue, and 2. to keep track of whether all threads have handled their + /// queues. + std::atomic_size_t threads_handling_queue_; + + constexpr static std::invocable auto FN = []() noexcept { + }; + + std::barrier barrier_; + + /// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html + inline uint64_t mix_select(uint64_t key) { + key ^= (key >> 33); + key *= 0xff51afd7ed558ccd; + key ^= (key >> 33); + key *= 0xc4ceb9fe1a85ec53; + key ^= (key >> 33); + return key % thread_count_; + } + +public: + std::atomic_size_t num_updates_; + std::atomic_size_t num_inserts_; + + std::condition_variable aa; + // + /// @brief Creates a new sharded map. + /// + /// @param fill_threshold The fill percentage (between 0 and 1) above which + /// a thread is signaled to handle its own tasks. + /// @param thread_count The exact number of threads working on this map. + /// @param queue_capacity The maximum amount of tasks allowed in each queue. + /// + SyncShardedMap(double fill_threshold, + size_t thread_count, + size_t queue_capacity) + : fill_threshold_(fill_threshold), + thread_count_(thread_count), + map_(), + task_queue_(), + task_count_(), + threads_handling_queue_(0), + barrier_(thread_count, FN), + aa() { + assert(0 <= fill_threshold && fill_threshold <= 1); + map_.reserve(thread_count); + task_queue_.reserve(thread_count); + auto* task_arr = new std::atomic_size_t[thread_count]; + task_count_ = std::span(task_arr, thread_count); + for (size_t i = 0; i < thread_count; i++) { + map_.emplace_back(); + task_queue_.emplace_back(queue_capacity); + task_count_[i] = 0; + } + } + + ~SyncShardedMap() { + delete[] task_count_.data(); + } + + class Shard { + SyncShardedMap& sharded_map_; + const size_t thread_id_; + SeqHashMap& map_; + std::vector& task_queue_; + std::atomic_size_t& task_count_; + size_t last_cycle; + + public: + Shard(SyncShardedMap& sharded_map, size_t thread_id) + : sharded_map_(sharded_map), + thread_id_(thread_id), + map_(sharded_map_.map_[thread_id]), + task_queue_(sharded_map_.task_queue_[thread_id]), + task_count_(sharded_map.task_count_[thread_id]), + last_cycle(0) {} + + /// @brief Inserts or updates a new value in the map, depending on whether + /// @param k The key to insert or update a value for. + /// @param in_value The value with which to insert or update. + inline void insert_or_update_direct(K& k, InputValue&& in_value) { + auto res = map_.find(k); + if (res == map_.end()) { + // If the value does not exist, insert it + V initial = UpdateFn::init(k, std::move(in_value)); + K key = k; + map_.emplace(key, std::move(initial)); + sharded_map_.num_inserts_.fetch_add(1, mem::acq_rel); + } else { + // Otherwise, update it. + V& val = res->second; + UpdateFn::update(k, val, std::move(in_value)); + sharded_map_.num_updates_.fetch_add(1, mem::acq_rel); + } + } + + void handle_queue_sync() { + sharded_map_.threads_handling_queue_.fetch_add(1, mem::seq_cst); + sharded_map_.barrier_.arrive_and_wait(); + + handle_queue(); + + sharded_map_.barrier_.arrive_and_wait(); + sharded_map_.threads_handling_queue_.fetch_sub(1, mem::seq_cst); + } + + /// @brief Handles this thread's queue, inserting or updating all values in + /// its queue, waiting for other threads to be + /// done with their handle_queue call. + void handle_queue() { + const size_t num_tasks = + std::min(task_count_.exchange(0, mem::acq_rel), task_queue_.size()); + // Handle all tasks in the queue + for (size_t i = 0; i < num_tasks; ++i) { + auto entry = task_queue_[i]; + insert_or_update_direct(entry.first, std::move(entry.second)); + } + // All tasks are handled and this thread is done + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param pair The key-value pair to insert or update. + void insert(StoredValue&& pair, std::condition_variable& cv) { + if (sharded_map_.threads_handling_queue_.load(mem::acquire) > 0) { + handle_queue_sync(); + } + const size_t hash = Hasher{}(pair.first); + const size_t target_thread_id = sharded_map_.mix_select(hash); + + // Otherwise enqueue the new value in the target thread + std::vector& q = sharded_map_.task_queue_[target_thread_id]; + std::atomic_size_t& target_task_count = + sharded_map_.task_count_[target_thread_id]; + size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); + // If the target queue is full, signal to the other threads, that they + // need to handle their queue and handle this thread's queue + if (task_idx >= q.size() || + sharded_map_.threads_handling_queue_.load(mem::acquire)) { + handle_queue_sync(); + // Since the queue was handled, the task count is now 0 + task_idx = + sharded_map_.task_count_[target_thread_id].fetch_add(1, + mem::acq_rel); + // std::cout << "e" << task_idx << std::endl; + // assert(prev_task_idx == 0 || prev_task_idx > task_idx); + } + if (task_idx >= q.size()) { + std::cout << "i: " << task_idx << ", qsize: " << q.size() << std::endl; + } + assert(task_idx < q.size()); + // Insert the value into the queue + q.at(task_idx) = std::move(pair); + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param key The key of the value to insert. + /// @param value The value to associate with the key. + inline void insert(K& key, InputValue value, std::condition_variable& cv) { + insert(StoredValue(key, value), cv); + } + }; + + Shard get_shard(const size_t thread_id) { + return Shard(*this, thread_id); + } + + /// @brief Returns the number of key-value pairs in the map. + /// + /// Note, that this method calculates the size for each map separately and + /// is therefore not O(1). + /// @return The number of key-value pairs in the map. + [[nodiscard]] size_t size() const { + size_t size = 0; + for (const SeqHashMap& map : map_) { + size += map.size(); + } + return size; + } + + /// @brief Runs a method for each value in the map. + /// + /// The given function must take const references to a key and a value + /// respectively. + /// @param f The function or lambda to run for each value. + void for_each(std::invocable auto f) { + for (const SeqHashMap& map : map_) { + for (const auto& [k, v] : map) { + f(k, v); + } + } + } + + SeqHashMap::iterator end() { + return map_.back().end(); + } + + SeqHashMap::iterator find(const K& key) { + const size_t hash = Hasher{}(key); + const size_t target_thread_id = mix_select(hash); + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(key); + if (it == map.end()) { + return end(); + } + return it; + } + + void print_map_loads() { + for (size_t i = 0; i < map_.size(); ++i) { + std::cout << "Map " << i << " load: " << map_[i].size() << std::endl; + } + } + + void print_ins_upd() { + std::cout << "Inserts: " << num_inserts_.load() << std::endl; + std::cout << "Updates: " << num_updates_.load() << std::endl; + } + + std::barrier& barrier() { + return barrier_; + } + +}; // namespace pasta + +} // namespace pasta From 9ff25879495ed557a17c3e2d2c509c81e02e67a2 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 9 Oct 2023 20:13:13 +0200 Subject: [PATCH 23/92] somewhat working sharded sync version --- CMakeLists.txt | 1 + CMakePresets.json | 15 +- examples/build_bt.cpp | 7 - .../block_tree_fp_par_sync_sharded.hpp | 190 ++++++++++---- .../block_tree/utils/MersenneRabinKarp.hpp | 2 +- include/pasta/block_tree/utils/concepts.hpp | 2 +- .../block_tree/utils/sync_sharded_map.hpp | 231 ++++++++++++------ 7 files changed, 304 insertions(+), 144 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 857a514..b176619 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -67,6 +67,7 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) +add_definitions(-w) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite) add_library(waitfree-mpsc-queue diff --git a/CMakePresets.json b/CMakePresets.json index 533cc69..4507a1b 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -17,7 +17,7 @@ "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -fsanitize=address -fsanitize=leak -fsanitize=undefined" } }, { @@ -53,13 +53,10 @@ "description": "Default build using Ninja Multi-Config generator", "generator": "Ninja Multi-Config", "binaryDir": "${sourceDir}/build_multi", + "inherits": "default", "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_TESTS": "ON", - "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", - "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", - "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address -static-libasan" + "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON" } }, { @@ -68,12 +65,10 @@ "description": "Use Ninja Multi-Config generator with lax warnings", "generator": "Ninja Multi-Config", "binaryDir": "${sourceDir}/build_lax", + "inherits": "default", "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", - "CMAKE_CXX_FLAGS": "-fopenmp -w -march=native -fdiagnostics-color=always", - "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address -static-libasan" + "CMAKE_CXX_FLAGS": "-fopenmp -w -march=native -fdiagnostics-color=always" } } ], diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 1512134..ee42f3a 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -70,13 +70,6 @@ int main(int argc, char** argv) { size_t leaf_length = atoi(argv[4]); - JiffyQueue, std::pair>> q(100, 8); - - auto p = - std::make_unique, std::pair>>( - std::make_pair(MersenneHash(), std::make_pair(0, 0))); - q.enqueue(std::move(p)); - std::stringstream ss; ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" << leaf_length << "_new.bt"; diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 3d9fa5d..80220b8 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -24,7 +24,6 @@ #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" #include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" @@ -39,9 +38,10 @@ #include #include -#define BT_NUM_THREADS 4 -#define BT_FILL_THRESHOLD 0.5 -#define BT_QUEUE_CAPACITY 1024 +#define BT_NUM_THREADS 12 +#define BT_QUEUE_CAPACITY 163840 +#define BT_DBG_PRINT +#undef BT_DBG_PRINT __extension__ typedef unsigned __int128 uint128_t; @@ -78,7 +78,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_node_map>; + robin_hood::unordered_map>; + // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters @@ -249,7 +250,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// block index and updating the first occurrence if needed /// @param occurrences A reference to the occurrences in the map /// @param input_value The new block index to add to the occurrences - inline static void update(RabinKarpHash&, + inline static void update(const RabinKarpHash&, PairOccurrences& occurrences, InputValue&& input_value) { occurrences.add_block_pair(input_value); @@ -259,7 +260,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Initialize the occurrences of a hashed block pair /// @param input_value The block index of the pair's first block /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(RabinKarpHash&, + inline static PairOccurrences init(const RabinKarpHash&, InputValue&& input_value) { PairOccurrences occurrences(input_value); occurrences.add_block_pair(input_value); @@ -280,10 +281,12 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @param occurrences A reference to the occurrences in the map /// @param input_value The new block index and offset to add to the /// occurrences - inline static void update(RabinKarpHash&, + inline static void update(const RabinKarpHash&, BlockOccurrences& occurrences, InputValue&& input_value) { + size_t prev = occurrences.occurrences.size(); occurrences.add_block(input_value.first); + assert(occurrences.occurrences.size() == prev + 1); occurrences.update(input_value.first, input_value.second); } @@ -291,7 +294,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @param input_value A pair of the block index and offset of one of the /// block's occurrences /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(RabinKarpHash&, + inline static BlockOccurrences init(const RabinKarpHash&, InputValue&& input_value) { BlockOccurrences occurrences(input_value.first); occurrences.add_block(input_value.first); @@ -339,7 +342,8 @@ class BlockTreeFPParShardedSync : public BlockTree { // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { - std::cout << "level " << level << std::endl; + std::cout << "----------------- level " << level << " -----------------" + << std::endl; LevelData& current = levels.back(); TimePoint now = Clock::now(); @@ -416,14 +420,13 @@ class BlockTreeFPParShardedSync : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - BlockPairMap map(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + BlockPairMap map(BT_NUM_THREADS, BT_QUEUE_CAPACITY); TimePoint now = Clock::now(); std::atomic_size_t num_threads_done = 0; - std::atomic_size_t insert_ops = 0; + std::atomic_size_t insert_ops; + insert_ops.store(0); std::atomic_bool last_thread_done = false; - std::mutex m; - std::condition_variable cv; auto& barrier = map.barrier(); #pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ @@ -432,12 +435,9 @@ class BlockTreeFPParShardedSync : public BlockTree { text, \ now, \ is_padded, \ - std::cout, \ num_threads_done, \ last_thread_done, \ insert_ops, \ - m, \ - cv, \ barrier) { const size_t thread_id = omp_get_thread_num(); @@ -469,8 +469,8 @@ class BlockTreeFPParShardedSync : public BlockTree { RabinKarpHash hash = rk.current_hash(); // Try to find the hash in the map, insert a new entry if it doesn't // exist, and add the current block to the entry - shard.insert(hash, i, cv); - insert_ops.fetch_add(1, std::memory_order_acq_rel); + shard.insert(hash, i); + insert_ops.fetch_add(1, std::memory_order_seq_cst); } const size_t thread_order = num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -480,12 +480,15 @@ class BlockTreeFPParShardedSync : public BlockTree { if (is_last_thread) { last_thread_done.store(true, std::memory_order_release); } - barrier.arrive_and_drop(); + // Now, we handle the queue asynchronously while (!last_thread_done.load(std::memory_order::acquire)) { - shard.handle_queue(); + shard.handle_queue_sync(false); } + barrier.arrive_and_drop(); +#pragma omp barrier + shard.handle_queue(); #pragma omp barrier #pragma omp single { @@ -511,12 +514,20 @@ class BlockTreeFPParShardedSync : public BlockTree { std::chrono::duration_cast(Clock::now() - now) .count(); +#ifdef BT_DBG_PRINT std::cout << "Pairs: " << std::endl; map.print_map_loads(); + map.print_queue_upd(); map.print_ins_upd(); std::cout << "Size: " << map.size() << std::endl; - std::cout << "Insert Ops: " << insert_ops.load() << std::endl; - // assert(map.size() == (insert_ops.load() - map.num_updates_.load())); + std::cout << "Insert Cycles: " << insert_ops.load() << std::endl; + std::cout << "Actual Ops: " + << map.num_updates_.load() + map.num_inserts_.load() << std::endl; +#endif + assert(map.num_updates_.load() + map.num_inserts_.load() == + insert_ops.load()); + assert(map.size() == (insert_ops.load() - map.num_updates_.load())); + assert(map.num_inserts_.load() == map.size()); level.is_internal = std::make_unique(level.num_blocks); fill_is_internal(*level.is_internal, map); @@ -524,12 +535,11 @@ class BlockTreeFPParShardedSync : public BlockTree { } /// @brief Fills the bit vector `is_internal` based on the values in the - /// given - /// map. + /// given map. /// @param is_internal An unfilled bit vector with a bit for each block on - /// this level. + /// this level. /// @param map A map, mapping hashed block pairs to their first occurrence's - /// block index. + /// block index. void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { const size_type num_blocks = is_internal.size(); TimePoint now = Clock::now(); @@ -553,7 +563,6 @@ class BlockTreeFPParShardedSync : public BlockTree { now = Clock::now(); // Generate the bit vector indicating which blocks are internal - is_internal[0] = true; is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; for (size_type i = 0; i < num_blocks - 1; ++i) { @@ -565,6 +574,19 @@ class BlockTreeFPParShardedSync : public BlockTree { .count(); } + template + typename Map, + typename Fn> + void print_full_size(const SyncShardedMap& map) noexcept { + size_t full_size = 0; + map.for_each([&full_size](const K& k, const V& v) { + full_size += v.occurrences.size(); + }); + std::cout << "Full size: " << full_size << std::endl; + } + /// @brief Scan through the windows starting in a block and mark /// them accordingly if they represent the earliest occurrence of some /// block hash. @@ -575,7 +597,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// block indexes at which they occur. /// @param num_iterations The number of contiguous windows to hash. /// @param current_block_index The index of the block being currently - /// hashed. + /// hashed. static inline void scan_windows_in_block_pair(RabinKarp& rk, BlockPairMap& map, @@ -618,7 +640,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + BlockMap links(BT_NUM_THREADS, BT_QUEUE_CAPACITY); TimePoint now = Clock::now(); @@ -626,9 +648,8 @@ class BlockTreeFPParShardedSync : public BlockTree { // The number of threads finished with hashing blocks std::atomic_size_t num_threads_done = 0; std::atomic_bool last_thread_done = false; - std::mutex m; - std::condition_variable cv; auto& barrier = links.barrier(); + BitVector tester_pivka(num_blocks, false); #pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ shared(level_data, \ text, \ @@ -638,14 +659,15 @@ class BlockTreeFPParShardedSync : public BlockTree { num_threads_done, \ last_thread_done, \ insert_ops, \ - cv, \ - m, \ - barrier) + barrier, \ + tester_pivka, \ + std::cout) { const size_t num_threads = omp_get_num_threads(); const size_t thread_id = omp_get_thread_num(); typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = level_data.block_size; + const size_t block_size = + std::min(level_data.block_size, text.size()); const std::vector& block_starts = *level_data.block_starts; // Number of total iterations the for loop should do const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; @@ -656,12 +678,19 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); +#ifdef BT_DBG_PRINT + std::osyncstream(std::cout) + << "Thread " << thread_id << " -> start: " << start + << ", end: " << end << std::endl; +#endif + // Hash each block and store their hashes in the map for (size_t i = start; i < end; ++i) { const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}, cv); + shard.insert(hash, {i, 0}); insert_ops.fetch_add(1, std::memory_order_acq_rel); + tester_pivka[i] = true; } const size_t thread_order = num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -670,13 +699,14 @@ class BlockTreeFPParShardedSync : public BlockTree { if (is_last_thread) { last_thread_done.store(true, std::memory_order_release); - cv.notify_all(); } - barrier.arrive_and_drop(); while (!last_thread_done.load(std::memory_order::acquire)) { - shard.handle_queue(); + shard.handle_queue_sync(false); } + barrier.arrive_and_drop(); +#pragma omp barrier + shard.handle_queue(); #pragma omp barrier #pragma omp single { @@ -685,7 +715,6 @@ class BlockTreeFPParShardedSync : public BlockTree { now) .count(); now = Clock::now(); - // links.print_map_loads(); } // Hash every window and find the first occurrences for every block. @@ -707,11 +736,29 @@ class BlockTreeFPParShardedSync : public BlockTree { .count(); now = Clock::now(); - std::cout << "Pairs: " << std::endl; +#ifdef BT_DBG_PRINT + std::cout << "Blocks: " << std::endl; links.print_map_loads(); + links.print_queue_upd(); links.print_ins_upd(); std::cout << "Size: " << links.size() << std::endl; - std::cout << "Insert Ops: " << insert_ops.load() << std::endl; + std::cout << "Insert Cycles: " << insert_ops.load() << std::endl; + std::cout << "Actual Ops: " + << links.num_updates_.load() + links.num_inserts_.load() + << std::endl; + print_full_size(links); +#endif + assert(links.num_updates_.load() + links.num_inserts_.load() == + insert_ops.load()); + assert(links.num_inserts_.load() == links.size()); + +#ifdef BT_DBG_PRINT + for (size_t i = 0; i < tester_pivka.size(); ++i) { + if (!tester_pivka[i]) { + std::cout << "Block " << i << " was not inserted" << std::endl; + } + } +#endif // By this point, the map should contain the first occurrences of every // respective block's content. We then fill the pointers and offsets with @@ -725,6 +772,10 @@ class BlockTreeFPParShardedSync : public BlockTree { continue; } +#ifdef BT_DBG_PRINT + std::cout << occ << " -> " << first_occ.block << "@" + << first_occ.offset << std::endl; +#endif (*level_data.pointers)[occ] = first_occ.block; (*level_data.offsets)[occ] = first_occ.offset; const bool is_back_block = !(*level_data.is_internal)[occ]; @@ -734,6 +785,56 @@ class BlockTreeFPParShardedSync : public BlockTree { } }); +#ifdef BT_DBG_PRINT + for (size_t i = 0; i < num_blocks - is_padded; ++i) { + const RabinKarp rk(text, + SIGMA, + (*level_data.block_starts)[i], + std::min(level_data.block_size, text.size()), + PRIME); + RabinKarpHash hash = rk.current_hash(); + if ((*level_data.is_internal)[i]) { + continue; + } + if ((*level_data.pointers)[i] < 0) { + std::cout << "level " << level_data.level_index << ", block " << i + << " / " << level_data.num_blocks << ", starting at " + << (*level_data.block_starts)[i] << " with length " + << level_data.block_size << " missing pointer, " << std::endl; + if (tester_pivka[i]) { + std::cout << "and was apparently inserted" << std::endl; + } else { + std::cout << "WASN'T inserted" << std::endl; + } + std::cout << "is: "; + switch (links.where(hash)) { + case pasta::Whereabouts::NOWHERE: + std::cout << "nowhere"; + break; + case pasta::Whereabouts::IN_QUEUE: + std::cout << "in queue"; + break; + case pasta::Whereabouts::IN_MAP: + std::cout << "in map"; + break; + } + std::cout << std::endl; + auto found = links.find(hash); + if (found == links.end()) { + std::cout << "and it doesn't have an entry in the map" << std::endl; + } else { + BlockOccurrences& bo = found->second; + typename BlockOccurrences::FirstOccurrence fo = bo.first_occ.load(); + std::cout << "ENTRY EXISTS:\n\tBlock: " << fo.block + << "\n\tOffset: " << fo.offset << "\n\tPosition:" + << ((*level_data.block_starts)[fo.block] + fo.offset) + << std::endl; + } + } + assert((*level_data.pointers)[i] >= 0); + } +#endif + b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -760,7 +861,8 @@ class BlockTreeFPParShardedSync : public BlockTree { if (found == links.end()) { continue; } - found->second.update(current_block_index, offset); + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); } } diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index b9491ff..18b2d55 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -73,7 +73,7 @@ class MersenneRabinKarp { uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { fp = fp * sigma; - fp = mersenneModulo(fp + text_[i]); + fp = mersenneModulo(fp + text_.at(i)); } for (uint64_t i = 0; i < length_ - 1; i++) { sigma_c = mersenneModulo(sigma_c * sigma_); diff --git a/include/pasta/block_tree/utils/concepts.hpp b/include/pasta/block_tree/utils/concepts.hpp index 8fbac04..8ebd2a7 100644 --- a/include/pasta/block_tree/utils/concepts.hpp +++ b/include/pasta/block_tree/utils/concepts.hpp @@ -15,7 +15,7 @@ namespace pasta { /// required to be the same as the map's value type. /// template -concept UpdateFunction = requires(K k, V& v_lv, Fn::InputValue in_v_rv) { +concept UpdateFunction = requires(const K& k, V& v_lv, Fn::InputValue in_v_rv) { typename Fn::InputValue; // Updates a pre-existing value in the map. // Arguments are the key, the value in the map, diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 6d2e6f6..939de4f 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -5,32 +5,32 @@ #include #include #include -#include #include #include #include -#include #include +#include #include #include namespace pasta { +enum Whereabouts { NOWHERE, IN_MAP, IN_QUEUE }; + /// /// @brief An update function which on update just overwrites the value. /// /// @tparam K The key type saved in the hash map. /// @tparam V The value type saved in the hash map. -/// template -struct Overwrite { - using InputValue = V; +struct [[maybe_unused]] Overwrite { + using InputValue [[maybe_unused]] = V; - inline static void update(K&, V& value, V&& input_value) { + inline static void update(const K&, V& value, V&& input_value) { value = input_value; } - inline static V init(K&, V&& input_value) { + inline static V init(const K&, V&& input_value) { return input_value; } }; @@ -43,11 +43,11 @@ struct Overwrite { /// @tparam V The value type saved in the hash map. /// template -struct Keep { - using InputValue = V; - inline static void update(K&, V&, V&&) {} +struct [[maybe_unused]] Keep { + using InputValue [[maybe_unused]] = V; + inline static void update(const K&, V&, V&&) {} - inline static V init(K&, V&& input_value) { + inline static V init(const K&, V&& input_value) { return input_value; } }; @@ -60,8 +60,8 @@ struct Keep { /// This should be compatible with std::unordered_map. /// @tparam UpdateFn The update function deciding how to insert or update values /// in the map. -template typename SeqHashMapType = std::unordered_map, UpdateFunction UpdateFn = Overwrite> @@ -74,31 +74,28 @@ class SyncShardedMap { /// The type used for updates using InputValue = UpdateFn::InputValue; + /// The actual pair of key and value stored in the map using StoredValue = std::pair; + using Queue = std::span; + + /// The memory order namespace from the standard library using mem = std::memory_order; - /// @brief A value between 0 and 1, determining to which extent - /// each thread's queue should be filled, before the thread is signaled to - /// handle its queued operations. - /// - /// If this value is 0.25, then the thread's value in threshold_met_ - /// is set to true, signaling that the thread should handle its requests - /// in task_queue_ - const double fill_threshold_; /// @brief The number of threads operating on this map. const size_t thread_count_; /// @brief Contains a hash map for each thread std::vector map_; /// @brief Contains a task queue for each thread, holding insert - /// operations for each thread. - std::vector> task_queue_; + /// operations for each thread. + std::vector task_queue_; + std::vector task_queue_swap_; /// @brief Contains the number of tasks in each thread's queue. std::span task_count_; /// @brief Contains the number of threads currently handling their queues. - /// This is used 1. signal to other threads that they should handle their - /// queue, and 2. to keep track of whether all threads have handled their - /// queues. + /// This is used 1. signal to other threads that they should handle their + /// queue, and 2. to keep track of whether all threads have handled their + /// queues. std::atomic_size_t threads_handling_queue_; constexpr static std::invocable auto FN = []() noexcept { @@ -106,12 +103,14 @@ class SyncShardedMap { std::barrier barrier_; + std::mutex mtx_; + /// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html inline uint64_t mix_select(uint64_t key) { - key ^= (key >> 33); - key *= 0xff51afd7ed558ccd; - key ^= (key >> 33); - key *= 0xc4ceb9fe1a85ec53; + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; key ^= (key >> 33); return key % thread_count_; } @@ -119,8 +118,6 @@ class SyncShardedMap { public: std::atomic_size_t num_updates_; std::atomic_size_t num_inserts_; - - std::condition_variable aa; // /// @brief Creates a new sharded map. /// @@ -129,40 +126,48 @@ class SyncShardedMap { /// @param thread_count The exact number of threads working on this map. /// @param queue_capacity The maximum amount of tasks allowed in each queue. /// - SyncShardedMap(double fill_threshold, - size_t thread_count, - size_t queue_capacity) - : fill_threshold_(fill_threshold), - thread_count_(thread_count), + SyncShardedMap(size_t thread_count, size_t queue_capacity) + : thread_count_(thread_count), map_(), task_queue_(), task_count_(), threads_handling_queue_(0), barrier_(thread_count, FN), - aa() { - assert(0 <= fill_threshold && fill_threshold <= 1); + num_updates_(0), + num_inserts_(0), + mtx_() { map_.reserve(thread_count); task_queue_.reserve(thread_count); - auto* task_arr = new std::atomic_size_t[thread_count]; - task_count_ = std::span(task_arr, thread_count); + task_queue_swap_.reserve(thread_count); + task_count_ = + std::span(new std::atomic_size_t[thread_count], + thread_count); for (size_t i = 0; i < thread_count; i++) { map_.emplace_back(); - task_queue_.emplace_back(queue_capacity); + task_queue_.emplace_back(new StoredValue[queue_capacity], queue_capacity); + task_queue_swap_.emplace_back(new StoredValue[queue_capacity], + queue_capacity); task_count_[i] = 0; } } ~SyncShardedMap() { delete[] task_count_.data(); + for (auto& queue : task_queue_) { + delete[] queue.data(); + } + for (auto& queue : task_queue_swap_) { + delete[] queue.data(); + } } class Shard { SyncShardedMap& sharded_map_; const size_t thread_id_; SeqHashMap& map_; - std::vector& task_queue_; + Queue& task_queue_; + Queue& task_queue_swap_; std::atomic_size_t& task_count_; - size_t last_cycle; public: Shard(SyncShardedMap& sharded_map, size_t thread_id) @@ -170,19 +175,22 @@ class SyncShardedMap { thread_id_(thread_id), map_(sharded_map_.map_[thread_id]), task_queue_(sharded_map_.task_queue_[thread_id]), - task_count_(sharded_map.task_count_[thread_id]), - last_cycle(0) {} + task_queue_swap_(sharded_map_.task_queue_swap_[thread_id]), + task_count_(sharded_map.task_count_[thread_id]) {} /// @brief Inserts or updates a new value in the map, depending on whether /// @param k The key to insert or update a value for. /// @param in_value The value with which to insert or update. - inline void insert_or_update_direct(K& k, InputValue&& in_value) { + inline void insert_or_update_direct(const K& k, InputValue&& in_value) { + std::lock_guard lock(sharded_map_.mtx_); auto res = map_.find(k); + assert(k.hash_ != 0); if (res == map_.end()) { // If the value does not exist, insert it - V initial = UpdateFn::init(k, std::move(in_value)); K key = k; - map_.emplace(key, std::move(initial)); + V initial = UpdateFn::init(key, std::move(in_value)); + auto [a, b] = map_.emplace(key, std::move(initial)); + assert(b); sharded_map_.num_inserts_.fetch_add(1, mem::acq_rel); } else { // Otherwise, update it. @@ -192,28 +200,44 @@ class SyncShardedMap { } } - void handle_queue_sync() { - sharded_map_.threads_handling_queue_.fetch_add(1, mem::seq_cst); + void handle_queue_sync(bool make_others_wait = true) { + if (make_others_wait) { + // If this value is >0 then other threads will also handle their queue + // when trying to insert + sharded_map_.threads_handling_queue_.fetch_add(1, mem::seq_cst); + } sharded_map_.barrier_.arrive_and_wait(); handle_queue(); sharded_map_.barrier_.arrive_and_wait(); - sharded_map_.threads_handling_queue_.fetch_sub(1, mem::seq_cst); + if (make_others_wait) { + sharded_map_.threads_handling_queue_.fetch_sub(1, mem::seq_cst); + } } /// @brief Handles this thread's queue, inserting or updating all values in /// its queue, waiting for other threads to be /// done with their handle_queue call. void handle_queue() { - const size_t num_tasks = - std::min(task_count_.exchange(0, mem::acq_rel), task_queue_.size()); - // Handle all tasks in the queue + const size_t num_tasks_raw = task_count_.exchange(0, mem::seq_cst); + // assert(num_tasks_raw <= task_queue_.size()); + const size_t num_tasks = std::min(num_tasks_raw, task_queue_.size()); + if (num_tasks == 0) { + return; + } + std::swap(task_queue_, task_queue_swap_); + + static unsigned char zeroed[sizeof(StoredValue)]; + memset(&zeroed, 0, sizeof(StoredValue)); + // Handle all tasks in the queue for (size_t i = 0; i < num_tasks; ++i) { - auto entry = task_queue_[i]; + // bool is_eq = memcmp(zeroed, &task_queue_swap_[i], + // sizeof(StoredValue)); assert(!"hello" || is_eq); + auto& entry = task_queue_swap_[i]; insert_or_update_direct(entry.first, std::move(entry.second)); + // memset(&task_queue_swap_[i], 0, sizeof(StoredValue)); } - // All tasks are handled and this thread is done } /// @brief Inserts or updates a new value in the map. @@ -224,36 +248,53 @@ class SyncShardedMap { /// around to handle its queue using the handle_queue method. /// /// @param pair The key-value pair to insert or update. - void insert(StoredValue&& pair, std::condition_variable& cv) { - if (sharded_map_.threads_handling_queue_.load(mem::acquire) > 0) { - handle_queue_sync(); - } + void insert(StoredValue&& pair) { const size_t hash = Hasher{}(pair.first); const size_t target_thread_id = sharded_map_.mix_select(hash); + if (target_thread_id == thread_id_) { + // If the target thread is this thread, insert the value directly + insert_or_update_direct(pair.first, std::move(pair.second)); + if (sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { + handle_queue_sync(); + } + return; + } // Otherwise enqueue the new value in the target thread - std::vector& q = sharded_map_.task_queue_[target_thread_id]; + Queue* q = &sharded_map_.task_queue_[target_thread_id]; std::atomic_size_t& target_task_count = sharded_map_.task_count_[target_thread_id]; - size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); + // size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); + + sharded_map_.mtx_.lock(); + size_t task_idx = target_task_count.load(mem::seq_cst); // If the target queue is full, signal to the other threads, that they // need to handle their queue and handle this thread's queue - if (task_idx >= q.size() || - sharded_map_.threads_handling_queue_.load(mem::acquire)) { + if (task_idx >= sharded_map_.task_queue_[target_thread_id].size() || + sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { + sharded_map_.mtx_.unlock(); + // Since we incremented that thread's task count, but didn't insert + // anything, we need to decrement it again so that it has the correct + // value + // target_task_count.fetch_sub(1, mem::seq_cst); handle_queue_sync(); // Since the queue was handled, the task count is now 0 - task_idx = - sharded_map_.task_count_[target_thread_id].fetch_add(1, - mem::acq_rel); - // std::cout << "e" << task_idx << std::endl; - // assert(prev_task_idx == 0 || prev_task_idx > task_idx); - } - if (task_idx >= q.size()) { - std::cout << "i: " << task_idx << ", qsize: " << q.size() << std::endl; + // task_idx = target_task_count.fetch_add(1, mem::seq_cst); + insert(std::move(pair)); + } else { + // assert(task_idx < q->size()); + // Insert the value into the queue + + size_t num_tasks_raw = target_task_count.fetch_add(1); + sharded_map_.mtx_.unlock(); + + if (num_tasks_raw >= task_queue_.size()) { + std::cerr << "man: " << num_tasks_raw << std::endl; + } + assert(num_tasks_raw < task_queue_.size()); + sharded_map_.task_queue_[target_thread_id][num_tasks_raw] = + std::move(pair); } - assert(task_idx < q.size()); - // Insert the value into the queue - q.at(task_idx) = std::move(pair); } /// @brief Inserts or updates a new value in the map. @@ -265,8 +306,8 @@ class SyncShardedMap { /// /// @param key The key of the value to insert. /// @param value The value to associate with the key. - inline void insert(K& key, InputValue value, std::condition_variable& cv) { - insert(StoredValue(key, value), cv); + inline void insert(K& key, InputValue value) { + insert(StoredValue(key, value)); } }; @@ -287,12 +328,29 @@ class SyncShardedMap { return size; } + Whereabouts where(const K& k) { + const size_t hash = Hasher{}(k); + const size_t target_thread_id = mix_select(hash); + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(k); + if (it != map.end()) { + return IN_MAP; + } + Queue& queue = task_queue_[target_thread_id]; + for (size_t i = 0; i < task_count_[target_thread_id]; ++i) { + if (queue[i].first == k) { + return IN_QUEUE; + } + } + return NOWHERE; + } + /// @brief Runs a method for each value in the map. /// /// The given function must take const references to a key and a value /// respectively. /// @param f The function or lambda to run for each value. - void for_each(std::invocable auto f) { + void for_each(std::invocable auto f) const { for (const SeqHashMap& map : map_) { for (const auto& [k, v] : map) { f(k, v); @@ -321,9 +379,20 @@ class SyncShardedMap { } } + void print_queue_upd() { + auto so = std::osyncstream(std::cout); + + for (size_t i = 0; i < map_.size(); ++i) { + so << "Queue " << i << " load: " << task_count_[i].load(mem::acquire) + << "\n"; + } + so << std::endl; + } + void print_ins_upd() { - std::cout << "Inserts: " << num_inserts_.load() << std::endl; - std::cout << "Updates: " << num_updates_.load() << std::endl; + std::osyncstream(std::cout) + << "Inserts: " << num_inserts_.load() + << "\nUpdates: " << num_updates_.load() << std::endl; } std::barrier& barrier() { From 9187632dd98205faebce43a1265df110089e30ae Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 10 Oct 2023 14:45:31 +0200 Subject: [PATCH 24/92] sharded version without locks? --- .../block_tree_fp_par_sync_sharded.hpp | 2 +- .../block_tree/utils/sync_sharded_map.hpp | 55 +++++++++---------- 2 files changed, 27 insertions(+), 30 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 80220b8..3bf2af6 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -38,7 +38,7 @@ #include #include -#define BT_NUM_THREADS 12 +#define BT_NUM_THREADS 8 #define BT_QUEUE_CAPACITY 163840 #define BT_DBG_PRINT #undef BT_DBG_PRINT diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 939de4f..2c2e326 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -182,7 +182,6 @@ class SyncShardedMap { /// @param k The key to insert or update a value for. /// @param in_value The value with which to insert or update. inline void insert_or_update_direct(const K& k, InputValue&& in_value) { - std::lock_guard lock(sharded_map_.mtx_); auto res = map_.find(k); assert(k.hash_ != 0); if (res == map_.end()) { @@ -221,12 +220,12 @@ class SyncShardedMap { /// done with their handle_queue call. void handle_queue() { const size_t num_tasks_raw = task_count_.exchange(0, mem::seq_cst); - // assert(num_tasks_raw <= task_queue_.size()); + assert(num_tasks_raw <= task_queue_.size()); const size_t num_tasks = std::min(num_tasks_raw, task_queue_.size()); if (num_tasks == 0) { return; } - std::swap(task_queue_, task_queue_swap_); + // std::swap(task_queue_, task_queue_swap_); static unsigned char zeroed[sizeof(StoredValue)]; memset(&zeroed, 0, sizeof(StoredValue)); @@ -234,7 +233,7 @@ class SyncShardedMap { for (size_t i = 0; i < num_tasks; ++i) { // bool is_eq = memcmp(zeroed, &task_queue_swap_[i], // sizeof(StoredValue)); assert(!"hello" || is_eq); - auto& entry = task_queue_swap_[i]; + auto& entry = task_queue_[i]; insert_or_update_direct(entry.first, std::move(entry.second)); // memset(&task_queue_swap_[i], 0, sizeof(StoredValue)); } @@ -249,14 +248,14 @@ class SyncShardedMap { /// /// @param pair The key-value pair to insert or update. void insert(StoredValue&& pair) { + if (sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { + handle_queue_sync(); + } const size_t hash = Hasher{}(pair.first); const size_t target_thread_id = sharded_map_.mix_select(hash); if (target_thread_id == thread_id_) { // If the target thread is this thread, insert the value directly insert_or_update_direct(pair.first, std::move(pair.second)); - if (sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { - handle_queue_sync(); - } return; } @@ -266,35 +265,33 @@ class SyncShardedMap { sharded_map_.task_count_[target_thread_id]; // size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); - sharded_map_.mtx_.lock(); - size_t task_idx = target_task_count.load(mem::seq_cst); + // sharded_map_.mtx_.lock(); + size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); // If the target queue is full, signal to the other threads, that they // need to handle their queue and handle this thread's queue - if (task_idx >= sharded_map_.task_queue_[target_thread_id].size() || - sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { - sharded_map_.mtx_.unlock(); - // Since we incremented that thread's task count, but didn't insert - // anything, we need to decrement it again so that it has the correct - // value - // target_task_count.fetch_sub(1, mem::seq_cst); + if (task_idx >= sharded_map_.task_queue_[target_thread_id].size()) { + // sharded_map_.mtx_.unlock(); + // Since we incremented that thread's task count, but didn't insert + // anything, we need to decrement it again so that it has the correct + // value + target_task_count.fetch_sub(1, mem::seq_cst); handle_queue_sync(); // Since the queue was handled, the task count is now 0 // task_idx = target_task_count.fetch_add(1, mem::seq_cst); insert(std::move(pair)); - } else { - // assert(task_idx < q->size()); - // Insert the value into the queue - - size_t num_tasks_raw = target_task_count.fetch_add(1); - sharded_map_.mtx_.unlock(); - - if (num_tasks_raw >= task_queue_.size()) { - std::cerr << "man: " << num_tasks_raw << std::endl; - } - assert(num_tasks_raw < task_queue_.size()); - sharded_map_.task_queue_[target_thread_id][num_tasks_raw] = - std::move(pair); + return; } + // assert(task_idx < q->size()); + + // size_t num_tasks_raw = target_task_count.fetch_add(1); + // sharded_map_.mtx_.unlock(); + + // if (num_tasks_raw >= task_queue_.size()) { + // std::cerr << "man: " << num_tasks_raw << std::endl; + // } + assert(task_idx < task_queue_.size()); + // Insert the value into the queue + sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); } /// @brief Inserts or updates a new value in the map. From f5de0ea21e33b4314593aaf01aedc3e71bfd6484 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 11 Oct 2023 18:55:40 +0200 Subject: [PATCH 25/92] add more statistics --- .../block_tree_fp_par_sync_sharded.hpp | 309 +++++++----------- .../block_tree/utils/sync_sharded_map.hpp | 83 +++-- 2 files changed, 177 insertions(+), 215 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 3bf2af6..80eb72e 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -37,6 +37,7 @@ #include #include #include +#include #define BT_NUM_THREADS 8 #define BT_QUEUE_CAPACITY 163840 @@ -340,6 +341,8 @@ class BlockTreeFPParShardedSync : public BlockTree { size_t blocks_ns = 0; size_t generate_ns = 0; + std::cout << "using " << BT_NUM_THREADS << " threads" << std::endl; + // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { std::cout << "----------------- level " << level << " -----------------" @@ -400,6 +403,18 @@ class BlockTreeFPParShardedSync : public BlockTree { return 1 + ((x - 1) / y); } + template + void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10d, max: %10d, avg: %10d, dev: %10d\n", + name, + agg.min() / div, + agg.max() / div, + agg.avg() / div, + agg.standard_deviation(0) / div); + } + /// @brief Scan through the blocks pairwise in order to identify which blocks /// should be replaced with back blocks. /// @@ -424,10 +439,13 @@ class BlockTreeFPParShardedSync : public BlockTree { TimePoint now = Clock::now(); std::atomic_size_t num_threads_done = 0; - std::atomic_size_t insert_ops; - insert_ops.store(0); std::atomic_bool last_thread_done = false; auto& barrier = map.barrier(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; #pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ shared(level, \ @@ -437,8 +455,13 @@ class BlockTreeFPParShardedSync : public BlockTree { is_padded, \ num_threads_done, \ last_thread_done, \ - insert_ops, \ - barrier) + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + std::cout) { const size_t thread_id = omp_get_thread_num(); typename BlockPairMap::Shard shard = map.get_shard(thread_id); @@ -448,8 +471,8 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t num_block_pairs = level.num_blocks - 1 - is_padded; const auto& block_starts = *level.block_starts; - // Hash every window and determine for all block pairs whether they have - // previous occurrences. + // Hash every window and determine for all block pairs whether + // they have previous occurrences. size_t segment_size = std::max(1, ceil_div(num_block_pairs, num_threads)); @@ -459,18 +482,17 @@ class BlockTreeFPParShardedSync : public BlockTree { std::min(num_block_pairs, (thread_id + 1) * segment_size); for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair starting - // at the current block + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block if (!level.next_is_adjacent(i)) { continue; } // Move the hasher to the current block pair RabinKarp rk(text, SIGMA, block_starts[i], pair_size, PRIME); RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it doesn't - // exist, and add the current block to the entry + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry shard.insert(hash, i); - insert_ops.fetch_add(1, std::memory_order_seq_cst); } const size_t thread_order = num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -489,7 +511,6 @@ class BlockTreeFPParShardedSync : public BlockTree { #pragma omp barrier shard.handle_queue(); -#pragma omp barrier #pragma omp single { bp_hash_pairs_ns += @@ -498,6 +519,7 @@ class BlockTreeFPParShardedSync : public BlockTree { .count(); now = Clock::now(); } + tlx::Aggregate thread_scan_hits; if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); @@ -505,28 +527,39 @@ class BlockTreeFPParShardedSync : public BlockTree { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; } - scan_windows_in_block_pair(rk, map, block_size, i); + scan_windows_in_block_pair(rk, map, block_size, i, thread_scan_hits); } } + + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +#pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; + } + + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); } + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (μs) ", total_idle_ns, 1'000); + print_aggregate("Pair Handle Queue (μs) ", finish_idle_ns, 1'000); + bp_scan_pairs_ns += std::chrono::duration_cast(Clock::now() - now) .count(); -#ifdef BT_DBG_PRINT - std::cout << "Pairs: " << std::endl; - map.print_map_loads(); - map.print_queue_upd(); - map.print_ins_upd(); - std::cout << "Size: " << map.size() << std::endl; - std::cout << "Insert Cycles: " << insert_ops.load() << std::endl; - std::cout << "Actual Ops: " - << map.num_updates_.load() + map.num_inserts_.load() << std::endl; -#endif - assert(map.num_updates_.load() + map.num_inserts_.load() == - insert_ops.load()); - assert(map.size() == (insert_ops.load() - map.num_updates_.load())); assert(map.num_inserts_.load() == map.size()); level.is_internal = std::make_unique(level.num_blocks); @@ -545,8 +578,9 @@ class BlockTreeFPParShardedSync : public BlockTree { TimePoint now = Clock::now(); // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. - // The MSB is 1 iff the block and its successor have a prior occurrence. - // The LSB is 1 iff the block and its predecessor have a prior occurrence. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. sdsl::int_vector<2> markings(num_blocks, 0); map.for_each( [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { @@ -602,14 +636,18 @@ class BlockTreeFPParShardedSync : public BlockTree { scan_windows_in_block_pair(RabinKarp& rk, BlockPairMap& map, const size_t num_iterations, - const size_type current_block_index) { + const size_type current_block_index, + tlx::Aggregate& agg) { for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { RabinKarpHash current_hash = rk.current_hash(); - // Find the hash of the current window among the hashed block pairs. + // Find the hash of the current window among the hashed block + // pairs. auto found = map.find(current_hash); if (found == map.end()) { - // TODO count how often this actually happens + agg.add(0); continue; + } else { + agg.add(100); } PairOccurrences& occurrences = found->second; occurrences.update(current_block_index); @@ -644,12 +682,16 @@ class BlockTreeFPParShardedSync : public BlockTree { TimePoint now = Clock::now(); - std::atomic_size_t insert_ops = 0; // The number of threads finished with hashing blocks std::atomic_size_t num_threads_done = 0; std::atomic_bool last_thread_done = false; auto& barrier = links.barrier(); - BitVector tester_pivka(num_blocks, false); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + #pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ shared(level_data, \ text, \ @@ -658,10 +700,12 @@ class BlockTreeFPParShardedSync : public BlockTree { is_padded, \ num_threads_done, \ last_thread_done, \ - insert_ops, \ barrier, \ - tester_pivka, \ - std::cout) + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) { const size_t num_threads = omp_get_num_threads(); const size_t thread_id = omp_get_thread_num(); @@ -678,19 +722,11 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); -#ifdef BT_DBG_PRINT - std::osyncstream(std::cout) - << "Thread " << thread_id << " -> start: " << start - << ", end: " << end << std::endl; -#endif - // Hash each block and store their hashes in the map for (size_t i = start; i < end; ++i) { const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); RabinKarpHash hash = rk.current_hash(); shard.insert(hash, {i, 0}); - insert_ops.fetch_add(1, std::memory_order_acq_rel); - tester_pivka[i] = true; } const size_t thread_order = num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -717,7 +753,9 @@ class BlockTreeFPParShardedSync : public BlockTree { now = Clock::now(); } - // Hash every window and find the first occurrences for every block. + tlx::Aggregate thread_scan_hits; + // Hash every window and find the first occurrences for every + // block. if (start < block_starts.size() - is_padded) { RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); for (size_t i = start; i < end; ++i) { @@ -727,42 +765,43 @@ class BlockTreeFPParShardedSync : public BlockTree { if (static_cast(rk.init_) != block_starts[i]) { rk.restart(block_starts[i]); } - scan_windows_in_block(rk, links, level_data, i); + scan_windows_in_block(rk, links, level_data, i, thread_scan_hits); } } + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +#pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; } b_scan_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); -#ifdef BT_DBG_PRINT - std::cout << "Blocks: " << std::endl; - links.print_map_loads(); - links.print_queue_upd(); - links.print_ins_upd(); - std::cout << "Size: " << links.size() << std::endl; - std::cout << "Insert Cycles: " << insert_ops.load() << std::endl; - std::cout << "Actual Ops: " - << links.num_updates_.load() + links.num_inserts_.load() - << std::endl; - print_full_size(links); -#endif - assert(links.num_updates_.load() + links.num_inserts_.load() == - insert_ops.load()); - assert(links.num_inserts_.load() == links.size()); + tlx::Aggregate map_loads; -#ifdef BT_DBG_PRINT - for (size_t i = 0; i < tester_pivka.size(); ++i) { - if (!tester_pivka[i]) { - std::cout << "Block " << i << " was not inserted" << std::endl; - } + for (size_t load : links.map_loads()) { + map_loads.add(load); } -#endif - // By this point, the map should contain the first occurrences of every - // respective block's content. We then fill the pointers and offsets with - // this data and increment counters accordingly + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Idle (μs) ", total_idle_ns, 1'000); + print_aggregate("Block Handle Queue (μs)", finish_idle_ns, 1'000); + + assert(links.num_inserts_.load() == links.size()); + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly links.for_each( [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { auto first_occ = occs.first_occ.load(); @@ -772,10 +811,6 @@ class BlockTreeFPParShardedSync : public BlockTree { continue; } -#ifdef BT_DBG_PRINT - std::cout << occ << " -> " << first_occ.block << "@" - << first_occ.offset << std::endl; -#endif (*level_data.pointers)[occ] = first_occ.block; (*level_data.offsets)[occ] = first_occ.offset; const bool is_back_block = !(*level_data.is_internal)[occ]; @@ -785,56 +820,6 @@ class BlockTreeFPParShardedSync : public BlockTree { } }); -#ifdef BT_DBG_PRINT - for (size_t i = 0; i < num_blocks - is_padded; ++i) { - const RabinKarp rk(text, - SIGMA, - (*level_data.block_starts)[i], - std::min(level_data.block_size, text.size()), - PRIME); - RabinKarpHash hash = rk.current_hash(); - if ((*level_data.is_internal)[i]) { - continue; - } - if ((*level_data.pointers)[i] < 0) { - std::cout << "level " << level_data.level_index << ", block " << i - << " / " << level_data.num_blocks << ", starting at " - << (*level_data.block_starts)[i] << " with length " - << level_data.block_size << " missing pointer, " << std::endl; - if (tester_pivka[i]) { - std::cout << "and was apparently inserted" << std::endl; - } else { - std::cout << "WASN'T inserted" << std::endl; - } - std::cout << "is: "; - switch (links.where(hash)) { - case pasta::Whereabouts::NOWHERE: - std::cout << "nowhere"; - break; - case pasta::Whereabouts::IN_QUEUE: - std::cout << "in queue"; - break; - case pasta::Whereabouts::IN_MAP: - std::cout << "in map"; - break; - } - std::cout << std::endl; - auto found = links.find(hash); - if (found == links.end()) { - std::cout << "and it doesn't have an entry in the map" << std::endl; - } else { - BlockOccurrences& bo = found->second; - typename BlockOccurrences::FirstOccurrence fo = bo.first_occ.load(); - std::cout << "ENTRY EXISTS:\n\tBlock: " << fo.block - << "\n\tOffset: " << fo.offset << "\n\tPosition:" - << ((*level_data.block_starts)[fo.block] + fo.offset) - << std::endl; - } - } - assert((*level_data.pointers)[i] >= 0); - } -#endif - b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -852,14 +837,18 @@ class BlockTreeFPParShardedSync : public BlockTree { static void scan_windows_in_block(RabinKarp& rk, BlockMap& links, LevelData& level_data, - const size_type current_block_index) { + const size_type current_block_index, + tlx::Aggregate& hits) { for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { const RabinKarpHash hash = rk.current_hash(); // Find all blocks in the multimap that match our hash auto found = links.find(hash); if (found == links.end()) { + hits.add(0); continue; + } else { + hits.add(100); } BlockOccurrences& occurrences = found->second; occurrences.update(current_block_index, offset); @@ -1060,9 +1049,9 @@ class BlockTreeFPParShardedSync : public BlockTree { // Number of pruned blocks before the current block size_type num_pruned = 0; - // We will reuse the allocated memory of the pointers vector to store - // the number of pruned blocks before the block. - // The invariant is that all values up to i are overwritten while all + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all // values starting after i will still be valid pointers // This contains the number of pruned blocks before the block i std::vector& prefix_pruned_blocks = *level.pointers; @@ -1120,11 +1109,12 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @return Whether this block is/stays internal after the pruning process bool prune_block(std::vector& levels, const size_t level_index, - const size_t block_index) { + const size_t block_index) const { LevelData& level = levels[level_index]; BitVector& is_internal = *level.is_internal; - // If the current block is a back block already, there is nothing to prune + // If the current block is a back block already, there is nothing + // to prune if (!is_internal[block_index]) { return false; } @@ -1135,8 +1125,8 @@ class BlockTreeFPParShardedSync : public BlockTree { bool has_internal_children = false; // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are not on the - // last level. + // none of which can be pointed to. So only recurse, if we are + // not on the last level. if (level_index < levels.size() - 1) { const size_type last_child = std::min(first_child + this->tau_ - 1, @@ -1147,7 +1137,8 @@ class BlockTreeFPParShardedSync : public BlockTree { } } - // If any of the children is internal, this block stays internal as well + // If any of the children is internal, this block stays internal + // as well if (has_internal_children) { return true; } @@ -1155,8 +1146,8 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_type pointer = (*level.pointers)[block_index]; const size_type offset = (*level.offsets)[block_index]; const size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing to this, - // then this must stay internal + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal if (pointer == NO_EARLIER_OCC || counter > 0) { return true; } @@ -1232,53 +1223,7 @@ class BlockTreeFPParShardedSync : public BlockTree { delete offsets; } } - - /// @brief Validates that a back-pointer actually points to the same text - /// content. - /// @param text The input text. - /// @param level_index The index of the current level. - /// @param block_index The block index. - /// @param block_start The start index of the block's content in the text. - /// @param source_start The start index of the source block's content in the - /// text. - /// @param source_pointer The block index of the source block. - /// @param source_offset The offset from which the block copies out of the - /// source block. - /// @param block_size The block size. - /// @return `true`, iff the pointer is valid. false otherwise - bool debug_validate_pointer(const std::vector& text, - const size_type level_index, - const size_type block_index, - const size_type block_start, - const size_type source_start, - const size_type source_pointer, - const size_type source_offset, - const size_type block_size) const { - if (source_start + block_size > block_start) { - std::cerr << "source overlapping block on level " << level_index - << ":\n\tBlock Start: " << block_start - << "\n\tSource Start: " << source_start - << "\n\tBlock Size: " << block_size - << "\n\tBlock: " << block_index - << "\n\tSource Block: " << source_pointer - << "\n\tSource Offset: " << source_offset << std::endl; - return false; - } - for (size_type i = 0; i < block_size; i++) { - if (text[block_start + i] != text[source_start + i]) { - std::cerr << "source block mismatch on level " << level_index << ": " - << "\n\tBlock Start: " << block_start - << "\n\tSource Start: " << source_start - << "\n\tBlock Size: " << block_size - << "\n\tBlock: " << block_index - << "\n\tSource Block: " << source_pointer - << "\n\tSource Offset: " << source_offset << std::endl; - return false; - } - } - return true; - } -}; // namespace pasta +}; } // namespace pasta diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 2c2e326..ba2d474 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -89,7 +89,6 @@ class SyncShardedMap { /// @brief Contains a task queue for each thread, holding insert /// operations for each thread. std::vector task_queue_; - std::vector task_queue_swap_; /// @brief Contains the number of tasks in each thread's queue. std::span task_count_; /// @brief Contains the number of threads currently handling their queues. @@ -138,15 +137,12 @@ class SyncShardedMap { mtx_() { map_.reserve(thread_count); task_queue_.reserve(thread_count); - task_queue_swap_.reserve(thread_count); task_count_ = std::span(new std::atomic_size_t[thread_count], thread_count); for (size_t i = 0; i < thread_count; i++) { map_.emplace_back(); task_queue_.emplace_back(new StoredValue[queue_capacity], queue_capacity); - task_queue_swap_.emplace_back(new StoredValue[queue_capacity], - queue_capacity); task_count_[i] = 0; } } @@ -156,9 +152,6 @@ class SyncShardedMap { for (auto& queue : task_queue_) { delete[] queue.data(); } - for (auto& queue : task_queue_swap_) { - delete[] queue.data(); - } } class Shard { @@ -166,8 +159,10 @@ class SyncShardedMap { const size_t thread_id_; SeqHashMap& map_; Queue& task_queue_; - Queue& task_queue_swap_; std::atomic_size_t& task_count_; + tlx::Aggregate start_idle_ns_; + tlx::Aggregate handle_queue_ns_; + tlx::Aggregate finish_idle_ns_; public: Shard(SyncShardedMap& sharded_map, size_t thread_id) @@ -175,8 +170,10 @@ class SyncShardedMap { thread_id_(thread_id), map_(sharded_map_.map_[thread_id]), task_queue_(sharded_map_.task_queue_[thread_id]), - task_queue_swap_(sharded_map_.task_queue_swap_[thread_id]), - task_count_(sharded_map.task_count_[thread_id]) {} + task_count_(sharded_map.task_count_[thread_id]), + start_idle_ns_(), + handle_queue_ns_(), + finish_idle_ns_() {} /// @brief Inserts or updates a new value in the map, depending on whether /// @param k The key to insert or update a value for. @@ -203,15 +200,30 @@ class SyncShardedMap { if (make_others_wait) { // If this value is >0 then other threads will also handle their queue // when trying to insert - sharded_map_.threads_handling_queue_.fetch_add(1, mem::seq_cst); + sharded_map_.threads_handling_queue_.fetch_add(1, mem::acq_rel); } + auto now = std::chrono::high_resolution_clock::now(); sharded_map_.barrier_.arrive_and_wait(); + size_t ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + start_idle_ns_.add(ns_count); + now = std::chrono::high_resolution_clock::now(); handle_queue(); + ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + handle_queue_ns_.add(ns_count); + now = std::chrono::high_resolution_clock::now(); sharded_map_.barrier_.arrive_and_wait(); + ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + finish_idle_ns_.add(ns_count); if (make_others_wait) { - sharded_map_.threads_handling_queue_.fetch_sub(1, mem::seq_cst); + sharded_map_.threads_handling_queue_.fetch_sub(1, mem::acq_rel); } } @@ -219,23 +231,19 @@ class SyncShardedMap { /// its queue, waiting for other threads to be /// done with their handle_queue call. void handle_queue() { - const size_t num_tasks_raw = task_count_.exchange(0, mem::seq_cst); + const size_t num_tasks_raw = task_count_.exchange(0, mem::acq_rel); assert(num_tasks_raw <= task_queue_.size()); const size_t num_tasks = std::min(num_tasks_raw, task_queue_.size()); if (num_tasks == 0) { return; } - // std::swap(task_queue_, task_queue_swap_); static unsigned char zeroed[sizeof(StoredValue)]; memset(&zeroed, 0, sizeof(StoredValue)); // Handle all tasks in the queue for (size_t i = 0; i < num_tasks; ++i) { - // bool is_eq = memcmp(zeroed, &task_queue_swap_[i], - // sizeof(StoredValue)); assert(!"hello" || is_eq); auto& entry = task_queue_[i]; insert_or_update_direct(entry.first, std::move(entry.second)); - // memset(&task_queue_swap_[i], 0, sizeof(StoredValue)); } } @@ -248,7 +256,7 @@ class SyncShardedMap { /// /// @param pair The key-value pair to insert or update. void insert(StoredValue&& pair) { - if (sharded_map_.threads_handling_queue_.load(mem::seq_cst) > 0) { + if (sharded_map_.threads_handling_queue_.load(mem::acquire) > 0) { handle_queue_sync(); } const size_t hash = Hasher{}(pair.first); @@ -260,35 +268,23 @@ class SyncShardedMap { } // Otherwise enqueue the new value in the target thread - Queue* q = &sharded_map_.task_queue_[target_thread_id]; std::atomic_size_t& target_task_count = sharded_map_.task_count_[target_thread_id]; - // size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); - // sharded_map_.mtx_.lock(); - size_t task_idx = target_task_count.fetch_add(1, mem::seq_cst); + size_t task_idx = target_task_count.fetch_add(1, mem::acq_rel); // If the target queue is full, signal to the other threads, that they // need to handle their queue and handle this thread's queue if (task_idx >= sharded_map_.task_queue_[target_thread_id].size()) { - // sharded_map_.mtx_.unlock(); // Since we incremented that thread's task count, but didn't insert // anything, we need to decrement it again so that it has the correct // value - target_task_count.fetch_sub(1, mem::seq_cst); + target_task_count.fetch_sub(1, mem::acq_rel); handle_queue_sync(); // Since the queue was handled, the task count is now 0 // task_idx = target_task_count.fetch_add(1, mem::seq_cst); insert(std::move(pair)); return; } - // assert(task_idx < q->size()); - - // size_t num_tasks_raw = target_task_count.fetch_add(1); - // sharded_map_.mtx_.unlock(); - - // if (num_tasks_raw >= task_queue_.size()) { - // std::cerr << "man: " << num_tasks_raw << std::endl; - // } assert(task_idx < task_queue_.size()); // Insert the value into the queue sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); @@ -306,6 +302,18 @@ class SyncShardedMap { inline void insert(K& key, InputValue value) { insert(StoredValue(key, value)); } + + [[nodiscard]] const tlx::Aggregate& start_idle_ns() const { + return start_idle_ns_; + } + + [[nodiscard]] const tlx::Aggregate& handle_queue_ns() const { + return handle_queue_ns_; + } + + [[nodiscard]] const tlx::Aggregate& finish_idle_ns() const { + return finish_idle_ns_; + } }; Shard get_shard(const size_t thread_id) { @@ -376,7 +384,7 @@ class SyncShardedMap { } } - void print_queue_upd() { + void print_queue_loads() { auto so = std::osyncstream(std::cout); for (size_t i = 0; i < map_.size(); ++i) { @@ -386,6 +394,15 @@ class SyncShardedMap { so << std::endl; } + [[nodiscard]] std::vector map_loads() const { + std::vector loads; + loads.reserve(thread_count_); + for (size_t i = 0; i < thread_count_; ++i) { + loads.push_back(map_[i].size()); + } + return loads; + } + void print_ins_upd() { std::osyncstream(std::cout) << "Inserts: " << num_inserts_.load() From a71946e156526a216016de92de2dd24a913a279e Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 12 Oct 2023 00:41:09 +0200 Subject: [PATCH 26/92] fix formatting issues --- .../block_tree_fp_par_sync_sharded.hpp | 41 ++++++++++++------- 1 file changed, 27 insertions(+), 14 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 80eb72e..74126af 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -403,16 +403,29 @@ class BlockTreeFPParShardedSync : public BlockTree { return 1 + ((x - 1) / y); } - template void print_aggregate(const char* name, - const tlx::Aggregate& agg, + const tlx::Aggregate& agg, size_t div = 1) { - printf("%s -> min: %10d, max: %10d, avg: %10d, dev: %10d\n", + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", name, agg.min() / div, agg.max() / div, - agg.avg() / div, - agg.standard_deviation(0) / div); + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + agg.count()); + } + + void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + auto divf = static_cast(div); + printf("%s -> min: %10f, max: %10f, avg: %10.2f, dev: %10.2f, #: #10u\n", + name, + agg.min() / divf, + agg.max() / divf, + agg.avg() / divf, + agg.standard_deviation(0) / divf, + agg.count()); } /// @brief Scan through the blocks pairwise in order to identify which blocks @@ -441,7 +454,7 @@ class BlockTreeFPParShardedSync : public BlockTree { std::atomic_size_t num_threads_done = 0; std::atomic_bool last_thread_done = false; auto& barrier = map.barrier(); - tlx::Aggregate scan_hits; + tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; tlx::Aggregate finish_idle_ns; tlx::Aggregate total_idle_ns; @@ -519,7 +532,7 @@ class BlockTreeFPParShardedSync : public BlockTree { .count(); now = Clock::now(); } - tlx::Aggregate thread_scan_hits; + tlx::Aggregate thread_scan_hits; if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); @@ -637,7 +650,7 @@ class BlockTreeFPParShardedSync : public BlockTree { BlockPairMap& map, const size_t num_iterations, const size_type current_block_index, - tlx::Aggregate& agg) { + tlx::Aggregate& agg) { for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { RabinKarpHash current_hash = rk.current_hash(); // Find the hash of the current window among the hashed block @@ -686,7 +699,7 @@ class BlockTreeFPParShardedSync : public BlockTree { std::atomic_size_t num_threads_done = 0; std::atomic_bool last_thread_done = false; auto& barrier = links.barrier(); - tlx::Aggregate scan_hits; + tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; tlx::Aggregate finish_idle_ns; tlx::Aggregate total_idle_ns; @@ -753,7 +766,7 @@ class BlockTreeFPParShardedSync : public BlockTree { now = Clock::now(); } - tlx::Aggregate thread_scan_hits; + tlx::Aggregate thread_scan_hits; // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { @@ -792,8 +805,8 @@ class BlockTreeFPParShardedSync : public BlockTree { map_loads.add(load); } - print_aggregate("Block Map Hits ", scan_hits); print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); print_aggregate("Block Idle (μs) ", total_idle_ns, 1'000); print_aggregate("Block Handle Queue (μs)", finish_idle_ns, 1'000); @@ -838,17 +851,17 @@ class BlockTreeFPParShardedSync : public BlockTree { BlockMap& links, LevelData& level_data, const size_type current_block_index, - tlx::Aggregate& hits) { + tlx::Aggregate& hits) { for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { const RabinKarpHash hash = rk.current_hash(); // Find all blocks in the multimap that match our hash auto found = links.find(hash); if (found == links.end()) { - hits.add(0); + hits.add(0.0); continue; } else { - hits.add(100); + hits.add(100.0); } BlockOccurrences& occurrences = found->second; occurrences.update(current_block_index, offset); From a10dac21001fff0749154899751cbe9b4b284c91 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 12 Oct 2023 17:56:06 +0200 Subject: [PATCH 27/92] replace comparison loop with memcmp --- .../block_tree_fp_par_sync_sharded.hpp | 9 ++- .../pasta/block_tree/utils/MersenneHash.hpp | 63 ++++++++++++++++++- 2 files changed, 65 insertions(+), 7 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 74126af..0e1247c 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -566,8 +566,8 @@ class BlockTreeFPParShardedSync : public BlockTree { print_aggregate("Pair Map Loads ", map_loads); print_aggregate("Pair Map Hits ", scan_hits); - print_aggregate("Pair Idle (μs) ", total_idle_ns, 1'000); - print_aggregate("Pair Handle Queue (μs) ", finish_idle_ns, 1'000); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); bp_scan_pairs_ns += std::chrono::duration_cast(Clock::now() - now) @@ -756,7 +756,6 @@ class BlockTreeFPParShardedSync : public BlockTree { barrier.arrive_and_drop(); #pragma omp barrier shard.handle_queue(); -#pragma omp barrier #pragma omp single { b_hash_blocks_ns += @@ -807,8 +806,8 @@ class BlockTreeFPParShardedSync : public BlockTree { print_aggregate("Block Map Loads ", map_loads); print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (μs) ", total_idle_ns, 1'000); - print_aggregate("Block Handle Queue (μs)", finish_idle_ns, 1'000); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); assert(links.num_inserts_.load() == links.size()); diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 8773cc5..990f891 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -29,6 +29,8 @@ namespace pasta { +__extension__ typedef unsigned __int128 uint128_t; + template class MersenneHash { public: @@ -55,18 +57,75 @@ class MersenneHash { bool operator==(const MersenneHash& other) const { // std::cout << start_ << " " << other.start_ << std::endl; - if (length_ != other.length_) + // if (length_ != other.length_) + // return false; + if (hash_ != other.hash_) return false; const std::vector& text = *text_; const std::vector& other_text = *other.text_; +#define MH_MEMCMP +#ifdef MH_LOOP for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { return false; } } - return hash_ == other.hash_; + return true; +#elifdef MH_PACKED_LOOP + const size_t num_blocks = length_ / 8; + for (int block = 0; block < num_blocks; ++block) { + uint64_t a = + *reinterpret_cast(text.data() + start_ + block * 8); + uint64_t b = *reinterpret_cast(other_text.data() + + other.start_ + block * 8); + if (a != b) { + return false; + } + } + return memcmp(text_->data() + start_ + num_blocks * 8, + other.text_->data() + other.start_ + num_blocks * 8, + length_ - num_blocks * 8) == 0; + return true; +#elifdef MH_SSE + // Requires SSE2 + const size_t num_blocks = length_ / 16; + for (int block = 0; block < num_blocks; ++block) { + __m128i_u a = _mm_loadu_si128(reinterpret_cast( + text.data() + start_ + block * 16)); + __m128i_u b = _mm_loadu_si128(reinterpret_cast( + other_text.data() + other.start_ + block * 16)); + __m128i_u res = _mm_cmpeq_epi8(a, b); + if (0xFFFF != _mm_movemask_epi8(res)) { + return false; + } + } + return memcmp(text_->data() + start_ + num_blocks * 16, + other.text_->data() + other.start_ + num_blocks * 16, + length_ - num_blocks * 16) == 0; +#elifdef MH_AVX + // Requires AVX-2 + const size_t num_blocks = length_ / 32; + for (int block = 0; block < num_blocks; ++block) { + __m256i a = _mm256_loadu_si256( + reinterpret_cast(text.data() + start_ + block * 32)); + __m256i b = _mm256_loadu_si256(reinterpret_cast( + other_text.data() + other.start_ + block * 32)); + __m256i res = _mm256_cmpeq_epi8(a, b); + if (0xFFFFFFFF != _mm256_movemask_epi8(res)) { + return false; + } + } + return memcmp(text_->data() + start_ + num_blocks * 32, + other.text_->data() + other.start_ + num_blocks * 32, + length_ - num_blocks * 32) == 0; + +#elifdef MH_MEMCMP + return memcmp(text.data() + start_, + other_text.data() + other.start_, + length_) == 0; +#endif } }; From 6dc1e3d4db0d06d367bec045ec960f144fa54db9 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sun, 15 Oct 2023 20:42:45 +0200 Subject: [PATCH 28/92] add macro to disable debug prints and instrumentation --- .../block_tree_fp_par_sync_sharded.hpp | 246 +++++++++++------- .../pasta/block_tree/utils/MersenneHash.hpp | 11 +- include/pasta/block_tree/utils/debug.hpp | 8 + .../block_tree/utils/sync_sharded_map.hpp | 41 ++- 4 files changed, 191 insertions(+), 115 deletions(-) create mode 100644 include/pasta/block_tree/utils/debug.hpp diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 0e1247c..3bae30c 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -24,16 +24,13 @@ #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include #include #include -#include #include -#include #include #include #include @@ -41,8 +38,6 @@ #define BT_NUM_THREADS 8 #define BT_QUEUE_CAPACITY 163840 -#define BT_DBG_PRINT -#undef BT_DBG_PRINT __extension__ typedef unsigned __int128 uint128_t; @@ -80,7 +75,6 @@ class BlockTreeFPParShardedSync : public BlockTree { template using SeqHashMap = robin_hood::unordered_map>; - // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters @@ -94,6 +88,7 @@ class BlockTreeFPParShardedSync : public BlockTree { using RabinKarpMap = SyncShardedMap; +#ifdef BT_DBG public: size_t bp_hash_pairs_ns = 0; size_t bp_scan_pairs_ns = 0; @@ -103,6 +98,7 @@ class BlockTreeFPParShardedSync : public BlockTree { size_t b_hash_blocks_ns = 0; size_t b_scan_blocks_ns = 0; size_t b_update_blocks_ns = 0; +#endif private: /// @brief Contains data about a block tree level under construction @@ -285,9 +281,7 @@ class BlockTreeFPParShardedSync : public BlockTree { inline static void update(const RabinKarpHash&, BlockOccurrences& occurrences, InputValue&& input_value) { - size_t prev = occurrences.occurrences.size(); occurrences.add_block(input_value.first); - assert(occurrences.occurrences.size() == prev + 1); occurrences.update(input_value.first, input_value.second); } @@ -337,38 +331,49 @@ class BlockTreeFPParShardedSync : public BlockTree { top_level.block_size = top_block_size; top_level.num_blocks = top_level.block_starts->size(); +#ifdef BT_DBG size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; std::cout << "using " << BT_NUM_THREADS << " threads" << std::endl; +#endif // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG std::cout << "----------------- level " << level << " -----------------" << std::endl; - LevelData& current = levels.back(); TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); scan_block_pairs(text, current, is_padded); +#ifdef BT_DBG pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); +#endif scan_blocks(text, current, is_padded); +#ifdef BT_DBG blocks_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the next level (if we're not at the last level) if (level < static_cast(tree_height) - 1) { levels.push_back(std::move(generate_next_level(text, current))); } +#ifdef BT_DBG generate_ns += std::chrono::duration_cast( Clock::now() - now) .count(); +#endif } +#ifdef BT_DBG TimePoint now = Clock::now(); std::cout << "pairs: " << (pairs_ns / 1'000'000) @@ -382,18 +387,23 @@ class BlockTreeFPParShardedSync : public BlockTree { << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," << std::endl; +#endif prune(levels); +#ifdef BT_DBG size_t prune_ns = std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif make_tree(text, levels, padding); +#ifdef BT_DBG size_t make_ns = std::chrono::duration_cast(Clock::now() - now) .count(); std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif } /// @brief Returns the ceiling of x / y for x > 0; @@ -406,26 +416,14 @@ class BlockTreeFPParShardedSync : public BlockTree { void print_aggregate(const char* name, const tlx::Aggregate& agg, size_t div = 1) { - printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", - name, - agg.min() / div, - agg.max() / div, - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - agg.count()); - } - - void print_aggregate(const char* name, - const tlx::Aggregate& agg, - size_t div = 1) { - auto divf = static_cast(div); - printf("%s -> min: %10f, max: %10f, avg: %10.2f, dev: %10.2f, #: #10u\n", - name, - agg.min() / divf, - agg.max() / divf, - agg.avg() / divf, - agg.standard_deviation(0) / divf, - agg.count()); + printf( + "%s -> min: %10u, max: %10u, avCriscog: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); } /// @brief Scan through the blocks pairwise in order to identify which blocks @@ -450,31 +448,35 @@ class BlockTreeFPParShardedSync : public BlockTree { // pairs' first block respectively BlockPairMap map(BT_NUM_THREADS, BT_QUEUE_CAPACITY); - TimePoint now = Clock::now(); - std::atomic_size_t num_threads_done = 0; - std::atomic_bool last_thread_done = false; + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; auto& barrier = map.barrier(); +#ifdef BT_DBG + TimePoint now = Clock::now(); tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; tlx::Aggregate finish_idle_ns; tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level, \ - map, \ - text, \ - now, \ - is_padded, \ - num_threads_done, \ - last_thread_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits, \ - std::cout) +# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif { const size_t thread_id = omp_get_thread_num(); typename BlockPairMap::Shard shard = map.get_shard(thread_id); @@ -508,16 +510,16 @@ class BlockTreeFPParShardedSync : public BlockTree { shard.insert(hash, i); } const size_t thread_order = - num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; const bool is_last_thread = thread_order == num_threads; if (is_last_thread) { - last_thread_done.store(true, std::memory_order_release); + last_done.store(true, std::memory_order_release); } // Now, we handle the queue asynchronously - while (!last_thread_done.load(std::memory_order::acquire)) { + while (!last_done.load(std::memory_order::acquire)) { shard.handle_queue_sync(false); } barrier.arrive_and_drop(); @@ -525,6 +527,7 @@ class BlockTreeFPParShardedSync : public BlockTree { #pragma omp barrier shard.handle_queue(); #pragma omp single +#ifdef BT_DBG { bp_hash_pairs_ns += std::chrono::duration_cast(Clock::now() - @@ -533,6 +536,10 @@ class BlockTreeFPParShardedSync : public BlockTree { now = Clock::now(); } tlx::Aggregate thread_scan_hits; +#else + { + } +#endif if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); @@ -540,15 +547,24 @@ class BlockTreeFPParShardedSync : public BlockTree { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; } - scan_windows_in_block_pair(rk, map, block_size, i, thread_scan_hits); + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_DBG + , + thread_scan_hits +#endif + ); } } +#ifdef BT_DBG auto& start_idle = shard.start_idle_ns(); auto& finish_idle = shard.finish_idle_ns(); auto& handle_queue = shard.handle_queue_ns(); -#pragma omp critical +# pragma omp critical { start_idle_ns.add(start_idle.sum()); finish_idle_ns.add(finish_idle.sum()); @@ -556,8 +572,10 @@ class BlockTreeFPParShardedSync : public BlockTree { handle_queue_ns.add(handle_queue.sum()); scan_hits += thread_scan_hits; }; +#endif } +#ifdef BT_DBG tlx::Aggregate map_loads; for (size_t load : map.map_loads()) { @@ -573,7 +591,8 @@ class BlockTreeFPParShardedSync : public BlockTree { std::chrono::duration_cast(Clock::now() - now) .count(); - assert(map.num_inserts_.load() == map.size()); + BT_ASSERT(map.num_inserts_.load() == map.size()); +#endif level.is_internal = std::make_unique(level.num_blocks); fill_is_internal(*level.is_internal, map); @@ -588,7 +607,9 @@ class BlockTreeFPParShardedSync : public BlockTree { /// block index. void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { const size_type num_blocks = is_internal.size(); +#ifdef BT_DBG TimePoint now = Clock::now(); +#endif // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. // The MSB is 1 iff the block and its successor have a prior @@ -604,10 +625,12 @@ class BlockTreeFPParShardedSync : public BlockTree { } } }); +#ifdef BT_DBG bp_markings_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the bit vector indicating which blocks are internal is_internal[0] = true; @@ -616,22 +639,11 @@ class BlockTreeFPParShardedSync : public BlockTree { const bool block_is_internal = markings[i] != 0b11; is_internal[i] = block_is_internal; } +#ifdef BT_DBG bp_bitvec_ns += std::chrono::duration_cast(Clock::now() - now) .count(); - } - - template - typename Map, - typename Fn> - void print_full_size(const SyncShardedMap& map) noexcept { - size_t full_size = 0; - map.for_each([&full_size](const K& k, const V& v) { - full_size += v.occurrences.size(); - }); - std::cout << "Full size: " << full_size << std::endl; +#endif } /// @brief Scan through the windows starting in a block and mark @@ -649,18 +661,26 @@ class BlockTreeFPParShardedSync : public BlockTree { scan_windows_in_block_pair(RabinKarp& rk, BlockPairMap& map, const size_t num_iterations, - const size_type current_block_index, - tlx::Aggregate& agg) { + const size_type current_block_index +#ifdef BT_DBG + , + tlx::Aggregate& agg +#endif + ) { for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { RabinKarpHash current_hash = rk.current_hash(); // Find the hash of the current window among the hashed block // pairs. auto found = map.find(current_hash); if (found == map.end()) { +#ifdef BT_DBG agg.add(0); continue; } else { agg.add(100); +#else + continue; +#endif } PairOccurrences& occurrences = found->second; occurrences.update(current_block_index); @@ -693,32 +713,37 @@ class BlockTreeFPParShardedSync : public BlockTree { // A map hashing blocks and saving where they occur. BlockMap links(BT_NUM_THREADS, BT_QUEUE_CAPACITY); - TimePoint now = Clock::now(); - // The number of threads finished with hashing blocks - std::atomic_size_t num_threads_done = 0; - std::atomic_bool last_thread_done = false; + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; auto& barrier = links.barrier(); +#ifdef BT_DBG + TimePoint now = Clock::now(); tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; tlx::Aggregate finish_idle_ns; tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level_data, \ - text, \ - links, \ - now, \ - is_padded, \ - num_threads_done, \ - last_thread_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits) +# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif { const size_t num_threads = omp_get_num_threads(); const size_t thread_id = omp_get_thread_num(); @@ -742,21 +767,23 @@ class BlockTreeFPParShardedSync : public BlockTree { shard.insert(hash, {i, 0}); } const size_t thread_order = - num_threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; const bool is_last_thread = thread_order == num_threads; if (is_last_thread) { - last_thread_done.store(true, std::memory_order_release); + last_done.store(true, std::memory_order_release); } - while (!last_thread_done.load(std::memory_order::acquire)) { + while (!last_done.load(std::memory_order::acquire)) { shard.handle_queue_sync(false); } barrier.arrive_and_drop(); #pragma omp barrier shard.handle_queue(); #pragma omp single +#ifdef BT_DBG + { b_hash_blocks_ns += std::chrono::duration_cast(Clock::now() - @@ -766,6 +793,10 @@ class BlockTreeFPParShardedSync : public BlockTree { } tlx::Aggregate thread_scan_hits; +#else + { + } +#endif // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { @@ -777,14 +808,23 @@ class BlockTreeFPParShardedSync : public BlockTree { if (static_cast(rk.init_) != block_starts[i]) { rk.restart(block_starts[i]); } - scan_windows_in_block(rk, links, level_data, i, thread_scan_hits); + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_DBG + , + thread_scan_hits +#endif + ); } } +#ifdef BT_DBG auto& start_idle = shard.start_idle_ns(); auto& finish_idle = shard.finish_idle_ns(); auto& handle_queue = shard.handle_queue_ns(); -#pragma omp critical +# pragma omp critical { start_idle_ns.add(start_idle.sum()); finish_idle_ns.add(finish_idle.sum()); @@ -792,7 +832,9 @@ class BlockTreeFPParShardedSync : public BlockTree { handle_queue_ns.add(handle_queue.sum()); scan_hits += thread_scan_hits; }; +#endif } +#ifdef BT_DBG b_scan_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -809,7 +851,8 @@ class BlockTreeFPParShardedSync : public BlockTree { print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - assert(links.num_inserts_.load() == links.size()); + BT_ASSERT(links.num_inserts_.load() == links.size()); +#endif // By this point, the map should contain the first occurrences of // every respective block's content. We then fill the pointers @@ -832,9 +875,11 @@ class BlockTreeFPParShardedSync : public BlockTree { } }); +#ifdef BT_DBG b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif } /// @brief Scans through block-sized windows starting inside one block and @@ -849,18 +894,25 @@ class BlockTreeFPParShardedSync : public BlockTree { static void scan_windows_in_block(RabinKarp& rk, BlockMap& links, LevelData& level_data, - const size_type current_block_index, - tlx::Aggregate& hits) { + const size_type current_block_index +#ifdef BT_DBG + , + tlx::Aggregate& hits +#endif + ) { for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { const RabinKarpHash hash = rk.current_hash(); // Find all blocks in the multimap that match our hash auto found = links.find(hash); if (found == links.end()) { +#ifdef BT_DBG hits.add(0.0); continue; } else { hits.add(100.0); +#endif + continue; } BlockOccurrences& occurrences = found->second; occurrences.update(current_block_index, offset); @@ -1183,6 +1235,7 @@ class BlockTreeFPParShardedSync : public BlockTree { for (size_type child = last_child; child >= first_child; --child) { const size_type child_pointer = (*child_level.pointers)[child]; const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG if (!(*child_level.is_internal)[child] && child_pointer < 0) { std::cout << "non-internal node missing pointer" << std::endl; std::cout << level_index << ", " << block_index << " / " @@ -1190,8 +1243,9 @@ class BlockTreeFPParShardedSync : public BlockTree { } else if (child_pointer == PRUNED && child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } - assert(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - assert(child_pointer >= 0); + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif // Decrement the counter of where the child points (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 990f891..162365c 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -29,8 +30,6 @@ namespace pasta { -__extension__ typedef unsigned __int128 uint128_t; - template class MersenneHash { public: @@ -73,7 +72,7 @@ class MersenneHash { } } return true; -#elifdef MH_PACKED_LOOP +#elif defined MH_PACKED_LOOP const size_t num_blocks = length_ / 8; for (int block = 0; block < num_blocks; ++block) { uint64_t a = @@ -88,7 +87,7 @@ class MersenneHash { other.text_->data() + other.start_ + num_blocks * 8, length_ - num_blocks * 8) == 0; return true; -#elifdef MH_SSE +#elif defined MH_SSE // Requires SSE2 const size_t num_blocks = length_ / 16; for (int block = 0; block < num_blocks; ++block) { @@ -104,7 +103,7 @@ class MersenneHash { return memcmp(text_->data() + start_ + num_blocks * 16, other.text_->data() + other.start_ + num_blocks * 16, length_ - num_blocks * 16) == 0; -#elifdef MH_AVX +#elif defined MH_AVX // Requires AVX-2 const size_t num_blocks = length_ / 32; for (int block = 0; block < num_blocks; ++block) { @@ -121,7 +120,7 @@ class MersenneHash { other.text_->data() + other.start_ + num_blocks * 32, length_ - num_blocks * 32) == 0; -#elifdef MH_MEMCMP +#elif defined MH_MEMCMP return memcmp(text.data() + start_, other_text.data() + other.start_, length_) == 0; diff --git a/include/pasta/block_tree/utils/debug.hpp b/include/pasta/block_tree/utils/debug.hpp new file mode 100644 index 0000000..e64bea3 --- /dev/null +++ b/include/pasta/block_tree/utils/debug.hpp @@ -0,0 +1,8 @@ +// +// Created by skadic on 15.10.23. +// + +#ifndef PASTA_BLOCK_TREE_DEBUG_HPP +#define PASTA_BLOCK_TREE_DEBUG_HPP + +#endif // PASTA_BLOCK_TREE_DEBUG_HPP diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index ba2d474..65c541e 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -1,13 +1,12 @@ #pragma once #include -#include #include #include #include #include #include -#include +#include #include #include #include @@ -102,8 +101,6 @@ class SyncShardedMap { std::barrier barrier_; - std::mutex mtx_; - /// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html inline uint64_t mix_select(uint64_t key) { key ^= (key >> 31); @@ -133,8 +130,7 @@ class SyncShardedMap { threads_handling_queue_(0), barrier_(thread_count, FN), num_updates_(0), - num_inserts_(0), - mtx_() { + num_inserts_(0) { map_.reserve(thread_count); task_queue_.reserve(thread_count); task_count_ = @@ -160,9 +156,11 @@ class SyncShardedMap { SeqHashMap& map_; Queue& task_queue_; std::atomic_size_t& task_count_; +#ifdef BT_DBG tlx::Aggregate start_idle_ns_; tlx::Aggregate handle_queue_ns_; tlx::Aggregate finish_idle_ns_; +#endif public: Shard(SyncShardedMap& sharded_map, size_t thread_id) @@ -170,29 +168,36 @@ class SyncShardedMap { thread_id_(thread_id), map_(sharded_map_.map_[thread_id]), task_queue_(sharded_map_.task_queue_[thread_id]), - task_count_(sharded_map.task_count_[thread_id]), + task_count_(sharded_map.task_count_[thread_id]) +#ifdef BT_DBG + , start_idle_ns_(), handle_queue_ns_(), - finish_idle_ns_() {} + finish_idle_ns_() +#endif + { + } /// @brief Inserts or updates a new value in the map, depending on whether /// @param k The key to insert or update a value for. /// @param in_value The value with which to insert or update. inline void insert_or_update_direct(const K& k, InputValue&& in_value) { auto res = map_.find(k); - assert(k.hash_ != 0); if (res == map_.end()) { // If the value does not exist, insert it K key = k; V initial = UpdateFn::init(key, std::move(in_value)); - auto [a, b] = map_.emplace(key, std::move(initial)); - assert(b); + map_.emplace(key, std::move(initial)); +#ifdef BT_DBG sharded_map_.num_inserts_.fetch_add(1, mem::acq_rel); +#endif } else { // Otherwise, update it. V& val = res->second; UpdateFn::update(k, val, std::move(in_value)); +#ifdef BT_DBG sharded_map_.num_updates_.fetch_add(1, mem::acq_rel); +#endif } } @@ -202,26 +207,34 @@ class SyncShardedMap { // when trying to insert sharded_map_.threads_handling_queue_.fetch_add(1, mem::acq_rel); } +#ifdef BT_DBG auto now = std::chrono::high_resolution_clock::now(); +#endif sharded_map_.barrier_.arrive_and_wait(); +#ifdef BT_DBG size_t ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); start_idle_ns_.add(ns_count); now = std::chrono::high_resolution_clock::now(); +#endif handle_queue(); +#ifdef BT_DBG ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); handle_queue_ns_.add(ns_count); now = std::chrono::high_resolution_clock::now(); +#endif sharded_map_.barrier_.arrive_and_wait(); +#ifdef BT_DBG ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); finish_idle_ns_.add(ns_count); +#endif if (make_others_wait) { sharded_map_.threads_handling_queue_.fetch_sub(1, mem::acq_rel); } @@ -232,7 +245,7 @@ class SyncShardedMap { /// done with their handle_queue call. void handle_queue() { const size_t num_tasks_raw = task_count_.exchange(0, mem::acq_rel); - assert(num_tasks_raw <= task_queue_.size()); + BT_ASSERT(num_tasks_raw <= task_queue_.size()); const size_t num_tasks = std::min(num_tasks_raw, task_queue_.size()); if (num_tasks == 0) { return; @@ -285,7 +298,7 @@ class SyncShardedMap { insert(std::move(pair)); return; } - assert(task_idx < task_queue_.size()); + BT_ASSERT(task_idx < task_queue_.size()); // Insert the value into the queue sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); } @@ -303,6 +316,7 @@ class SyncShardedMap { insert(StoredValue(key, value)); } +#ifdef BT_DBG [[nodiscard]] const tlx::Aggregate& start_idle_ns() const { return start_idle_ns_; } @@ -314,6 +328,7 @@ class SyncShardedMap { [[nodiscard]] const tlx::Aggregate& finish_idle_ns() const { return finish_idle_ns_; } +#endif }; Shard get_shard(const size_t thread_id) { From 5c796b30848a51c3c47c3cfb91279880e9eb295f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sun, 15 Oct 2023 20:54:27 +0200 Subject: [PATCH 29/92] add gpl3 notice --- .../block_tree/utils/MersenneRabinKarp.hpp | 23 +++++++++--- include/pasta/block_tree/utils/debug.hpp | 35 +++++++++++++++---- .../pasta/block_tree/utils/sharded_map.hpp | 22 ++++++++++-- .../block_tree/utils/sync_sharded_map.hpp | 19 ++++++++++ 4 files changed, 84 insertions(+), 15 deletions(-) diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 18b2d55..bef4295 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -32,7 +33,9 @@ namespace pasta { /// @tparam T The type of the characters in the text. /// @tparam size_type The type to use for indexing etc. /// @tparam mersenne_exponent If using a mersenne prime 2^p-1, then this should -/// be p. If this is 0, a normal modulus operation will be used +/// be p. If this is 0, a normal modulus operation will be used. +/// Additionally, if this is != 0, then the prime_ attribute will be ignored +/// and '(1 << mersenne_exponent) - 1' will be used instead. /// template class MersenneRabinKarp { @@ -57,7 +60,8 @@ class MersenneRabinKarp { /// @param sigma The alphabet size. /// @param init The start index of the first hashed window in the text. /// @param length The window size. - /// @param prime A large prime used for modulus operations. + /// @param prime A large prime used for modulus operations + /// iff not using mersenne_exponent. MersenneRabinKarp(std::vector const& text, uint64_t sigma, uint64_t init, @@ -82,6 +86,8 @@ class MersenneRabinKarp { max_sigma_ = (uint64_t)(sigma_c); }; + /// @brief Moves the hasher to the specified start index in the backing + /// vector. void restart(uint64_t index) { if (index + length_ >= text_.size()) { return; @@ -99,26 +105,33 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - uint128_t i = (k & prime_) + (k >> mersenne_exponent); - return (i >= prime_) ? i - prime_ : i; + constexpr static uint128_t MERSENNE = (1ULL << mersenne_exponent) - 1; + uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); + return (i >= MERSENNE) ? i - MERSENNE : i; } }; + /// @brief Retrieves the hash value at the hasher's current position. + /// @return A MersenneHash object representing the current hash value. inline MersenneHash current_hash() const { return MersenneHash(text_, hash_, init_, length_); } + /// @brief Advances the hasher by one character. void next() { if (text_.size() <= init_ + length_) { return; } + constexpr static uint128_t MERSENNE = + mersenne_exponent == 0 ? prime_ : (1ULL << mersenne_exponent) - 1; + uint128_t fp = hash_; T out_char = text_[init_]; T in_char = text_[init_ + length_]; const uint128_t out_char_influence = mersenneModulo(out_char * max_sigma_); // Conditionally add the prime, of the out_char_influence is too large - fp += prime_ * (out_char_influence > hash_) - out_char_influence; + fp += MERSENNE * (out_char_influence > hash_) - out_char_influence; fp *= sigma_; fp += in_char; fp = mersenneModulo(fp); diff --git a/include/pasta/block_tree/utils/debug.hpp b/include/pasta/block_tree/utils/debug.hpp index e64bea3..fb661d3 100644 --- a/include/pasta/block_tree/utils/debug.hpp +++ b/include/pasta/block_tree/utils/debug.hpp @@ -1,8 +1,29 @@ -// -// Created by skadic on 15.10.23. -// +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ +#pragma once -#ifndef PASTA_BLOCK_TREE_DEBUG_HPP -#define PASTA_BLOCK_TREE_DEBUG_HPP - -#endif // PASTA_BLOCK_TREE_DEBUG_HPP +#ifdef BT_DBG +# include +/// @brief An assertion that is only active, if BT_DBG is defined. +# define BT_ASSERT(x) assert(x) +#else +/// @brief An assertion that is only active, if BT_DBG is defined. +# define BT_ASSERT(x) +#endif diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp index 45f8237..8e6ff73 100644 --- a/include/pasta/block_tree/utils/sharded_map.hpp +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -1,14 +1,30 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ #pragma once #include #include -#include #include #include #include #include -#include -#include #include #include diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 65c541e..7c59450 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -1,3 +1,22 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ #pragma once #include From d1c71a9f7ff1135f2d4151ba74da15ac0cecbbcf Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 16 Oct 2023 19:14:25 +0200 Subject: [PATCH 30/92] make build_bt a little more ergonomic --- CMakeLists.txt | 18 +- examples/build_bt.cpp | 174 ++++++++++++------ .../block_tree_fp_par_sharded.hpp | 23 ++- .../block_tree_fp_par_sync_sharded.hpp | 29 ++- .../pasta/block_tree/utils/MersenneHash.hpp | 29 ++- .../block_tree/utils/MersenneRabinKarp.hpp | 10 +- 6 files changed, 184 insertions(+), 99 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b176619..b8eb420 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -34,6 +34,8 @@ option(PASTA_BLOCK_TREE_BUILD_TESTS "Build blocktree's tests." OFF) option(PASTA_BLOCK_TREE_BUILD_EXAMPLES "Build blocktree's benchmarks." OFF) +option(PASTA_BLOCK_TREE_DEBUG + "Add instrumentation, calculating time etc." ON) # Optional test if (PASTA_BLOCK_TREE_BUILD_TESTS) @@ -59,6 +61,9 @@ if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) examples/build_bt.cpp) target_link_libraries(build_bt pasta_block_tree) + if (PASTA_BLOCK_TREE_DEBUG) + target_compile_definitions(build_bt PRIVATE BT_DBG) + endif () endif () @@ -67,8 +72,8 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) -add_definitions(-w) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite) + +#add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite) add_library(waitfree-mpsc-queue ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) @@ -87,15 +92,22 @@ add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) +file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp) +add_library(sdsl ${sdsl_sources}) +target_include_directories(sdsl SYSTEM INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) +set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") + target_link_libraries(pasta_block_tree INTERFACE libsais pasta_bit_vector tlx robin_hood waitfree-mpsc-queue - sdsl #jiffy jiffy1) +target_link_libraries(pasta_block_tree INTERFACE + sdsl) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index ee42f3a..c4c7bca 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,14 +22,99 @@ #include #include #include -// #include -// #include -// #include -// #include -// #include -#include -// #include -// #include +#include + +#define PAR_PHMAP +#ifdef FP +# include +std::unique_ptr make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t) { + ; + return std::unique_ptr>( + pasta::make_block_tree_fp(text, arity, leaf_length)); +} +# define ALGO_NAME "fp" +#elif defined FP2 +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t) { + ; + return std::make_unique>(text, + arity, + 1, + leaf_length); +} +# define ALGO_NAME "fp2" +#elif defined LPF +# include +std::unique_ptr make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::unique_ptr>( + pasta::make_block_tree_lpf_parallel(text, + arity, + leaf_length, + true, + threads)); +} +# define ALGO_NAME "lpf" +#elif defined PAR_SHARDED +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "shard" +#elif defined PAR_SHARDED_SYNC +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "shard_sync" +#elif defined PAR_PHMAP +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "par_map" +#endif + #include #include @@ -54,31 +139,32 @@ int main(int argc, char** argv) { exit(1); } - size_t arity = atoi(argv[2]); + const size_t arity = atoi(argv[2]); if (argc < 4) { - std::cerr << "Please input root arity (s)" << std::endl; + std::cerr << "Please input max leaf length" << std::endl; exit(1); } - size_t root_arity = atoi(argv[3]); + const size_t leaf_length = atoi(argv[3]); if (argc < 5) { - std::cerr << "Please input max leaf length" << std::endl; + std::cerr << "Please input number of threads (ignored if single threaded " + "algorithm)" + << std::endl; exit(1); } - size_t leaf_length = atoi(argv[4]); + const size_t threads = atoi(argv[4]); std::stringstream ss; - ss << argv[1] << "_arit" << arity << "_root" << root_arity << "_leaf" - << leaf_length << "_new.bt"; + ss << argv[1] << "_arit" << arity << "_leaf" << leaf_length << "_new.bt"; std::string out_path = ss.str(); std::cout << "building block tree with parameters:" - << "\narity: " << arity << "\nroot arity: " << root_arity - << "\nmax leaf length: " << leaf_length << "\nsaving to " - << out_path << std::endl; + << "\narity: " << arity << "\nmax leaf length: " << leaf_length + << "\nsaving to " << out_path << "\nusing " << threads << " threads" + << std::endl; std::vector text; { @@ -91,62 +177,32 @@ int main(int argc, char** argv) { } TimePoint now = Clock::now(); - - /* - auto bt = std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ - - /* - std::unique_ptr> bt( - pasta::make_block_tree_lpf_parallel(text, - arity, - leaf_length, - true, - 8)); - */ - /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length); - */ - auto bt = - std::make_unique>( - text, - arity, - root_arity, - leaf_length, - 8); - /* - auto bt = - std::make_unique>(text, - arity, - root_arity, - leaf_length, - 8); -*/ + auto bt = make_bt(text, arity, leaf_length, threads); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); - std::cout << "bt size: " << bt->print_space_usage() / 1000 << "kb\n" - << "Time: " << elapsed << "ms" << std::endl; + std::cout << "RESULT file=" + << std::filesystem::path(argv[1]).filename().string() + << " arity=" << arity << " leaf_length=" << leaf_length << " time + = " << elapsed << " threads = " << threads + << " space=" << bt->print_space_usage() + << std::endl; // std::ofstream ot(out_path); // bt->serialize(ot); + /* #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { - std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] + std::cerr << "Error at position " << i << "\nExpected: " << +(char)text[i] << "\nActual: " << c << std::endl; exit(1); } } + */ // ot.close(); return 0; diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp index 97eabf8..1e40298 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -26,7 +26,6 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" -#include "pasta/block_tree/utils/mpsc_queue/queue.hpp" #include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" #include "pasta/block_tree/utils/sharded_map.hpp" @@ -58,7 +57,7 @@ namespace pasta { template typename queue_type = StupidQueue> -class BlockTreeFPParShardedSync : public BlockTree { +class BlockTreeFPParSharded : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -259,7 +258,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// block index and updating the first occurrence if needed /// @param occurrences A reference to the occurrences in the map /// @param input_value The new block index to add to the occurrences - inline static void update(RabinKarpHash&, + inline static void update(const RabinKarpHash&, PairOccurrences& occurrences, InputValue&& input_value) { occurrences.add_block_pair(input_value); @@ -269,7 +268,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Initialize the occurrences of a hashed block pair /// @param input_value The block index of the pair's first block /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(RabinKarpHash&, + inline static PairOccurrences init(const RabinKarpHash&, InputValue&& input_value) { PairOccurrences occurrences(input_value); occurrences.add_block_pair(input_value); @@ -290,7 +289,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @param occurrences A reference to the occurrences in the map /// @param input_value The new block index and offset to add to the /// occurrences - inline static void update(RabinKarpHash&, + inline static void update(const RabinKarpHash&, BlockOccurrences& occurrences, InputValue&& input_value) { occurrences.add_block(input_value.first); @@ -301,7 +300,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @param input_value A pair of the block index and offset of one of the /// block's occurrences /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(RabinKarpHash&, + inline static BlockOccurrences init(const RabinKarpHash&, InputValue&& input_value) { BlockOccurrences occurrences(input_value.first); occurrences.add_block(input_value.first); @@ -1055,11 +1054,11 @@ class BlockTreeFPParShardedSync : public BlockTree { } public: - BlockTreeFPParShardedSync(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads) { + BlockTreeFPParSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { const auto old = omp_get_max_threads(); const auto old_dynamic = omp_get_dynamic(); omp_set_dynamic(0); @@ -1073,7 +1072,7 @@ class BlockTreeFPParShardedSync : public BlockTree { omp_set_num_threads(old); } - ~BlockTreeFPParShardedSync() { + ~BlockTreeFPParSharded() { for (auto& rank : this->block_tree_types_rs_) { delete rank; } diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 3bae30c..4394a41 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -36,7 +36,6 @@ #include #include -#define BT_NUM_THREADS 8 #define BT_QUEUE_CAPACITY 163840 __extension__ typedef unsigned __int128 uint128_t; @@ -305,7 +304,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Constructs the block tree. /// @param text The input text. - void construct(const std::vector& text) { + void construct(const std::vector& text, const size_t threads) { const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -348,14 +347,14 @@ class BlockTreeFPParShardedSync : public BlockTree { TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); - scan_block_pairs(text, current, is_padded); + scan_block_pairs(text, current, is_padded, threads); #ifdef BT_DBG pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); #endif - scan_blocks(text, current, is_padded); + scan_blocks(text, current, is_padded, threads); #ifdef BT_DBG blocks_ns += std::chrono::duration_cast( Clock::now() - now) @@ -437,7 +436,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @return The block start indices for the next level of the tree void scan_block_pairs(const std::vector& text, LevelData& level, - const bool is_padded) { + const bool is_padded, + const size_t threads) { if (level.num_blocks < 4) { level.is_internal = std::make_unique(level.num_blocks, true); level.is_internal_rank = std::make_unique(*level.is_internal); @@ -446,7 +446,7 @@ class BlockTreeFPParShardedSync : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - BlockPairMap map(BT_NUM_THREADS, BT_QUEUE_CAPACITY); + BlockPairMap map(threads, BT_QUEUE_CAPACITY); std::atomic_size_t threads_done = 0; std::atomic_bool last_done = false; @@ -459,7 +459,7 @@ class BlockTreeFPParShardedSync : public BlockTree { tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ +# pragma omp parallel default(none) threads(BT_NUM_THREADS) \ shared(level, \ map, \ text, \ @@ -474,16 +474,16 @@ class BlockTreeFPParShardedSync : public BlockTree { handle_queue_ns, \ scan_hits) #else -# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ +# pragma omp parallel default(none) num_threads(threads) \ shared(level, map, text, is_padded, threads_done, last_done, barrier) #endif { const size_t thread_id = omp_get_thread_num(); typename BlockPairMap::Shard shard = map.get_shard(thread_id); const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; const size_t block_size = level.block_size; const size_t pair_size = 2 * block_size; - const size_t num_block_pairs = level.num_blocks - 1 - is_padded; const auto& block_starts = *level.block_starts; // Hash every window and determine for all block pairs whether @@ -696,7 +696,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// end of the text void scan_blocks(const std::vector& text, LevelData& level_data, - const bool is_padded) { + const bool is_padded, + const size_t threads) { const size_t num_blocks = level_data.num_blocks; level_data.pointers = @@ -711,7 +712,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(BT_NUM_THREADS, BT_QUEUE_CAPACITY); + BlockMap links(threads, BT_QUEUE_CAPACITY); // The number of threads finished with hashing blocks std::atomic_size_t num_done = 0; @@ -741,7 +742,7 @@ class BlockTreeFPParShardedSync : public BlockTree { handle_queue_ns, \ scan_hits) #else -# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ +# pragma omp parallel default(none) num_threads(threads) \ shared(level_data, text, links, is_padded, num_done, last_done, barrier) #endif { @@ -1270,7 +1271,7 @@ class BlockTreeFPParShardedSync : public BlockTree { this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; this->map_unique_chars(text); - construct(text); + construct(text, threads); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } @@ -1292,5 +1293,3 @@ class BlockTreeFPParShardedSync : public BlockTree { }; } // namespace pasta - -#undef BT_NUM_THREADS diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 162365c..de12e96 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -64,7 +64,7 @@ class MersenneHash { const std::vector& text = *text_; const std::vector& other_text = *other.text_; -#define MH_MEMCMP +#define MH_PACKED_LOOP_UNROLL #ifdef MH_LOOP for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { @@ -74,7 +74,7 @@ class MersenneHash { return true; #elif defined MH_PACKED_LOOP const size_t num_blocks = length_ / 8; - for (int block = 0; block < num_blocks; ++block) { + for (size_t block = 0; block < num_blocks; ++block) { uint64_t a = *reinterpret_cast(text.data() + start_ + block * 8); uint64_t b = *reinterpret_cast(other_text.data() + @@ -86,11 +86,28 @@ class MersenneHash { return memcmp(text_->data() + start_ + num_blocks * 8, other.text_->data() + other.start_ + num_blocks * 8, length_ - num_blocks * 8) == 0; - return true; +#elif defined MH_PACKED_LOOP_UNROLL + const size_t num_blocks = length_ / 16; + for (size_t block = 0; block < num_blocks; ++block) { + uint64_t a1 = *reinterpret_cast(text.data() + start_ + + 2 * block * 8); + uint64_t a2 = *reinterpret_cast(text.data() + start_ + + (2 * block + 1) * 8); + uint64_t b1 = *reinterpret_cast( + other_text.data() + other.start_ + 2 * block * 8); + uint64_t b2 = *reinterpret_cast( + other_text.data() + other.start_ + (2 * block + 1) * 8); + if (a1 != b1 || a2 != b2) { + return false; + } + } + return memcmp(text_->data() + start_ + num_blocks * 16, + other.text_->data() + other.start_ + num_blocks * 16, + length_ - num_blocks * 16) == 0; #elif defined MH_SSE // Requires SSE2 const size_t num_blocks = length_ / 16; - for (int block = 0; block < num_blocks; ++block) { + for (size_t block = 0; block < num_blocks; ++block) { __m128i_u a = _mm_loadu_si128(reinterpret_cast( text.data() + start_ + block * 16)); __m128i_u b = _mm_loadu_si128(reinterpret_cast( @@ -106,13 +123,13 @@ class MersenneHash { #elif defined MH_AVX // Requires AVX-2 const size_t num_blocks = length_ / 32; - for (int block = 0; block < num_blocks; ++block) { + for (size_t block = 0; block < num_blocks; ++block) { __m256i a = _mm256_loadu_si256( reinterpret_cast(text.data() + start_ + block * 32)); __m256i b = _mm256_loadu_si256(reinterpret_cast( other_text.data() + other.start_ + block * 32)); __m256i res = _mm256_cmpeq_epi8(a, b); - if (0xFFFFFFFF != _mm256_movemask_epi8(res)) { + if (static_cast(0xFFFFFFFF) != _mm256_movemask_epi8(res)) { return false; } } diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index bef4295..1e048bf 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -123,15 +123,17 @@ class MersenneRabinKarp { return; } - constexpr static uint128_t MERSENNE = - mersenne_exponent == 0 ? prime_ : (1ULL << mersenne_exponent) - 1; - uint128_t fp = hash_; T out_char = text_[init_]; T in_char = text_[init_ + length_]; const uint128_t out_char_influence = mersenneModulo(out_char * max_sigma_); // Conditionally add the prime, of the out_char_influence is too large - fp += MERSENNE * (out_char_influence > hash_) - out_char_influence; + if constexpr (mersenne_exponent == 0) { + fp += prime_ * (out_char_influence > hash_) - out_char_influence; + } else { + fp += ((1ULL << mersenne_exponent) - 1) * (out_char_influence > hash_) - + out_char_influence; + } fp *= sigma_; fp += in_char; fp = mersenneModulo(fp); From 9751d4db1f3612b0679aa0fed5ba2166d50338de Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 17 Oct 2023 19:59:11 +0200 Subject: [PATCH 31/92] conditional debug prints in older versions --- .../construction/block_tree_fp_par_phmap.hpp | 90 +++++++++----- .../block_tree_fp_par_sharded.hpp | 113 ++++++++++++------ 2 files changed, 141 insertions(+), 62 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp index 722e16b..8067333 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -40,8 +40,6 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; -#define BT_NUM_THREADS 8 - __extension__ typedef unsigned __int128 uint128_t; namespace pasta { @@ -77,7 +75,8 @@ class BlockTreeFPParPH : public BlockTree { template > - using HashMap = robin_hood::unordered_node_map; + using HashMap = + robin_hood::unordered_node_map; /// A rabin karp hasher preconfigured for the current template parameters using RabinKarp = MersenneRabinKarp; @@ -88,10 +87,10 @@ class BlockTreeFPParPH : public BlockTree { template - using RabinKarpMap = HashMap>; + using RabinKarpMap = + HashMap>; +#ifdef BT_DBG public: size_t bp_hash_pairs_ns = 0; size_t bp_scan_pairs_ns = 0; @@ -103,6 +102,7 @@ class BlockTreeFPParPH : public BlockTree { size_t b_update_blocks_ns = 0; private: +#endif /// @brief Contains data about a block tree level under construction struct LevelData { /// Contains a 1 for each internal block (= block with children) @@ -153,7 +153,7 @@ class BlockTreeFPParPH : public BlockTree { } }; - void construct(const std::vector& text) { + void construct(const std::vector& text, const size_t threads) { const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -179,35 +179,46 @@ class BlockTreeFPParPH : public BlockTree { top_level.block_size = top_block_size; top_level.num_blocks = top_level.block_starts->size(); +#ifdef BT_DBG size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; +#endif // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { // std::cout << "level " << level << std::endl; - LevelData& current = levels.back(); +#ifdef BT_DBG TimePoint now = Clock::now(); - scan_block_pairs(text, current, is_padded); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); - scan_blocks(text, current, is_padded); +#endif + scan_blocks(text, current, is_padded, threads); +#ifdef BT_DBG blocks_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the next level (if we're not at the last level) if (level < static_cast(tree_height) - 1) { levels.push_back(std::move(generate_next_level(text, current))); } +#ifdef BT_DBG generate_ns += std::chrono::duration_cast( Clock::now() - now) .count(); +#endif } +#ifdef BT_DBG TimePoint now = Clock::now(); std::cout << "pairs: " << (pairs_ns / 1'000'000) @@ -221,18 +232,23 @@ class BlockTreeFPParPH : public BlockTree { << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," << std::endl; +#endif prune(levels); +#ifdef BT_DBG size_t prune_ns = std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif make_tree(text, levels, padding); +#ifdef BT_DBG size_t make_ns = std::chrono::duration_cast(Clock::now() - now) .count(); std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif } /// @brief Returns the ceiling of x / y for x > 0; @@ -269,7 +285,7 @@ class BlockTreeFPParPH : public BlockTree { } }; - /// @briefScan through the blocks pairwise in order to identify which blocks + /// @brief Scan through the blocks pairwise in order to identify which blocks /// should /// be replaced with back blocks. /// @@ -279,7 +295,8 @@ class BlockTreeFPParPH : public BlockTree { /// @return The block start indices for the next level of the tree void scan_block_pairs(const std::vector& text, LevelData& level, - const bool is_padded) { + const bool is_padded, + const size_t threads) { if (level.num_blocks < 4) { level.is_internal = std::make_unique(level.num_blocks, true); level.is_internal_rank = std::make_unique(*level.is_internal); @@ -290,10 +307,14 @@ class BlockTreeFPParPH : public BlockTree { // pairs' first block respectively RabinKarpMap map(level.num_blocks); +#ifdef BT_DBG TimePoint now = Clock::now(); - -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level, map, text, now, is_padded) +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded) +#endif { const size_t block_size = level.block_size; const size_t pair_size = 2 * block_size; @@ -321,7 +342,8 @@ class BlockTreeFPParPH : public BlockTree { ptr->second.update(i); } #pragma omp barrier -#pragma omp single +#ifdef BT_DBG +# pragma omp single { bp_hash_pairs_ns += std::chrono::duration_cast(Clock::now() - @@ -329,6 +351,7 @@ class BlockTreeFPParPH : public BlockTree { .count(); now = Clock::now(); } +#endif // Hash every window and determine for all block pairs whether they have // previous occurrences. @@ -352,10 +375,12 @@ class BlockTreeFPParPH : public BlockTree { } } +#ifdef BT_DBG bp_scan_pairs_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. @@ -372,10 +397,12 @@ class BlockTreeFPParPH : public BlockTree { } map.erase(it); } +#ifdef BT_DBG bp_markings_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the bit vector indicating which blocks are internal level.is_internal = std::make_unique(level.num_blocks); @@ -387,9 +414,11 @@ class BlockTreeFPParPH : public BlockTree { const bool block_is_internal = markings[i] != 0b11; is_internal[i] = block_is_internal; } +#ifdef BT_DBG bp_bitvec_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif level.is_internal_rank = std::make_unique(*level.is_internal); } @@ -485,7 +514,8 @@ class BlockTreeFPParPH : public BlockTree { /// end of the text void scan_blocks(const std::vector& text, LevelData& level_data, - const bool is_padded) { + const bool is_padded, + const size_t threads) { const size_t num_blocks = level_data.num_blocks; level_data.pointers = @@ -505,10 +535,14 @@ class BlockTreeFPParPH : public BlockTree { // hash has already been processed RabinKarpMap links(num_blocks); +#ifdef BT_DBG TimePoint now = Clock::now(); - -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level_data, text, links, now, is_padded, std::cout) +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded) +#endif { const std::vector& block_starts = *level_data.block_starts; #pragma omp single @@ -530,7 +564,8 @@ class BlockTreeFPParPH : public BlockTree { } #pragma omp barrier -#pragma omp single +#ifdef BT_DBG +# pragma omp single { b_hash_blocks_ns += std::chrono::duration_cast(Clock::now() - @@ -538,6 +573,7 @@ class BlockTreeFPParPH : public BlockTree { .count(); now = Clock::now(); } +#endif const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; @@ -565,12 +601,13 @@ class BlockTreeFPParPH : public BlockTree { scan_windows_in_block(rk, links, level_data, i); } } -#pragma omp barrier } +#ifdef BT_DBG b_scan_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // By this point, the map should contain the first occurrences of every // respective block's content. We then fill the pointers and offsets with @@ -592,9 +629,11 @@ class BlockTreeFPParPH : public BlockTree { is_back_block && (first_occ.offset > 0); } } +#ifdef BT_DBG b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif } /// @brief Scans through block-sized windows starting inside one block and /// tries to find earlier occurrences of blocks. Non-internal blocks will @@ -619,7 +658,6 @@ class BlockTreeFPParPH : public BlockTree { continue; } found->second.update(current_block_index, offset); - continue; } } @@ -961,7 +999,7 @@ class BlockTreeFPParPH : public BlockTree { this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; this->map_unique_chars(text); - construct(text); + construct(text, threads); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } @@ -1028,6 +1066,4 @@ class BlockTreeFPParPH : public BlockTree { } }; // namespace pasta -} // namespace pasta - -#undef BT_NUM_THREADS +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp index 1e40298..81c0509 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -40,9 +40,8 @@ #include #include -#define BT_NUM_THREADS 8 #define BT_FILL_THRESHOLD 0.5 -#define BT_QUEUE_CAPACITY 1024 +#define BT_QUEUE_CAPACITY 163840 __extension__ typedef unsigned __int128 uint128_t; @@ -56,7 +55,7 @@ namespace pasta { /// in the sharded hash map. template typename queue_type = StupidQueue> + template typename queue_type = JiffyQueue> class BlockTreeFPParSharded : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -101,6 +100,7 @@ class BlockTreeFPParSharded : public BlockTree { using RabinKarpMap = ShardedMap; +#ifdef BT_DBG public: size_t bp_hash_pairs_ns = 0; size_t bp_scan_pairs_ns = 0; @@ -112,6 +112,7 @@ class BlockTreeFPParSharded : public BlockTree { size_t b_update_blocks_ns = 0; private: +#endif /// @brief Contains data about a block tree level under construction struct LevelData { /// @brief Contains a 1 for each internal block (= block with children) @@ -316,7 +317,7 @@ class BlockTreeFPParSharded : public BlockTree { /// @brief Constructs the block tree. /// @param text The input text. - void construct(const std::vector& text) { + void construct(const std::vector& text, const size_t threads) { const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -342,35 +343,49 @@ class BlockTreeFPParSharded : public BlockTree { top_level.block_size = top_block_size; top_level.num_blocks = top_level.block_starts->size(); +#ifdef BT_DBG size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; +#endif // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG std::cout << "level " << level << std::endl; - LevelData& current = levels.back(); - TimePoint now = Clock::now(); - scan_block_pairs(text, current, is_padded); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); - scan_blocks(text, current, is_padded); +#endif + +#ifdef BT_DBG + scan_blocks(text, current, is_padded, threads); +#endif + +#ifdef BT_DBG blocks_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the next level (if we're not at the last level) if (level < static_cast(tree_height) - 1) { levels.push_back(std::move(generate_next_level(text, current))); } +#ifdef BT_DBG generate_ns += std::chrono::duration_cast( Clock::now() - now) .count(); +#endif } +#ifdef BT_DBG TimePoint now = Clock::now(); std::cout << "pairs: " << (pairs_ns / 1'000'000) @@ -384,18 +399,23 @@ class BlockTreeFPParSharded : public BlockTree { << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," << std::endl; +#endif prune(levels); +#ifdef BT_DBG size_t prune_ns = std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif make_tree(text, levels, padding); +#ifdef BT_DBG size_t make_ns = std::chrono::duration_cast(Clock::now() - now) .count(); std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif } /// @brief Returns the ceiling of x / y for x > 0; @@ -416,7 +436,8 @@ class BlockTreeFPParSharded : public BlockTree { /// @return The block start indices for the next level of the tree void scan_block_pairs(const std::vector& text, LevelData& level, - const bool is_padded) { + const bool is_padded, + const size_t threads) { if (level.num_blocks < 4) { level.is_internal = std::make_unique(level.num_blocks, true); level.is_internal_rank = std::make_unique(*level.is_internal); @@ -425,13 +446,18 @@ class BlockTreeFPParSharded : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - BlockPairMap map(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); + BlockPairMap map(BT_FILL_THRESHOLD, threads, BT_QUEUE_CAPACITY); - TimePoint now = Clock::now(); std::atomic_size_t num_threads_finished = 0; -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level, map, text, now, is_padded, std::cout, num_threads_finished) +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded, num_threads_finished) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, num_threads_finished) +#endif { const size_t thread_id = omp_get_thread_num(); typename BlockPairMap::Shard shard = map.get_shard(thread_id); @@ -475,7 +501,8 @@ class BlockTreeFPParSharded : public BlockTree { shard.handle_queue(); } while (num_threads_finished.load() < num_threads); #pragma omp barrier -#pragma omp single +#ifdef BT_DBG +# pragma omp single { bp_hash_pairs_ns += std::chrono::duration_cast(Clock::now() - @@ -483,6 +510,7 @@ class BlockTreeFPParSharded : public BlockTree { .count(); now = Clock::now(); } +#endif if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); @@ -495,9 +523,11 @@ class BlockTreeFPParSharded : public BlockTree { } } +#ifdef BT_DBG bp_scan_pairs_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif level.is_internal = std::make_unique(level.num_blocks); fill_is_internal(*level.is_internal, map); @@ -512,7 +542,9 @@ class BlockTreeFPParSharded : public BlockTree { /// block index. void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { const size_type num_blocks = is_internal.size(); +#ifdef BT_DBG TimePoint now = Clock::now(); +#endif // Set up the packed array holding the markings for each block. // Each mark is a 2-bit number. // The MSB is 1 iff the block and its successor have a prior occurrence. @@ -527,10 +559,13 @@ class BlockTreeFPParSharded : public BlockTree { } } }); + +#ifdef BT_DBG bp_markings_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // Generate the bit vector indicating which blocks are internal @@ -540,9 +575,11 @@ class BlockTreeFPParSharded : public BlockTree { const bool block_is_internal = markings[i] != 0b11; is_internal[i] = block_is_internal; } +#ifdef BT_DBG bp_bitvec_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif } /// @brief Scan through the windows starting in a block and mark @@ -582,7 +619,8 @@ class BlockTreeFPParSharded : public BlockTree { /// end of the text void scan_blocks(const std::vector& text, LevelData& level_data, - const bool is_padded) { + const bool is_padded, + const size_t threads) { const size_t num_blocks = level_data.num_blocks; level_data.pointers = @@ -597,14 +635,18 @@ class BlockTreeFPParSharded : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(BT_FILL_THRESHOLD, BT_NUM_THREADS, BT_QUEUE_CAPACITY); - - TimePoint now = Clock::now(); + BlockMap links(BT_FILL_THRESHOLD, threads, BT_QUEUE_CAPACITY); // The number of threads finished with hashing blocks std::atomic_size_t num_threads_finished = 0; -#pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ - shared(level_data, text, links, now, is_padded, num_threads_finished) +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, num_threads_finished) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_threads_finished) +#endif { const size_t num_threads = omp_get_num_threads(); const size_t thread_id = omp_get_thread_num(); @@ -621,11 +663,6 @@ class BlockTreeFPParSharded : public BlockTree { (thread_id + 1) * segment_size); // Hash each block and store their hashes in the map - // FIXME This causes issues when run in parallel - // In make_tree, we get an error when deallocating vectors in LevelData - // Seems like in this case, the algorithm fails to identify some earlier - // occurrences for non-internal blocks, leading to writes to offsets[-1] - // etc. later on. for (size_t i = start; i < end; ++i) { const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); RabinKarpHash hash = rk.current_hash(); @@ -642,15 +679,16 @@ class BlockTreeFPParSharded : public BlockTree { do { shard.handle_queue(); } while (num_threads_finished.load() < num_threads); -#pragma omp single +#ifdef BT_DBG +# pragma omp single { b_hash_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); - // links.print_map_loads(); } +#endif // Hash every window and find the first occurrences for every block. if (start < block_starts.size() - is_padded) { @@ -666,10 +704,12 @@ class BlockTreeFPParSharded : public BlockTree { } } } +#ifdef BT_DBG b_scan_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +#endif // By this point, the map should contain the first occurrences of every // respective block's content. We then fill the pointers and offsets with @@ -692,9 +732,11 @@ class BlockTreeFPParSharded : public BlockTree { } }); +#ifdef BT_DBG b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); +#endif } /// @brief Scans through block-sized windows starting inside one block and @@ -975,7 +1017,8 @@ class BlockTreeFPParSharded : public BlockTree { /// @return Whether this block is/stays internal after the pruning process bool prune_block(std::vector& levels, const size_t level_index, - const size_t block_index) { + const size_t block_index_) { + volatile size_t block_index = block_index_; LevelData& level = levels[level_index]; BitVector& is_internal = *level.is_internal; @@ -993,7 +1036,7 @@ class BlockTreeFPParSharded : public BlockTree { // none of which can be pointed to. So only recurse, if we are not on the // last level. if (level_index < levels.size() - 1) { - const size_type last_child = + volatile size_type last_child = std::min(first_child + this->tau_ - 1, levels[level_index + 1].is_internal->size() - 1); // Iterate through children in reverse @@ -1007,9 +1050,9 @@ class BlockTreeFPParSharded : public BlockTree { return true; } - const size_type pointer = (*level.pointers)[block_index]; - const size_type offset = (*level.offsets)[block_index]; - const size_type counter = (*level.counters)[block_index]; + volatile size_type pointer = (*level.pointers)[block_index]; + volatile size_type offset = (*level.offsets)[block_index]; + volatile size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing to this, // then this must stay internal if (pointer == NO_EARLIER_OCC || counter > 0) { @@ -1035,12 +1078,14 @@ class BlockTreeFPParSharded : public BlockTree { for (size_type child = last_child; child >= first_child; --child) { const size_type child_pointer = (*child_level.pointers)[child]; const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG if (!(*child_level.is_internal)[child] && child_pointer < 0) { std::cout << "non-internal node missing pointer" << std::endl; std::cout << level_index << ", " << block_index << std::endl; } else if (child_pointer == PRUNED && child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } +#endif assert(!(*child_level.is_internal)[child] || child_pointer == PRUNED); assert(child_pointer >= 0); // Decrement the counter of where the child points @@ -1067,7 +1112,7 @@ class BlockTreeFPParSharded : public BlockTree { this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; this->map_unique_chars(text); - construct(text); + construct(text, threads); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } @@ -1135,5 +1180,3 @@ class BlockTreeFPParSharded : public BlockTree { }; // namespace pasta } // namespace pasta - -#undef BT_NUM_THREADS From 0915d1d166f30539af04d023011c5bf51ae63f00 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 17 Oct 2023 20:02:55 +0200 Subject: [PATCH 32/92] adjust build_bt executable --- examples/build_bt.cpp | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index c4c7bca..8c4897c 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,13 +24,14 @@ #include #include -#define PAR_PHMAP +#define FP2 #ifdef FP # include -std::unique_ptr make_bt(std::vector& text, - const size_t arity, - const size_t leaf_length, - const size_t) { +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t) { ; return std::unique_ptr>( pasta::make_block_tree_fp(text, arity, leaf_length)); @@ -52,10 +53,11 @@ make_bt(std::vector& text, # define ALGO_NAME "fp2" #elif defined LPF # include -std::unique_ptr make_bt(std::vector& text, - const size_t arity, - const size_t leaf_length, - const size_t threads) { +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { ; return std::unique_ptr>( pasta::make_block_tree_lpf_parallel(text, @@ -161,10 +163,12 @@ int main(int argc, char** argv) { ss << argv[1] << "_arit" << arity << "_leaf" << leaf_length << "_new.bt"; std::string out_path = ss.str(); +#ifdef BT_DBG std::cout << "building block tree with parameters:" << "\narity: " << arity << "\nmax leaf length: " << leaf_length << "\nsaving to " << out_path << "\nusing " << threads << " threads" << std::endl; +#endif std::vector text; { @@ -182,27 +186,23 @@ int main(int argc, char** argv) { std::chrono::duration_cast(Clock::now() - now) .count(); - std::cout << "RESULT file=" - << std::filesystem::path(argv[1]).filename().string() - << " arity=" << arity << " leaf_length=" << leaf_length << " time - = " << elapsed << " threads = " << threads - << " space=" << bt->print_space_usage() - << std::endl; + std::cout << "RESULT algo=" << ALGO_NAME + << " file=" << std::filesystem::path(argv[1]).filename().string() + << " arity=" << arity << " leaf_length=" << leaf_length + << " time=" << elapsed << " threads=" << threads + << " space=" << bt->print_space_usage() << std::endl; // std::ofstream ot(out_path); // bt->serialize(ot); - /* #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { - std::cerr << "Error at position " << i << "\nExpected: " << -(char)text[i] + std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] << "\nActual: " << c << std::endl; exit(1); } } - */ // ot.close(); return 0; From 6480a646851dce31375a8be9a87b3bed8fca312b Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 17 Oct 2023 23:14:32 +0200 Subject: [PATCH 33/92] fix missing 64 build of libdivsufsort --- CMakeLists.txt | 37 ++++++++++++++++++++----------------- 1 file changed, 20 insertions(+), 17 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index b8eb420..3897f71 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,31 +39,31 @@ option(PASTA_BLOCK_TREE_DEBUG # Optional test if (PASTA_BLOCK_TREE_BUILD_TESTS) - include(FetchContent) - FetchContent_Declare( + include(FetchContent) + FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-1.12.1 ) - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - FetchContent_MakeAvailable(googletest) - enable_testing() - include(GoogleTest) - add_subdirectory(tests) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + enable_testing() + include(GoogleTest) + add_subdirectory(tests) endif () if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) - add_executable(block_tree_construction + add_executable(block_tree_construction examples/block_tree_construction.cpp) - target_link_libraries(block_tree_construction + target_link_libraries(block_tree_construction pasta_block_tree) - add_executable(build_bt + add_executable(build_bt examples/build_bt.cpp) - target_link_libraries(build_bt + target_link_libraries(build_bt pasta_block_tree) - if (PASTA_BLOCK_TREE_DEBUG) - target_compile_definitions(build_bt PRIVATE BT_DBG) - endif () + if (PASTA_BLOCK_TREE_DEBUG) + target_compile_definitions(build_bt PRIVATE BT_DBG) + endif () endif () @@ -73,7 +73,8 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) -#add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite) +set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode") +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort) add_library(waitfree-mpsc-queue ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) @@ -96,6 +97,9 @@ file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp) add_library(sdsl ${sdsl_sources}) target_include_directories(sdsl SYSTEM INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) +target_include_directories(sdsl SYSTEM INTERFACE + ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) +target_link_libraries(sdsl PUBLIC divsufsort64) set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") target_link_libraries(pasta_block_tree INTERFACE @@ -104,10 +108,9 @@ target_link_libraries(pasta_block_tree INTERFACE tlx robin_hood waitfree-mpsc-queue + sdsl #jiffy jiffy1) -target_link_libraries(pasta_block_tree INTERFACE - sdsl) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE From 59d345cfd3b16b8fe0645b6026d35e3318358ae6 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 18 Oct 2023 00:57:17 +0200 Subject: [PATCH 34/92] fix issue with debug prints --- CMakeLists.txt | 22 +++++++++++++++++ CMakePresets.json | 24 ++++++++----------- examples/build_bt.cpp | 14 +++++++++-- .../block_tree_fp_par_sync_sharded.hpp | 6 ++--- 4 files changed, 47 insertions(+), 19 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 3897f71..48e35c0 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -37,6 +37,7 @@ option(PASTA_BLOCK_TREE_BUILD_EXAMPLES option(PASTA_BLOCK_TREE_DEBUG "Add instrumentation, calculating time etc." ON) +include(ExternalProject) # Optional test if (PASTA_BLOCK_TREE_BUILD_TESTS) include(FetchContent) @@ -65,6 +66,27 @@ if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) target_compile_definitions(build_bt PRIVATE BT_DBG) endif () + if (PASTA_BLOCK_TREE_MALLOC_COUNT) + message(DEBUG "malloc_count enabled") + # malloc_count + ExternalProject_Add(malloc_count + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + GIT_REPOSITORY git@github.com:Skadic/malloc_count.git + BUILD_COMMAND gcc -c -fpic -ldl ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count/malloc_count.c + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + ) + + add_library(libmalloc_count INTERFACE) + target_include_directories(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) + target_link_libraries(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count-build/malloc_count.o) + + target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/extlib/malloc_count) + target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) + #target_link_libraries(build_bt libmalloc_count) + #target_compile_definitions(build_bt PRIVATE BT_MALLOC_COUNT) + endif() + endif () set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE) diff --git a/CMakePresets.json b/CMakePresets.json index 4507a1b..66f7d22 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -60,15 +60,17 @@ } }, { - "name": "lax", - "displayName": "Lax Ninja Multi-Config", - "description": "Use Ninja Multi-Config generator with lax warnings", - "generator": "Ninja Multi-Config", - "binaryDir": "${sourceDir}/build_lax", + "name": "bench", + "displayName": "Behcnmarking Configuration", + "description": "Release build with malloc_count", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build_bench", "inherits": "default", "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", - "CMAKE_CXX_FLAGS": "-fopenmp -w -march=native -fdiagnostics-color=always" + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -ldl", + "CMAKE_BUILD_TYPE": "Release", + "PASTA_BLOCK_TREE_MALLOC_COUNT": "ON" } } ], @@ -101,14 +103,8 @@ "configuration": "Debug" }, { - "name": "release-lax", - "configurePreset": "lax", - "configuration": "Release" - }, - { - "name": "relwithdeb-lax", - "configurePreset": "lax", - "configuration": "RelWithDebInfo" + "name": "bench", + "configurePreset": "bench" } ], "testPresets": [ diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 8c4897c..25fd749 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,7 @@ #include #include -#define FP2 +#define LPF #ifdef FP # include std::unique_ptr> @@ -117,6 +117,9 @@ make_bt(std::vector& text, # define ALGO_NAME "par_map" #endif +#ifdef BT_MALLOC_COUNT +#include +#endif #include #include @@ -190,7 +193,14 @@ int main(int argc, char** argv) { << " file=" << std::filesystem::path(argv[1]).filename().string() << " arity=" << arity << " leaf_length=" << leaf_length << " time=" << elapsed << " threads=" << threads - << " space=" << bt->print_space_usage() << std::endl; + << " space=" << bt->print_space_usage(); + + +#ifdef BT_MALLOC_COUNT + std::cout << " memory=" << malloc_count_peak(); +#endif + + std::cout << std::endl; // std::ofstream ot(out_path); // bt->serialize(ot); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 4394a41..781fa2f 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -334,7 +334,7 @@ class BlockTreeFPParShardedSync : public BlockTree { size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; - + std::cout << "using " << BT_NUM_THREADS << " threads" << std::endl; #endif @@ -459,7 +459,7 @@ class BlockTreeFPParShardedSync : public BlockTree { tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -# pragma omp parallel default(none) threads(BT_NUM_THREADS) \ +# pragma omp parallel default(none) threads(threads) \ shared(level, \ map, \ text, \ @@ -727,7 +727,7 @@ class BlockTreeFPParShardedSync : public BlockTree { tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -# pragma omp parallel default(none) num_threads(BT_NUM_THREADS) \ +# pragma omp parallel default(none) num_threads(threads) \ shared(level_data, \ text, \ links, \ From 10c12d2e8c792cc89965c9c2004d04c732012a4f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 19 Oct 2023 16:56:44 +0200 Subject: [PATCH 35/92] add more detailed result line output --- CMakeLists.txt | 86 ++++++------ CMakePresets.json | 4 +- examples/build_bt.cpp | 19 +-- .../block_tree_fp_par_sync_sharded.hpp | 123 +++++++++++------- .../block_tree/utils/sync_sharded_map.hpp | 24 ++-- 5 files changed, 149 insertions(+), 107 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 48e35c0..8b66ab1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -35,57 +35,65 @@ option(PASTA_BLOCK_TREE_BUILD_TESTS option(PASTA_BLOCK_TREE_BUILD_EXAMPLES "Build blocktree's benchmarks." OFF) option(PASTA_BLOCK_TREE_DEBUG - "Add instrumentation, calculating time etc." ON) + "Print debug information" OFF) +option(PASTA_BLOCK_TREE_BENCH + "Enable outputting benchmark information" OFF) include(ExternalProject) # Optional test if (PASTA_BLOCK_TREE_BUILD_TESTS) - include(FetchContent) - FetchContent_Declare( + include(FetchContent) + FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git GIT_TAG release-1.12.1 ) - set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) - FetchContent_MakeAvailable(googletest) - enable_testing() - include(GoogleTest) - add_subdirectory(tests) + set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) + FetchContent_MakeAvailable(googletest) + enable_testing() + include(GoogleTest) + add_subdirectory(tests) endif () if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) - add_executable(block_tree_construction + add_executable(block_tree_construction examples/block_tree_construction.cpp) - target_link_libraries(block_tree_construction + target_link_libraries(block_tree_construction pasta_block_tree) - add_executable(build_bt + add_executable(build_bt examples/build_bt.cpp) - target_link_libraries(build_bt + target_link_libraries(build_bt pasta_block_tree) - if (PASTA_BLOCK_TREE_DEBUG) - target_compile_definitions(build_bt PRIVATE BT_DBG) - endif () - - if (PASTA_BLOCK_TREE_MALLOC_COUNT) - message(DEBUG "malloc_count enabled") - # malloc_count - ExternalProject_Add(malloc_count - PREFIX ${CMAKE_CURRENT_BINARY_DIR} - GIT_REPOSITORY git@github.com:Skadic/malloc_count.git - BUILD_COMMAND gcc -c -fpic -ldl ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count/malloc_count.c - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - ) - - add_library(libmalloc_count INTERFACE) - target_include_directories(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) - target_link_libraries(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count-build/malloc_count.o) - - target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/extlib/malloc_count) - target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) - #target_link_libraries(build_bt libmalloc_count) - #target_compile_definitions(build_bt PRIVATE BT_MALLOC_COUNT) - endif() + if (PASTA_BLOCK_TREE_DEBUG) + target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) + target_compile_definitions(build_bt PRIVATE BT_DBG) + endif () + + if (PASTA_BLOCK_TREE_BENCH) + target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) + target_compile_definitions(build_bt PRIVATE BT_BENCH) + endif () + + if (PASTA_BLOCK_TREE_MALLOC_COUNT) + message(DEBUG "malloc_count enabled") + # malloc_count + ExternalProject_Add(malloc_count + PREFIX ${CMAKE_CURRENT_BINARY_DIR} + GIT_REPOSITORY git@github.com:Skadic/malloc_count.git + BUILD_COMMAND gcc -c -fpic -ldl ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count/malloc_count.c + CONFIGURE_COMMAND "" + INSTALL_COMMAND "" + ) + + add_library(libmalloc_count INTERFACE) + target_include_directories(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) + target_link_libraries(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count-build/malloc_count.o) + + target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/extlib/malloc_count) + target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) + #target_link_libraries(build_bt libmalloc_count) + #target_compile_definitions(build_bt PRIVATE BT_MALLOC_COUNT) + endif () endif () @@ -94,9 +102,12 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) +set(SPDLOG_USE_STD_FORMAT ON CACHE BOOL "Use std format library for spdlog" FORCE) +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/spdlog) set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort) +set_target_properties(spdlog PROPERTIES COMPILE_FLAGS "-w") add_library(waitfree-mpsc-queue ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) @@ -120,7 +131,7 @@ add_library(sdsl ${sdsl_sources}) target_include_directories(sdsl SYSTEM INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) target_include_directories(sdsl SYSTEM INTERFACE - ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) + ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) target_link_libraries(sdsl PUBLIC divsufsort64) set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") @@ -131,6 +142,7 @@ target_link_libraries(pasta_block_tree INTERFACE robin_hood waitfree-mpsc-queue sdsl + spdlog #jiffy jiffy1) target_include_directories(pasta_block_tree INTERFACE diff --git a/CMakePresets.json b/CMakePresets.json index 66f7d22..291b19e 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -62,7 +62,7 @@ { "name": "bench", "displayName": "Behcnmarking Configuration", - "description": "Release build with malloc_count", + "description": "Release build with extended debug information", "generator": "Ninja", "binaryDir": "${sourceDir}/build_bench", "inherits": "default", @@ -70,7 +70,7 @@ "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -ldl", "CMAKE_BUILD_TYPE": "Release", - "PASTA_BLOCK_TREE_MALLOC_COUNT": "ON" + "PASTA_BLOCK_TREE_BENCH": "ON" } } ], diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 25fd749..1e48f2c 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,13 +18,17 @@ * ******************************************************************************/ +#include "spdlog/sinks/basic_file_sink.h" +#include "spdlog/sinks/stdout_color_sinks.h" + #include #include #include #include #include +#include -#define LPF +#define PAR_SHARDED_SYNC #ifdef FP # include std::unique_ptr> @@ -118,7 +122,7 @@ make_bt(std::vector& text, #endif #ifdef BT_MALLOC_COUNT -#include +# include #endif #include #include @@ -183,18 +187,17 @@ int main(int argc, char** argv) { text = std::vector(input.begin(), input.end()); } + std::cout << "RESULT algo=" << ALGO_NAME + << " file=" << std::filesystem::path(argv[1]).filename().string() + << " threads=" << threads << " arity=" << arity + << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); auto bt = make_bt(text, arity, leaf_length, threads); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); - std::cout << "RESULT algo=" << ALGO_NAME - << " file=" << std::filesystem::path(argv[1]).filename().string() - << " arity=" << arity << " leaf_length=" << leaf_length - << " time=" << elapsed << " threads=" << threads - << " space=" << bt->print_space_usage(); - + std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); #ifdef BT_MALLOC_COUNT std::cout << " memory=" << malloc_count_peak(); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 781fa2f..3d80498 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -87,7 +87,7 @@ class BlockTreeFPParShardedSync : public BlockTree { using RabinKarpMap = SyncShardedMap; -#ifdef BT_DBG +#ifdef BT_INSTRUMENT public: size_t bp_hash_pairs_ns = 0; size_t bp_scan_pairs_ns = 0; @@ -330,12 +330,13 @@ class BlockTreeFPParShardedSync : public BlockTree { top_level.block_size = top_block_size; top_level.num_blocks = top_level.block_starts->size(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; - - std::cout << "using " << BT_NUM_THREADS << " threads" << std::endl; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; #endif // Construct the pre-pruned tree level by level @@ -343,19 +344,21 @@ class BlockTreeFPParShardedSync : public BlockTree { #ifdef BT_DBG std::cout << "----------------- level " << level << " -----------------" << std::endl; +#endif +#ifdef BT_INSTRUMENT TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); scan_block_pairs(text, current, is_padded, threads); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); #endif scan_blocks(text, current, is_padded, threads); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( Clock::now() - now) .count(); @@ -366,15 +369,14 @@ class BlockTreeFPParShardedSync : public BlockTree { if (level < static_cast(tree_height) - 1) { levels.push_back(std::move(generate_next_level(text, current))); } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT generate_ns += std::chrono::duration_cast( Clock::now() - now) .count(); #endif } -#ifdef BT_DBG - TimePoint now = Clock::now(); - +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) std::cout << "pairs: " << (pairs_ns / 1'000'000) << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) @@ -386,22 +388,44 @@ class BlockTreeFPParShardedSync : public BlockTree { << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + TimePoint now = Clock::now(); #endif prune(levels); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT size_t prune_ns = std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); - +# ifdef BT_DBG std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif #endif + make_tree(text, levels, padding); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT size_t make_ns = std::chrono::duration_cast(Clock::now() - now) .count(); +# ifdef BT_DBG std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif #endif } @@ -412,17 +436,16 @@ class BlockTreeFPParShardedSync : public BlockTree { return 1 + ((x - 1) / y); } - void print_aggregate(const char* name, - const tlx::Aggregate& agg, - size_t div = 1) { - printf( - "%s -> min: %10u, max: %10u, avCriscog: %10.2f, dev: %10.2f, #: %10u\n", - name, - static_cast(agg.min() / div), - static_cast(agg.max() / div), - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - static_cast(agg.count())); + [[maybe_unused]] void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); } /// @brief Scan through the blocks pairwise in order to identify which blocks @@ -451,7 +474,7 @@ class BlockTreeFPParShardedSync : public BlockTree { std::atomic_size_t threads_done = 0; std::atomic_bool last_done = false; auto& barrier = map.barrier(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT TimePoint now = Clock::now(); tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; @@ -459,7 +482,7 @@ class BlockTreeFPParShardedSync : public BlockTree { tlx::Aggregate total_idle_ns; tlx::Aggregate handle_queue_ns; -# pragma omp parallel default(none) threads(threads) \ +# pragma omp parallel default(none) num_threads(threads) \ shared(level, \ map, \ text, \ @@ -527,7 +550,7 @@ class BlockTreeFPParShardedSync : public BlockTree { #pragma omp barrier shard.handle_queue(); #pragma omp single -#ifdef BT_DBG +#ifdef BT_INSTRUMENT { bp_hash_pairs_ns += std::chrono::duration_cast(Clock::now() - @@ -551,7 +574,7 @@ class BlockTreeFPParShardedSync : public BlockTree { map, block_size, i -#ifdef BT_DBG +#ifdef BT_INSTRUMENT , thread_scan_hits #endif @@ -559,7 +582,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT auto& start_idle = shard.start_idle_ns(); auto& finish_idle = shard.finish_idle_ns(); auto& handle_queue = shard.handle_queue_ns(); @@ -574,8 +597,12 @@ class BlockTreeFPParShardedSync : public BlockTree { }; #endif } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); -#ifdef BT_DBG +# ifdef BT_DBG tlx::Aggregate map_loads; for (size_t load : map.map_loads()) { @@ -587,11 +614,8 @@ class BlockTreeFPParShardedSync : public BlockTree { print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); - bp_scan_pairs_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif #endif level.is_internal = std::make_unique(level.num_blocks); @@ -607,7 +631,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// block index. void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { const size_type num_blocks = is_internal.size(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT TimePoint now = Clock::now(); #endif // Set up the packed array holding the markings for each block. @@ -625,7 +649,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } } }); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT bp_markings_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -639,7 +663,7 @@ class BlockTreeFPParShardedSync : public BlockTree { const bool block_is_internal = markings[i] != 0b11; is_internal[i] = block_is_internal; } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT bp_bitvec_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -662,7 +686,7 @@ class BlockTreeFPParShardedSync : public BlockTree { BlockPairMap& map, const size_t num_iterations, const size_type current_block_index -#ifdef BT_DBG +#ifdef BT_INSTRUMENT , tlx::Aggregate& agg #endif @@ -673,7 +697,7 @@ class BlockTreeFPParShardedSync : public BlockTree { // pairs. auto found = map.find(current_hash); if (found == map.end()) { -#ifdef BT_DBG +#ifdef BT_INSTRUMENT agg.add(0); continue; } else { @@ -719,7 +743,7 @@ class BlockTreeFPParShardedSync : public BlockTree { // Whether the last thread is done std::atomic_bool last_done = false; auto& barrier = links.barrier(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT TimePoint now = Clock::now(); tlx::Aggregate scan_hits; tlx::Aggregate start_idle_ns; @@ -783,7 +807,7 @@ class BlockTreeFPParShardedSync : public BlockTree { #pragma omp barrier shard.handle_queue(); #pragma omp single -#ifdef BT_DBG +#ifdef BT_INSTRUMENT { b_hash_blocks_ns += @@ -813,14 +837,14 @@ class BlockTreeFPParShardedSync : public BlockTree { links, level_data, i -#ifdef BT_DBG +#ifdef BT_INSTRUMENT , thread_scan_hits #endif ); } } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT auto& start_idle = shard.start_idle_ns(); auto& finish_idle = shard.finish_idle_ns(); auto& handle_queue = shard.handle_queue_ns(); @@ -835,12 +859,13 @@ class BlockTreeFPParShardedSync : public BlockTree { }; #endif } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT b_scan_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); now = Clock::now(); +# ifdef BT_DBG tlx::Aggregate map_loads; for (size_t load : links.map_loads()) { @@ -853,6 +878,7 @@ class BlockTreeFPParShardedSync : public BlockTree { print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif #endif // By this point, the map should contain the first occurrences of @@ -876,7 +902,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } }); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT b_update_blocks_ns += std::chrono::duration_cast(Clock::now() - now) .count(); @@ -896,7 +922,7 @@ class BlockTreeFPParShardedSync : public BlockTree { BlockMap& links, LevelData& level_data, const size_type current_block_index -#ifdef BT_DBG +#ifdef BT_INSTRUMENT , tlx::Aggregate& hits #endif @@ -907,13 +933,14 @@ class BlockTreeFPParShardedSync : public BlockTree { // Find all blocks in the multimap that match our hash auto found = links.find(hash); if (found == links.end()) { -#ifdef BT_DBG +#ifdef BT_INSTRUMENT hits.add(0.0); continue; } else { hits.add(100.0); -#endif +#else continue; +#endif } BlockOccurrences& occurrences = found->second; occurrences.update(current_block_index, offset); diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 7c59450..480380a 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -175,7 +175,7 @@ class SyncShardedMap { SeqHashMap& map_; Queue& task_queue_; std::atomic_size_t& task_count_; -#ifdef BT_DBG +#ifdef BT_INSTRUMENT tlx::Aggregate start_idle_ns_; tlx::Aggregate handle_queue_ns_; tlx::Aggregate finish_idle_ns_; @@ -188,7 +188,7 @@ class SyncShardedMap { map_(sharded_map_.map_[thread_id]), task_queue_(sharded_map_.task_queue_[thread_id]), task_count_(sharded_map.task_count_[thread_id]) -#ifdef BT_DBG +#ifdef BT_INSTRUMENT , start_idle_ns_(), handle_queue_ns_(), @@ -207,14 +207,14 @@ class SyncShardedMap { K key = k; V initial = UpdateFn::init(key, std::move(in_value)); map_.emplace(key, std::move(initial)); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT sharded_map_.num_inserts_.fetch_add(1, mem::acq_rel); #endif } else { // Otherwise, update it. V& val = res->second; UpdateFn::update(k, val, std::move(in_value)); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT sharded_map_.num_updates_.fetch_add(1, mem::acq_rel); #endif } @@ -226,11 +226,11 @@ class SyncShardedMap { // when trying to insert sharded_map_.threads_handling_queue_.fetch_add(1, mem::acq_rel); } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT auto now = std::chrono::high_resolution_clock::now(); #endif sharded_map_.barrier_.arrive_and_wait(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT size_t ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); @@ -239,7 +239,7 @@ class SyncShardedMap { now = std::chrono::high_resolution_clock::now(); #endif handle_queue(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); @@ -248,7 +248,7 @@ class SyncShardedMap { now = std::chrono::high_resolution_clock::now(); #endif sharded_map_.barrier_.arrive_and_wait(); -#ifdef BT_DBG +#ifdef BT_INSTRUMENT ns_count = std::chrono::duration_cast( std::chrono::high_resolution_clock::now() - now) .count(); @@ -335,7 +335,7 @@ class SyncShardedMap { insert(StoredValue(key, value)); } -#ifdef BT_DBG +#ifdef BT_INSTRUMENT [[nodiscard]] const tlx::Aggregate& start_idle_ns() const { return start_idle_ns_; } @@ -412,13 +412,13 @@ class SyncShardedMap { return it; } - void print_map_loads() { + [[maybe_unused]] void print_map_loads() { for (size_t i = 0; i < map_.size(); ++i) { std::cout << "Map " << i << " load: " << map_[i].size() << std::endl; } } - void print_queue_loads() { + [[maybe_unused]] void print_queue_loads() { auto so = std::osyncstream(std::cout); for (size_t i = 0; i < map_.size(); ++i) { @@ -437,7 +437,7 @@ class SyncShardedMap { return loads; } - void print_ins_upd() { + [[maybe_unused]] void print_ins_upd() { std::osyncstream(std::cout) << "Inserts: " << num_inserts_.load() << "\nUpdates: " << num_updates_.load() << std::endl; From d178ae7c9dd2023fe96f430d77da762ddaa95215 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 19 Oct 2023 16:58:56 +0200 Subject: [PATCH 36/92] remove missing includes --- CMakeLists.txt | 4 ---- examples/build_bt.cpp | 4 ---- 2 files changed, 8 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 8b66ab1..2053e58 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,12 +102,9 @@ add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) -set(SPDLOG_USE_STD_FORMAT ON CACHE BOOL "Use std format library for spdlog" FORCE) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/spdlog) set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort) -set_target_properties(spdlog PROPERTIES COMPILE_FLAGS "-w") add_library(waitfree-mpsc-queue ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) @@ -142,7 +139,6 @@ target_link_libraries(pasta_block_tree INTERFACE robin_hood waitfree-mpsc-queue sdsl - spdlog #jiffy jiffy1) target_include_directories(pasta_block_tree INTERFACE diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 1e48f2c..2af3d7d 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,15 +18,11 @@ * ******************************************************************************/ -#include "spdlog/sinks/basic_file_sink.h" -#include "spdlog/sinks/stdout_color_sinks.h" - #include #include #include #include #include -#include #define PAR_SHARDED_SYNC #ifdef FP From a65ea6c061bc45e851ddd2f950a0b2e2e47ccbff Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 19 Oct 2023 17:03:38 +0200 Subject: [PATCH 37/92] make sdsl include public --- CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 2053e58..ceed1c1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -125,9 +125,9 @@ target_include_directories(pasta_block_tree INTERFACE file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp) add_library(sdsl ${sdsl_sources}) -target_include_directories(sdsl SYSTEM INTERFACE +target_include_directories(sdsl SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) -target_include_directories(sdsl SYSTEM INTERFACE + target_include_directories(sdsl SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) target_link_libraries(sdsl PUBLIC divsufsort64) set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") From d71907835db0604715963e5c82b12410f45b212f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 20 Oct 2023 15:18:57 +0200 Subject: [PATCH 38/92] possible fix for weird scaling behavior --- CMakeLists.txt | 4 +- CMakePresets.json | 4 +- examples/build_bt.cpp | 24 ++++++++--- .../block_tree_fp_par_sync_sharded.hpp | 41 ++++++++++++------- .../block_tree/utils/sync_sharded_map.hpp | 34 ++++++++++----- 5 files changed, 73 insertions(+), 34 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index ceed1c1..eac09dd 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -124,10 +124,10 @@ target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp) -add_library(sdsl ${sdsl_sources}) +add_library(sdsl STATIC ${sdsl_sources}) target_include_directories(sdsl SYSTEM PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) - target_include_directories(sdsl SYSTEM PUBLIC +target_include_directories(sdsl SYSTEM PUBLIC ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) target_link_libraries(sdsl PUBLIC divsufsort64) set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") diff --git a/CMakePresets.json b/CMakePresets.json index 291b19e..d3b5adb 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -68,8 +68,8 @@ "inherits": "default", "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", - "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -ldl", - "CMAKE_BUILD_TYPE": "Release", + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -lprofiler", + "CMAKE_BUILD_TYPE": "RelWithDebInfo", "PASTA_BLOCK_TREE_BENCH": "ON" } } diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 2af3d7d..bd4f581 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -31,6 +31,7 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, + const size_t, const size_t) { ; return std::unique_ptr>( @@ -43,6 +44,7 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, + const size_t, const size_t) { ; return std::make_unique>(text, @@ -57,7 +59,8 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t) { ; return std::unique_ptr>( pasta::make_block_tree_lpf_parallel(text, @@ -73,7 +76,8 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t) { ; return std::make_unique>( text, @@ -89,14 +93,16 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t queue_size) { ; return std::make_unique>( text, arity, 1, leaf_length, - threads); + threads, + queue_size); } # define ALGO_NAME "shard_sync" #elif defined PAR_PHMAP @@ -162,6 +168,14 @@ int main(int argc, char** argv) { const size_t threads = atoi(argv[4]); +#ifdef PAR_SHARDED_SYNC + if (argc < 6) { + std::cerr << "Please input queue size" << std::endl; + exit(1); + } + const size_t queue_size = atoi(argv[5]); +#endif + std::stringstream ss; ss << argv[1] << "_arit" << arity << "_leaf" << leaf_length << "_new.bt"; std::string out_path = ss.str(); @@ -188,7 +202,7 @@ int main(int argc, char** argv) { << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); - auto bt = make_bt(text, arity, leaf_length, threads); + auto bt = make_bt(text, arity, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 3d80498..f095fec 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -304,7 +304,9 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Constructs the block tree. /// @param text The input text. - void construct(const std::vector& text, const size_t threads) { + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -339,6 +341,10 @@ class BlockTreeFPParShardedSync : public BlockTree { std::cout << "using " << threads << " threads" << std::endl; #endif +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { #ifdef BT_DBG @@ -350,14 +356,14 @@ class BlockTreeFPParShardedSync : public BlockTree { TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); - scan_block_pairs(text, current, is_padded, threads); + scan_block_pairs(text, current, is_padded, threads, queue_size); #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); #endif - scan_blocks(text, current, is_padded, threads); + scan_blocks(text, current, is_padded, threads, queue_size); #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( Clock::now() - now) @@ -460,7 +466,8 @@ class BlockTreeFPParShardedSync : public BlockTree { void scan_block_pairs(const std::vector& text, LevelData& level, const bool is_padded, - const size_t threads) { + const size_t threads, + const size_t queue_size) { if (level.num_blocks < 4) { level.is_internal = std::make_unique(level.num_blocks, true); level.is_internal_rank = std::make_unique(*level.is_internal); @@ -469,7 +476,7 @@ class BlockTreeFPParShardedSync : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively - BlockPairMap map(threads, BT_QUEUE_CAPACITY); + BlockPairMap map(threads, queue_size); std::atomic_size_t threads_done = 0; std::atomic_bool last_done = false; @@ -519,14 +526,15 @@ class BlockTreeFPParShardedSync : public BlockTree { const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair // starting at the current block if (!level.next_is_adjacent(i)) { continue; } + rk.restart(block_starts[i]); // Move the hasher to the current block pair - RabinKarp rk(text, SIGMA, block_starts[i], pair_size, PRIME); RabinKarpHash hash = rk.current_hash(); // Try to find the hash in the map, insert a new entry if it // doesn't exist, and add the current block to the entry @@ -547,8 +555,8 @@ class BlockTreeFPParShardedSync : public BlockTree { } barrier.arrive_and_drop(); -#pragma omp barrier shard.handle_queue(); +#pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT { @@ -563,13 +571,15 @@ class BlockTreeFPParShardedSync : public BlockTree { { } #endif - if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } scan_windows_in_block_pair(rk, map, block_size, @@ -721,7 +731,8 @@ class BlockTreeFPParShardedSync : public BlockTree { void scan_blocks(const std::vector& text, LevelData& level_data, const bool is_padded, - const size_t threads) { + const size_t threads, + const size_t queue_size) { const size_t num_blocks = level_data.num_blocks; level_data.pointers = @@ -736,7 +747,7 @@ class BlockTreeFPParShardedSync : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(threads, BT_QUEUE_CAPACITY); + BlockMap links(threads, queue_size); // The number of threads finished with hashing blocks std::atomic_size_t num_done = 0; @@ -785,9 +796,10 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); // Hash each block and store their hashes in the map for (size_t i = start; i < end; ++i) { - const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); + rk.restart(block_starts[i]); RabinKarpHash hash = rk.current_hash(); shard.insert(hash, {i, 0}); } @@ -804,8 +816,8 @@ class BlockTreeFPParShardedSync : public BlockTree { shard.handle_queue_sync(false); } barrier.arrive_and_drop(); -#pragma omp barrier shard.handle_queue(); +#pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT @@ -1289,7 +1301,8 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t arity, const size_t root_arity, const size_t max_leaf_length, - const size_t threads) { + const size_t threads, + const size_t queue_size) { const auto old = omp_get_max_threads(); const auto old_dynamic = omp_get_dynamic(); omp_set_dynamic(0); @@ -1298,7 +1311,7 @@ class BlockTreeFPParShardedSync : public BlockTree { this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; this->map_unique_chars(text); - construct(text, threads); + construct(text, threads, queue_size); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 480380a..3a3320c 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -21,7 +21,6 @@ #include #include -#include #include #include #include @@ -115,8 +114,15 @@ class SyncShardedMap { /// queues. std::atomic_size_t threads_handling_queue_; +#ifdef BT_INSTRUMENT + std::atomic_size_t num_cycles_; + std::function FN = [this]() noexcept { + this->num_cycles_.fetch_add(1, mem::acq_rel); + }; +#else constexpr static std::invocable auto FN = []() noexcept { }; +#endif std::barrier barrier_; @@ -147,7 +153,7 @@ class SyncShardedMap { task_queue_(), task_count_(), threads_handling_queue_(0), - barrier_(thread_count, FN), + barrier_(static_cast(thread_count), FN), num_updates_(0), num_inserts_(0) { map_.reserve(thread_count); @@ -247,6 +253,9 @@ class SyncShardedMap { now = std::chrono::high_resolution_clock::now(); #endif + if (make_others_wait) { + sharded_map_.threads_handling_queue_.fetch_sub(1, mem::acq_rel); + } sharded_map_.barrier_.arrive_and_wait(); #ifdef BT_INSTRUMENT ns_count = std::chrono::duration_cast( @@ -254,9 +263,6 @@ class SyncShardedMap { .count(); finish_idle_ns_.add(ns_count); #endif - if (make_others_wait) { - sharded_map_.threads_handling_queue_.fetch_sub(1, mem::acq_rel); - } } /// @brief Handles this thread's queue, inserting or updating all values in @@ -293,11 +299,11 @@ class SyncShardedMap { } const size_t hash = Hasher{}(pair.first); const size_t target_thread_id = sharded_map_.mix_select(hash); - if (target_thread_id == thread_id_) { - // If the target thread is this thread, insert the value directly - insert_or_update_direct(pair.first, std::move(pair.second)); - return; - } + // if (target_thread_id == thread_id_) { + // // If the target thread is this thread, insert the value directly + // insert_or_update_direct(pair.first, std::move(pair.second)); + // return; + // } // Otherwise enqueue the new value in the target thread std::atomic_size_t& target_task_count = @@ -350,6 +356,12 @@ class SyncShardedMap { #endif }; +#ifdef BT_INSTRUMENT + [[nodiscard]] size_t num_cycles() const { + return num_cycles_.load(mem::acquire); + } +#endif + Shard get_shard(const size_t thread_id) { return Shard(*this, thread_id); } @@ -367,7 +379,7 @@ class SyncShardedMap { return size; } - Whereabouts where(const K& k) { + [[maybe_unused]] Whereabouts where(const K& k) { const size_t hash = Hasher{}(k); const size_t target_thread_id = mix_select(hash); SeqHashMap& map = map_[target_thread_id]; From 9a48a16cf93f32e7b53249f7fc133ae3be242cda Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 20 Oct 2023 17:35:23 +0200 Subject: [PATCH 39/92] =?UTF-8?q?r=20=20=2015=20=20#=20Changes=20not=20sta?= =?UTF-8?q?ged=20for=20commit:=20=20=2016=20=20#=20=20=20=20=20=20=20modif?= =?UTF-8?q?ied:=20=20=20include/pasta/block=5Ftree/utils/sync=5Fsharded=5F?= =?UTF-8?q?map.hpp=20=20=2017=20=20#=20=20=2018=20=20#=20Untracked=20files?= =?UTF-8?q?:=20=20=2019=20=20#=20=20=20=20=20=20=20.clang-format=20=20=202?= =?UTF-8?q?0=20=20#=20=20=20=20=20=20=20.vscode/=20=20=2021=20=20#=20=20?= =?UTF-8?q?=20=20=20=20=20Doxyfile=20=20=2022=20=20#=20=20=20=20=20=20=20T?= =?UTF-8?q?esting/=20=20NORMAL=20=20=EE=82=B8=E2=96=88=EF=83=B6=20COMMIT?= =?UTF-8?q?=5FEDITMSG=E2=96=881:1=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20=20?= =?UTF-8?q?=20=20=20=20=20=20=20=20=20=20=20=20=20left=5FslantGITCOMMITrig?= =?UTF-8?q?ht=5Fslant=E2=96=88Top=E2=96=88=E2=96=81=E2=96=81=20=E2=96=88?= =?UTF-8?q?=EF=83=B6=20COMMIT=5FEDITMSG=E2=96=88=20=20=20=201=20=20remove?= =?UTF-8?q?=20recursive=20call=20from=20insert=20method=20=20=20=202=20=20?= =?UTF-8?q?=20=203=20=20#=20Please=20enter=20the=20commit=20message=20for?= =?UTF-8?q?=20your=20changes.=20Lines=20starting=20=20=20=204=20=20#=20wit?= =?UTF-8?q?h=20'#'=20will=20be=20ignored,=20and=20an=20empty=20message=20a?= =?UTF-8?q?borts=20the=20commit.=20=20=20=205=20=20#=20=20=20=206=20=20#?= =?UTF-8?q?=20Date:=20=20=20=20=20=20Fri=20Oct=2020=2017:35:23=202023=20+0?= =?UTF-8?q?200=20=20=20=207=20=20#=20=20=20=208=20=20#=20On=20branch=20mai?= =?UTF-8?q?n=20=20=20=209=20=20#=20Your=20branch=20is=20ahead=20of=20'orig?= =?UTF-8?q?in/main'=20by=201=20commit.=20=20=2010=20=20#=20=20=20(use=20"g?= =?UTF-8?q?it=20push"=20to=20publish=20your=20local=20commits)=20=20=2011?= =?UTF-8?q?=20=20#=20=20=2012=20=20#=20Changes=20to=20be=20committed:=20?= =?UTF-8?q?=20=2013=20=20#=20=20=20=20=20=20=20modified:=20=20=20include/p?= =?UTF-8?q?aemove=20recursive=20call=20from=20insert=20method?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- .../pasta/block_tree/utils/sync_sharded_map.hpp | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 3a3320c..bf6bcd1 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -114,6 +114,8 @@ class SyncShardedMap { /// queues. std::atomic_size_t threads_handling_queue_; + const size_t queue_capacity_; + #ifdef BT_INSTRUMENT std::atomic_size_t num_cycles_; std::function FN = [this]() noexcept { @@ -153,6 +155,7 @@ class SyncShardedMap { task_queue_(), task_count_(), threads_handling_queue_(0), + queue_capacity_(queue_capacity), barrier_(static_cast(thread_count), FN), num_updates_(0), num_inserts_(0) { @@ -270,8 +273,9 @@ class SyncShardedMap { /// done with their handle_queue call. void handle_queue() { const size_t num_tasks_raw = task_count_.exchange(0, mem::acq_rel); - BT_ASSERT(num_tasks_raw <= task_queue_.size()); - const size_t num_tasks = std::min(num_tasks_raw, task_queue_.size()); + BT_ASSERT(num_tasks_raw <= sharded_map_.queue_capacity_); + const size_t num_tasks = + std::min(num_tasks_raw, sharded_map_.queue_capacity_); if (num_tasks == 0) { return; } @@ -312,18 +316,15 @@ class SyncShardedMap { size_t task_idx = target_task_count.fetch_add(1, mem::acq_rel); // If the target queue is full, signal to the other threads, that they // need to handle their queue and handle this thread's queue - if (task_idx >= sharded_map_.task_queue_[target_thread_id].size()) { + if (task_idx >= sharded_map_.queue_capacity_) { // Since we incremented that thread's task count, but didn't insert // anything, we need to decrement it again so that it has the correct // value target_task_count.fetch_sub(1, mem::acq_rel); handle_queue_sync(); // Since the queue was handled, the task count is now 0 - // task_idx = target_task_count.fetch_add(1, mem::seq_cst); - insert(std::move(pair)); - return; + task_idx = target_task_count.fetch_add(1, mem::acq_rel); } - BT_ASSERT(task_idx < task_queue_.size()); // Insert the value into the queue sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); } From d88c1673ca1f61fcf56f9f1de3ba3c0e36072015 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 23 Oct 2023 16:57:12 +0200 Subject: [PATCH 40/92] track setup run time --- .../block_tree_fp_par_sync_sharded.hpp | 16 +++++++++++++--- .../pasta/block_tree/utils/sync_sharded_map.hpp | 7 ++----- 2 files changed, 15 insertions(+), 8 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index f095fec..77164c2 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -36,8 +36,6 @@ #include #include -#define BT_QUEUE_CAPACITY 163840 - __extension__ typedef unsigned __int128 uint128_t; namespace pasta { @@ -307,6 +305,9 @@ class BlockTreeFPParShardedSync : public BlockTree { void construct(const std::vector& text, const size_t threads, const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif const size_type text_len = text.size(); /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -333,6 +334,15 @@ class BlockTreeFPParShardedSync : public BlockTree { top_level.num_blocks = top_level.block_starts->size(); #ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + size_t pairs_ns = 0; size_t blocks_ns = 0; size_t generate_ns = 0; @@ -407,7 +417,7 @@ class BlockTreeFPParShardedSync : public BlockTree { << " generate_next=" << (generate_ns / 1'000'000); # endif - TimePoint now = Clock::now(); + now = Clock::now(); #endif prune(levels); #ifdef BT_INSTRUMENT diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index bf6bcd1..4654dfa 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -303,11 +303,6 @@ class SyncShardedMap { } const size_t hash = Hasher{}(pair.first); const size_t target_thread_id = sharded_map_.mix_select(hash); - // if (target_thread_id == thread_id_) { - // // If the target thread is this thread, insert the value directly - // insert_or_update_direct(pair.first, std::move(pair.second)); - // return; - // } // Otherwise enqueue the new value in the target thread std::atomic_size_t& target_task_count = @@ -324,6 +319,8 @@ class SyncShardedMap { handle_queue_sync(); // Since the queue was handled, the task count is now 0 task_idx = target_task_count.fetch_add(1, mem::acq_rel); + // TODO It might be worth considering the recursive call again + // It might be the cause of some segfaults } // Insert the value into the queue sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); From 316ea3956d80f1cf4327a1c25a15a98516337e25 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 25 Oct 2023 14:52:54 +0200 Subject: [PATCH 41/92] re-add recursive call to insert --- include/pasta/block_tree/utils/MersenneHash.hpp | 2 +- include/pasta/block_tree/utils/sync_sharded_map.hpp | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index de12e96..db72e9b 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -64,7 +64,7 @@ class MersenneHash { const std::vector& text = *text_; const std::vector& other_text = *other.text_; -#define MH_PACKED_LOOP_UNROLL +#define MH_MEMCMP #ifdef MH_LOOP for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 4654dfa..da7e270 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -318,9 +318,10 @@ class SyncShardedMap { target_task_count.fetch_sub(1, mem::acq_rel); handle_queue_sync(); // Since the queue was handled, the task count is now 0 - task_idx = target_task_count.fetch_add(1, mem::acq_rel); // TODO It might be worth considering the recursive call again // It might be the cause of some segfaults + insert(std::move(pair)); + return; } // Insert the value into the queue sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); From 41ee7126000ba84822394e0bc4ff179a86297a6f Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 30 Oct 2023 18:34:58 +0100 Subject: [PATCH 42/92] parallel variant using parlayhash (WIP) --- .gitmodules | 9 + CMakeLists.txt | 2 + examples/build_bt.cpp | 48 +- extlib/parlayhash | 1 + .../construction/block_tree_fp_par_parlay.hpp | 1107 +++++++++++++++++ 5 files changed, 1163 insertions(+), 4 deletions(-) create mode 160000 extlib/parlayhash create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp diff --git a/.gitmodules b/.gitmodules index 576d401..77f8d8a 100644 --- a/.gitmodules +++ b/.gitmodules @@ -32,3 +32,12 @@ [submodule "extlib/sdsl-lite"] path = extlib/sdsl-lite url = https://github.com/Skadic/sdsl-lite +[submodule "extlib/--force"] + path = extlib/--force + url = https://github.com/rizkg/BBHash +[submodule "extlib/BBHash"] + path = extlib/BBHash + url = https://github.com/rizkg/BBHash +[submodule "extlib/parlayhash"] + path = extlib/parlayhash + url = https://github.com/cmuparlay/parlayhash diff --git a/CMakeLists.txt b/CMakeLists.txt index eac09dd..d3548b4 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -145,5 +145,7 @@ target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parallel-hashmap/parallel_hashmap) +target_include_directories(pasta_block_tree SYSTEM INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parlayhash/include) ################################################################################ diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index bd4f581..46c9b52 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,7 @@ #include #include -#define PAR_SHARDED_SYNC +#define PAR_PARLAY #ifdef FP # include std::unique_ptr> @@ -89,14 +89,14 @@ make_bt(std::vector& text, # define ALGO_NAME "shard" #elif defined PAR_SHARDED_SYNC # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -121,6 +121,38 @@ make_bt(std::vector& text, threads); } # define ALGO_NAME "par_map" +#elif defined PAR_PARLAY +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "par_parlay" +#elif defined PAR_PHF +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "par_phf" #endif #ifdef BT_MALLOC_COUNT @@ -202,7 +234,15 @@ int main(int argc, char** argv) { << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); - auto bt = make_bt(text, arity, leaf_length, threads, queue_size); + auto bt = make_bt(text, + arity, + leaf_length, + threads +#ifdef PAR_SHARDED_SYNC + , + queue_size +#endif + ); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); diff --git a/extlib/parlayhash b/extlib/parlayhash new file mode 160000 index 0000000..dda80fc --- /dev/null +++ b/extlib/parlayhash @@ -0,0 +1 @@ +Subproject commit dda80fcb90fb6f8ff5b5a7a730b202a7ceca6200 diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp new file mode 100644 index 0000000..cdc6269 --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp @@ -0,0 +1,1107 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFPParParlay : public BlockTree { + constexpr static size_type NO_EARLIER_OCC = -1; + constexpr static size_type PRUNED = -2; + + static constexpr size_type SIGMA = 256; + static constexpr uint128_t K_PRIME = 2305843009213693951ULL; + static constexpr uint8_t MERSENNE_EXPONENT = 61; + + using BitVector = pasta::BitVector; + using Rank = pasta::RankSelect; + + /// A concurrent hash map + /*template , + size_t num_submaps = 6, + typename mutex_type = phmap::NullMutex> + using HashMap = phmap::parallel_flat_hash_map< + key_type, + value_type, + hash_type, + phmap::priv::hash_default_eq, + phmap::priv::Allocator< + typename phmap::priv::Pair>, + num_submaps, + mutex_type>;*/ + + template > + using HashMap = parlay::unordered_map; + // robin_hood::unordered_node_map; + + /// A rabin karp hasher preconfigured for the current template parameters + using RabinKarp = MersenneRabinKarp; + /// A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// A hash map with rabin karp hashes as keys + template + using RabinKarpMap = + HashMap>; + +#ifdef BT_DBG +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: +#endif + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// The block from which a back block is copying + std::unique_ptr> pointers; + /// The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// The number of back blocks pointing to the block + std::unique_ptr> counters; + /// Block start indices + std::unique_ptr> block_starts; + /// The block size on this level + size_type block_size; + /// The index of the current level. First level is 0, second level is 1 etc. + size_type level_index; + /// The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + + /// @brief Checks whether a block is adjacent in the text + /// to its predecessor on this level + [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { + return (*block_starts)[i - 1] + static_cast(block_size) == + (*block_starts)[i]; + } + }; + + void construct(const std::vector& text, const size_t threads) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_DBG + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + // std::cout << "level " << level << std::endl; + +#ifdef BT_DBG + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads); +#ifdef BT_DBG + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + +#endif + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_DBG + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_DBG + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +#endif + prune(levels); +#ifdef BT_DBG + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif + make_tree(text, levels, padding); +#ifdef BT_DBG + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + inline PairOccurrences() + : first_occ_block(std::numeric_limits::max()), + occurrences() {} + + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + inline PairOccurrences(const PairOccurrences&) = default; + + inline PairOccurrences& operator=(const PairOccurrences&) = default; + + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + inline void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + static_assert(std::is_copy_assignable(), + "Must be copy-assignable for parlay"); + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap map(level.num_blocks); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded) +#endif + { + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_blocks = level.num_blocks; + const size_t num_block_pairs = num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; +#pragma omp single + for (size_t i = 0; i < num_block_pairs; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + RabinKarp rk(text, SIGMA, block_starts[i], pair_size, K_PRIME); + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + map.upsert(hash, [i](std::optional occs) { + if (!occs) { + occs.emplace(i); + } + occs->add_block(i); + occs->update(i); + return *occs; + }); + /* + std::optional occs = map.find(hash); + if (occs) { + occs->add_block(i); + occs->update(i); + } else { + PairOccurrences p(i); + map.insert(hash, p); + }*/ + } +#pragma omp barrier +#ifdef BT_DBG +# pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); + const size_t thread_id = omp_get_thread_num(); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + +#ifdef BT_DBG + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(level.num_blocks, 0); + parlay::sequence> entries = + map.entries(); + for (auto& [hash, pair_occs] : entries) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + map.remove(hash); + } +#ifdef BT_DBG + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(level.num_blocks); + + auto& is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[level.num_blocks - 1] = markings[level.num_blocks - 1] != 0b01; + for (size_type i = 0; i < level.num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_DBG + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Scan through the windows starting in a block and mark + /// them + /// accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param current_block_index The index of the block being currently hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap& map, + const size_t block_size, + const size_type current_block_index) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (!found) { + continue; + } + found->update(current_block_index); + } + } + + struct BlockOccurrences { + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + std::mutex list_mutex; + + explicit BlockOccurrences() + : first_occ({std::numeric_limits::max(), 0}), + occurrences(), + list_mutex() {} + + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences(), + list_mutex() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences), + list_mutex() {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)), + list_mutex() {} + + inline BlockOccurrences& operator=(const BlockOccurrences& other) { + first_occ = other.first_occ.load(); + occurrences = other.occurrences; + new (&list_mutex) std::mutex; + return *this; + } + + inline void add_block(size_type block_index) { + const std::lock_guard lock(list_mutex); + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + inline void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + //||(first_occ_block == block_index && block_offset < first_occ_offset)) { + } + }; + static_assert(std::is_copy_assignable(), + "Must be copy-assignable for parlay"); + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap links(num_blocks); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded) +#endif + { + const std::vector& block_starts = *level_data.block_starts; +#pragma omp single + for (size_type i = 0; i < level_data.num_blocks - is_padded; ++i) { + const RabinKarp rk(text, + SIGMA, + block_starts[i], + level_data.block_size, + K_PRIME); + const RabinKarpHash hash = rk.current_hash(); + links.upsert(hash, [i](std::optional occs) { + if (!occs) { + occs.emplace(i); + } + occs->add_block(i); + occs->update(i, 0); + return *occs; + }); + + // ptr->second.add_block(i); + // ptr->second.update(i, 0); + } +#pragma omp barrier + +#ifdef BT_DBG +# pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + + const size_t thread_id = omp_get_thread_num(); + const size_t segment_size = + ceil_div(num_total_iterations, omp_get_num_threads()); + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + SIGMA, + block_starts[start], + level_data.block_size, + K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } +#ifdef BT_DBG + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + parlay::sequence> entries = + links.entries(); + for (std::pair entry : entries) { + // The occurrences of all blocks with a given hash + const BlockOccurrences& occs = entry.second; + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + } +#ifdef BT_DBG + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (!found) { + continue; + } + found->update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParParlay(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParParlay() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } +}; // namespace pasta + +} // namespace pasta From 4fad4766117ca8aa7f912012a90a4cafe58b9897 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 1 Nov 2023 23:42:53 +0100 Subject: [PATCH 43/92] tentative parlayhash version --- CMakeLists.txt | 7 +- CMakePresets.json | 2 +- extlib/BBHash | 1 + .../construction/block_tree_fp_par_parlay.hpp | 2 +- .../construction/block_tree_fp_par_phf.hpp | 1433 +++++++++++++++++ .../construction/block_tree_fp_par_phf2.hpp | 1432 ++++++++++++++++ 6 files changed, 2874 insertions(+), 3 deletions(-) create mode 160000 extlib/BBHash create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp diff --git a/CMakeLists.txt b/CMakeLists.txt index d3548b4..0afde89 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -111,7 +111,6 @@ add_library(waitfree-mpsc-queue target_include_directories(waitfree-mpsc-queue PUBLIC ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue) - add_library(jiffy INTERFACE) target_include_directories(jiffy INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy) @@ -119,6 +118,11 @@ add_library(jiffy1 INTERFACE) target_include_directories(jiffy1 INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy-1) +add_library(bbhash STATIC ${CMAKE_CURRENT_SOURCE_DIR}/extlib/BBHash/example.cpp) +target_include_directories(bbhash SYSTEM PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/BBHash) +set_target_properties(bbhash PROPERTIES COMPILE_FLAGS "-w") + add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -140,6 +144,7 @@ target_link_libraries(pasta_block_tree INTERFACE waitfree-mpsc-queue sdsl #jiffy + bbhash jiffy1) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) diff --git a/CMakePresets.json b/CMakePresets.json index d3b5adb..15aa4a4 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -69,7 +69,7 @@ "cacheVariables": { "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -lprofiler", - "CMAKE_BUILD_TYPE": "RelWithDebInfo", + "CMAKE_BUILD_TYPE": "Release", "PASTA_BLOCK_TREE_BENCH": "ON" } } diff --git a/extlib/BBHash b/extlib/BBHash new file mode 160000 index 0000000..1803c23 --- /dev/null +++ b/extlib/BBHash @@ -0,0 +1 @@ +Subproject commit 1803c2325afab8ad045c94ef7872a319bc44a5e5 diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp index cdc6269..a8a69a7 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp @@ -566,7 +566,7 @@ class BlockTreeFPParParlay : public BlockTree { // describing a link between a (potential) back block to their source block. // In addition to the vector, there is a boolean which denotes whether a // hash has already been processed - RabinKarpMap links(num_blocks); + RabinKarpMap links(std::max(num_blocks / 20, 4)); #ifdef BT_DBG TimePoint now = Clock::now(); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp new file mode 100644 index 0000000..8ed82fd --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp @@ -0,0 +1,1433 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#include +#pragma GCC diagnostic pop + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template +class BlockTreeFPParPHF : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 61; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + robin_hood::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + SyncShardedMap; + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + size_type block_size; + /// @brief The index of the current level. First level is 0, second level is + /// 1 etc. + size_type level_index; + /// @brief The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + inline PairOccurrences() + : first_occ_block{std::numeric_limits::max()}, + occurrences{} {} + + bool operator==(const PairOccurrences& other) const { + return first_occ_block == other.first_occ_block && + occurrences == other.occurrences; + }; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + [[gnu::noinline]] void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + + void invalidate() { + first_occ_block = std::numeric_limits::max(); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + [[gnu::noinline]] inline void update(size_type block_index, + size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + struct SHash { + uint64_t operator()(const uint64_t& key, + uint64_t = 0xAAAAAAAA55555555ULL) const { + return key; + } + }; + + // using hasher_t = boomphf::SingleHashFunctor; + using hasher_t = SHash; + using phf_t = boomphf::mphf; + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + std::vector hashes(level.num_blocks - 1 - is_padded); + std::vector> values(level.num_blocks - 1 - + is_padded); + hashes.resize(level.num_blocks - 1 - is_padded, + std::numeric_limits::max()); + values.resize(level.num_blocks - 1 - is_padded, + {std::numeric_limits::max(), + std::numeric_limits::max()}); + + std::vector table; + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); + + phf_t* phf; + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout, \ + hashes, \ + values, \ + phf, \ + table) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + hashes[i] = hash.hash_; + values[i] = {hash.hash_, i}; + } +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + +#pragma omp single + { + std::sort(hashes.begin(), hashes.end()); + size_t num_distinct = 1; + { + uint64_t last = hashes[0]; + for (size_t i = 1; i < hashes.size(); ++i) { + if (hashes[i] > last) { + hashes[num_distinct++] = hashes[i]; + last = hashes[i]; + } + } + } + values.resize(num_distinct); + phf = new boomphf::mphf(num_distinct, + hashes, + 1, + 3.0, + false, + false, + 0.1); + // std::cout << "number keys for level " << level.level_index << ": " + // << phf->nbKeys() << " while num_distinct is " << + // num_distinct + // << std::endl; + table.resize(phf->nbKeys() + 1, PairOccurrences()); + + // TODO Could be parallel + for (auto& [hash, occ] : values) { + if (hash == std::numeric_limits::max()) { + continue; + } + volatile uint64_t pos = phf->lookup(hash); + if (pos >= table.size()) { + table.resize(pos + 1, PairOccurrences()); + } + PairOccurrences& occs = table[pos]; + // std::cout << "yo hash " << hash << " occ " << occ << " -> " << pos + // << std::endl; + occs.add_block_pair(occ); + occs.update(occ); + } + now = Clock::now(); + } +#pragma omp barrier + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + {table}, + *phf, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, std::span(table)); + level.is_internal_rank = std::make_unique(*level.is_internal); + delete phf; + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, + std::span map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + for (const PairOccurrences& pair_occs : map) { + if (pair_occs.first_occ_block == ~0) { + continue; + } + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + } +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + std::span map, + phf_t& phf, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + const uint64_t idx = phf.lookup(current_hash.hash_); + PairOccurrences* occurrences; + if (idx >= map.size() || + (occurrences = &map[idx])->first_occ_block == ~0) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + occurrences->update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + // Hash each block and store their hashes in the map + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParPHF(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, 1000); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParPHF() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp new file mode 100644 index 0000000..4cee5bd --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp @@ -0,0 +1,1432 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunknown-pragmas" +#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" +#include +#pragma GCC diagnostic pop + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template +class BlockTreeFPParPHF : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 61; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + robin_hood::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + SyncShardedMap; + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + size_type block_size; + /// @brief The index of the current level. First level is 0, second level is + /// 1 etc. + size_type level_index; + /// @brief The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + inline PairOccurrences() + : first_occ_block{std::numeric_limits::max()}, + occurrences{} {} + + bool operator==(const PairOccurrences& other) const { + return first_occ_block == other.first_occ_block && + occurrences == other.occurrences; + }; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + [[gnu::noinline]] void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + + void invalidate() { + first_occ_block = std::numeric_limits::max(); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + [[gnu::noinline]] inline void update(size_type block_index, + size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + struct SHash { + uint64_t operator()(const uint64_t& key, + uint64_t = 0xAAAAAAAA55555555ULL) const { + return key; + } + }; + + // using hasher_t = boomphf::SingleHashFunctor; + using hasher_t = SHash; + using phf_t = boomphf::mphf; + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + std::vector hashes(level.num_blocks - 1 - is_padded); + std::vector> values(level.num_blocks - 1 - + is_padded); + hashes.resize(level.num_blocks - 1 - is_padded, + std::numeric_limits::max()); + values.resize(level.num_blocks - 1 - is_padded, + {std::numeric_limits::max(), + std::numeric_limits::max()}); + + std::vector table; + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); + + phf_t* phf; + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout, \ + hashes, \ + values, \ + phf, \ + table) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + hashes[i] = hash.hash_; + values[i] = {hash.hash_, i}; + } +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + +#pragma omp single + { + std::sort(hashes.begin(), hashes.end()); + size_t num_distinct = 1; + { + uint64_t last = hashes[0]; + for (size_t i = 1; i < hashes.size(); ++i) { + if (hashes[i] > last) { + hashes[num_distinct++] = hashes[i]; + last = hashes[i]; + } + } + } + values.resize(num_distinct); + phf = new boomphf::mphf(num_distinct, + hashes, + 1, + 3.0, + false, + false, + 0.1); + // std::cout << "number keys for level " << level.level_index << ": " + // << phf->nbKeys() << " while num_distinct is " << + // num_distinct + // << std::endl; + table.resize(phf->nbKeys() + 1, PairOccurrences()); + + // TODO Could be parallel + for (auto& [hash, occ] : values) { + if (hash == std::numeric_limits::max()) { + continue; + } + volatile uint64_t pos = phf->lookup(hash); + if (pos >= table.size()) { + table.resize(pos + 1, PairOccurrences()); + } + PairOccurrences& occs = table[pos]; + // std::cout << "yo hash " << hash << " occ " << occ << " -> " << pos + // << std::endl; + occs.add_block_pair(occ); + occs.update(occ); + } + } +#pragma omp barrier + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + {table}, + *phf, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, std::span(table)); + level.is_internal_rank = std::make_unique(*level.is_internal); + delete phf; + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, + std::span map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + for (const PairOccurrences& pair_occs : map) { + if (pair_occs.first_occ_block == ~0) { + continue; + } + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + } +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + std::span map, + phf_t& phf, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + const uint64_t idx = phf.lookup(current_hash.hash_); + PairOccurrences* occurrences; + if (idx >= map.size() || + (occurrences = &map[idx])->first_occ_block == ~0) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + occurrences->update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + // Hash each block and store their hashes in the map + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParPHF(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, 1000); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParPHF() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +} // namespace pasta From 20499d6b0f650c8d31a34172b21b5b07a611fe97 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 8 Nov 2023 18:06:07 +0100 Subject: [PATCH 44/92] make rabin-karp hasher use 128 bit hashes --- examples/build_bt.cpp | 25 ++++-- .../block_tree_fp_par_sync_sharded.hpp | 21 +++-- .../pasta/block_tree/utils/MersenneHash.hpp | 89 +++++++++++++++++-- .../block_tree/utils/MersenneRabinKarp.hpp | 30 ++++--- 4 files changed, 134 insertions(+), 31 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 46c9b52..d53f69d 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,13 +18,16 @@ * ******************************************************************************/ +#include "pasta/block_tree/utils/MersenneHash.hpp" + #include #include #include #include #include +#include -#define PAR_PARLAY +#define PAR_SHARDED_SYNC #ifdef FP # include std::unique_ptr> @@ -89,14 +92,14 @@ make_bt(std::vector& text, # define ALGO_NAME "shard" #elif defined PAR_SHARDED_SYNC # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -255,14 +258,26 @@ int main(int argc, char** argv) { std::cout << std::endl; + #ifdef BT_INSTRUMENT + std::cout << "comparisons: " << mersenne_hash_comparisons + << ", equals: " << mersenne_hash_equals + << ", collisions: " << mersenne_hash_collisions + << ", percent equals: " + << mersenne_hash_equals / ((double)mersenne_hash_comparisons) + << ", percent collisions: " + << mersenne_hash_collisions / ((double)mersenne_hash_comparisons) + << std::endl; + #endif + // std::ofstream ot(out_path); // bt->serialize(ot); #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { - std::cerr << "Error at position " << i << "\nExpected: " << (char)text[i] - << "\nActual: " << c << std::endl; + std::osyncstream(std::cerr) + << "Error at position " << i << "\nExpected: " << (char)text[i] + << "\nActual: " << (char)c << std::endl; exit(1); } } diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 77164c2..524c38d 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -58,10 +58,11 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Base of the polynomial used for the Rabin-Karp hasher constexpr static size_type SIGMA = 256; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 61; + constexpr static uint8_t PRIME_EXPONENT = 107; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); /// @brief A bit vector using BitVector = pasta::BitVector; @@ -165,13 +166,13 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - inline void add_block_pair(size_type block_index) { + [[gnu::noinline]] inline void add_block_pair(size_type block_index) { occurrences.push_back(block_index); } /// @brief If the given block index is an earlier occurrence, update it /// @param block_index The block index of an occurrence - inline void update(size_type block_index) { + [[gnu::noinline]] void update(size_type block_index) { first_occ_block = std::min(first_occ_block, block_index); } }; @@ -218,7 +219,7 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - inline void add_block(size_type block_index) { + [[gnu::noinline]] inline void add_block(size_type block_index) { occurrences.push_back(block_index); } @@ -226,7 +227,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// update them /// @param block_index The block index of an occurrence /// @param block_index The offset of that occurrence - inline void update(size_type block_index, size_type block_offset) { + [[gnu::noinline]] inline void update(size_type block_index, + size_type block_offset) { FirstOccurrence prev_first_occ = this->first_occ.load(); FirstOccurrence set(block_index, block_offset); while (block_index < prev_first_occ.block && @@ -512,7 +514,9 @@ class BlockTreeFPParShardedSync : public BlockTree { finish_idle_ns, \ total_idle_ns, \ handle_queue_ns, \ - scan_hits) + scan_hits, \ + threads, \ + std::cout) #else # pragma omp parallel default(none) num_threads(threads) \ shared(level, map, text, is_padded, threads_done, last_done, barrier) @@ -581,6 +585,7 @@ class BlockTreeFPParShardedSync : public BlockTree { { } #endif + if (start < static_cast(num_block_pairs)) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); for (size_t i = start; i < end; ++i) { diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index db72e9b..4f51cd9 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -21,24 +21,35 @@ #pragma once +#include #include #include +#include #include #include +#include #include #include +#include namespace pasta { +#ifdef BT_INSTRUMENT +static std::atomic_size_t mersenne_hash_comparisons = 0; +static std::atomic_size_t mersenne_hash_equals = 0; +static std::atomic_size_t mersenne_hash_collisions = 0; +#endif + template class MersenneHash { public: + __extension__ typedef unsigned __int128 uint128_t; const std::vector* text_; - uint64_t hash_; + uint128_t hash_; uint32_t start_; uint32_t length_; MersenneHash(std::vector const& text, - size_t hash, + uint128_t hash, uint64_t start, uint64_t length) : text_(&text), @@ -55,16 +66,21 @@ class MersenneHash { MersenneHash& operator=(MersenneHash&& other) = default; bool operator==(const MersenneHash& other) const { - // std::cout << start_ << " " << other.start_ << std::endl; +#ifdef BT_INSTRUMENT + mersenne_hash_comparisons++; +#endif // if (length_ != other.length_) // return false; if (hash_ != other.hash_) return false; +#define MH_MEMCMP + +#ifndef MH_NOCMP const std::vector& text = *text_; const std::vector& other_text = *other.text_; +#endif -#define MH_MEMCMP #ifdef MH_LOOP for (uint64_t i = 0; i < length_; i++) { if (text[start_ + i] != other_text[other.start_ + i]) { @@ -138,9 +154,65 @@ class MersenneHash { length_ - num_blocks * 32) == 0; #elif defined MH_MEMCMP - return memcmp(text.data() + start_, - other_text.data() + other.start_, - length_) == 0; + const bool b = memcmp(text.data() + start_, + other_text.data() + other.start_, + length_) == 0; + +# ifdef BT_INSTRUMENT + if (!b) { + mersenne_hash_collisions++; + } else { + mersenne_hash_equals++; + } +# endif + return b; + +#elif defined MH_SPARSECMP + // Just compare as much data as fits into a word, choosing the largest + // integer type that fits into the length of this window + static constexpr size_t bytes_per_t = sizeof(T); + const T* text_ptr = text.data() + start_; + const T* other_text_ptr = other_text.data() + other.start_; + bool b; + switch (length_ * bytes_per_t) { + case 1: + b = text[0] == other_text[0]; + break; + case 2: + case 3: { + const uint16_t* ptr = reinterpret_cast(text_ptr); + const uint16_t* other_ptr = + reinterpret_cast(other_text_ptr); + b = *ptr == *other_ptr; + break; + } + case 4: + case 5: + case 6: + case 7: { + const uint32_t* ptr = reinterpret_cast(text_ptr); + const uint32_t* other_ptr = + reinterpret_cast(other_text_ptr); + b = *ptr == *other_ptr; + break; + } + default: { + const uint64_t* ptr = reinterpret_cast(text_ptr); + const uint64_t* other_ptr = + reinterpret_cast(other_text_ptr); + b = *ptr == *other_ptr; + } + } +# ifdef BT_INSTRUMENT + if (!b) { + mersenne_hash_collisions++; + } else { + mersenne_hash_equals++; + } +# endif + return b; +#elif defined MH_NOCMP + return true; #endif } }; @@ -150,7 +222,8 @@ class MersenneHash { namespace std { template struct hash> { - std::size_t operator()(const pasta::MersenneHash& hS) const { + pasta::MersenneHash::uint128_t + operator()(const pasta::MersenneHash& hS) const { return hS.hash_; } }; diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 1e048bf..a16a174 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -27,6 +27,17 @@ namespace pasta { +__extension__ typedef unsigned __int128 uint128_t; + +template +static constexpr uint128_t primer() { + uint128_t res = 1; + for (size_t i = 0; i < exponent; i++) { + res <<= 1; + } + return res - 1; +} + /// /// @brief A Rabin-Karp rolling hasher. /// @@ -39,8 +50,6 @@ namespace pasta { /// template class MersenneRabinKarp { - __extension__ typedef unsigned __int128 uint128_t; - public: /// The text being hashed std::vector const& text_; @@ -52,7 +61,7 @@ class MersenneRabinKarp { /// A large prime used for modulus operations uint128_t prime_; /// The current hash value - uint64_t hash_; + uint128_t hash_; uint128_t max_sigma_; /// @brief Construct a new Rabin Karp hasher. @@ -82,8 +91,8 @@ class MersenneRabinKarp { for (uint64_t i = 0; i < length_ - 1; i++) { sigma_c = mersenneModulo(sigma_c * sigma_); } - hash_ = (uint64_t)(fp); - max_sigma_ = (uint64_t)(sigma_c); + hash_ = fp; + max_sigma_ = sigma_c; }; /// @brief Moves the hasher to the specified start index in the backing @@ -98,16 +107,17 @@ class MersenneRabinKarp { fp = fp * sigma_; fp = mersenneModulo(fp + text_[i]); } - hash_ = (uint64_t)(fp); + hash_ = fp; }; inline uint128_t mersenneModulo(uint128_t k) { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - constexpr static uint128_t MERSENNE = (1ULL << mersenne_exponent) - 1; + constexpr static uint128_t MERSENNE = primer(); uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); - return (i >= MERSENNE) ? i - MERSENNE : i; + i -= (i >= MERSENNE) * MERSENNE; + return i; } }; @@ -131,13 +141,13 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { fp += prime_ * (out_char_influence > hash_) - out_char_influence; } else { - fp += ((1ULL << mersenne_exponent) - 1) * (out_char_influence > hash_) - + fp += primer() * (out_char_influence > hash_) - out_char_influence; } fp *= sigma_; fp += in_char; fp = mersenneModulo(fp); - hash_ = (uint64_t)(fp); + hash_ = fp; init_++; }; }; From 3492f53404be71a7e9c7ee1f718031ffa13b77a9 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 8 Nov 2023 18:54:01 +0100 Subject: [PATCH 45/92] use smaller prime for higher stability --- examples/build_bt.cpp | 4 ++-- .../construction/block_tree_fp_par_sync_sharded.hpp | 5 ++++- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index d53f69d..86761a2 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -258,7 +258,7 @@ int main(int argc, char** argv) { std::cout << std::endl; - #ifdef BT_INSTRUMENT +#ifdef BT_INSTRUMENT std::cout << "comparisons: " << mersenne_hash_comparisons << ", equals: " << mersenne_hash_equals << ", collisions: " << mersenne_hash_collisions @@ -267,7 +267,7 @@ int main(int argc, char** argv) { << ", percent collisions: " << mersenne_hash_collisions / ((double)mersenne_hash_comparisons) << std::endl; - #endif +#endif // std::ofstream ot(out_path); // bt->serialize(ot); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 524c38d..a2f3f6f 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -60,7 +60,9 @@ class BlockTreeFPParShardedSync : public BlockTree { constexpr static size_type SIGMA = 256; /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; + //constexpr static uint8_t PRIME_EXPONENT = 107; + constexpr static uint8_t PRIME_EXPONENT = 89; + //constexpr static uint8_t PRIME_EXPONENT = 61; /// @brief A mersenne prime used for the Rabin-Karp hasher constexpr static uint128_t PRIME = pasta::primer(); @@ -73,6 +75,7 @@ class BlockTreeFPParShardedSync : public BlockTree { template using SeqHashMap = robin_hood::unordered_map>; + //std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters From c3b310ac5204184949f58b5ce6bdcbb78c8a1cfe Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 9 Nov 2023 16:00:58 +0100 Subject: [PATCH 46/92] add heap memory tracking --- examples/build_bt.cpp | 8 ++- examples/memphis.hpp | 118 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 124 insertions(+), 2 deletions(-) create mode 100644 examples/memphis.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 86761a2..841a737 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,6 +18,10 @@ * ******************************************************************************/ +// #ifdef BT_INSTRUMENT +#include "memphis.hpp" +// #endif + #include "pasta/block_tree/utils/MersenneHash.hpp" #include @@ -252,8 +256,8 @@ int main(int argc, char** argv) { std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); -#ifdef BT_MALLOC_COUNT - std::cout << " memory=" << malloc_count_peak(); +#ifdef MEMPHIS_ENABLED + std::cout << " memory=" << memphis::peak_heap_usage; #endif std::cout << std::endl; diff --git a/examples/memphis.hpp b/examples/memphis.hpp new file mode 100644 index 0000000..2543a5e --- /dev/null +++ b/examples/memphis.hpp @@ -0,0 +1,118 @@ +// Be very careful when including this header! It will overwrite memory allocation operators! +// Only include it, if you want to track your heap memory. +// Source: https://en.cppreference.com/w/cpp/memory/new/operator_new + +#pragma once +#pragma GCC system_header + +#include +#include +#include +#include +#include + +#ifndef MEMPHIS_ENABLED +# define MEMPHIS_ENABLED +#endif // !MEMPHIS_ENABLED + +namespace memphis { +/// @brief The current heap usage in bytes. +std::atomic_size_t current_heap_usage = 0; +/// @brief The number of heap allocations since the start of the program. +std::atomic_size_t num_allocations = 0; +/// @brief The maximum heap usage over the course of the program's runtime in +/// bytes. +std::atomic_size_t peak_heap_usage = 0; + +/// +/// @brief Retrieves the size of the allocation of the given pointer in bytes. +/// +/// @param ptr A pointer to some allocation. +/// @return The size of the allocation in bytes. +/// +[[nodiscard("allocation size determined but discarded")]] inline size_t +allocation_size(const void* ptr); +} // namespace memphis + +#ifdef _WIN32 +# include +inline size_t memphis::allocation_size(const void* ptr) { + return _msize(const_cast(ptr)); +} +#elif defined unix +# include +inline size_t memphis::allocation_size(const void* ptr) { + return malloc_usable_size(const_cast(ptr)); +} +#elif defined __APPLE__ +# include +inline size_t memphis::allocation_size(const void* ptr) { + return malloc_size(ptr); +} + +#endif + +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wmismatched-new-delete" +// no inline, required by [replacement.functions]/3 +void* operator new(std::size_t sz) { + if (sz == 0) + ++sz; // avoid std::malloc(0) which may return nullptr on success + + if (void* ptr = std::malloc(sz)) { + const std::size_t current = + memphis::current_heap_usage.fetch_add(sz, std::memory_order_relaxed) + + sz; + const std::size_t old_peak = + memphis::peak_heap_usage.load(std::memory_order_acquire); + memphis::peak_heap_usage.store(std::max(current, old_peak), + std::memory_order_release); + memphis::num_allocations.fetch_add(1, std::memory_order_relaxed); + return ptr; + } + + throw std::bad_alloc{}; // required by [new.delete.single]/3 +} + +// no inline, required by [replacement.functions]/3 +void* operator new[](std::size_t sz) { + if (sz == 0) + ++sz; // avoid std::malloc(0) which may return nullptr on success + + if (void* ptr = std::malloc(sz)) { + const std::size_t current = + memphis::current_heap_usage.fetch_add(sz, std::memory_order_relaxed) + + sz; + const std::size_t old_peak = + memphis::peak_heap_usage.load(std::memory_order_acquire); + memphis::peak_heap_usage.store(std::max(current, old_peak), + std::memory_order_release); + memphis::num_allocations.fetch_add(1, std::memory_order_relaxed); + return ptr; + } + + throw std::bad_alloc{}; // required by [new.delete.single]/3 +} + +void operator delete(void* ptr) noexcept { + std::size_t sz = memphis::allocation_size(ptr); + memphis::current_heap_usage.fetch_sub(sz, std::memory_order_relaxed); + std::free(ptr); +} + +void operator delete(void* ptr, std::size_t size) noexcept { + memphis::current_heap_usage.fetch_sub(size, std::memory_order_relaxed); + std::free(ptr); +} + +void operator delete[](void* ptr) noexcept { + std::size_t sz = memphis::allocation_size(ptr); + memphis::current_heap_usage.fetch_sub(sz, std::memory_order_relaxed); + std::free(ptr); +} + +void operator delete[](void* ptr, std::size_t size) noexcept { + memphis::current_heap_usage.fetch_sub(size, std::memory_order_relaxed); + std::free(ptr); +} +#pragma GCC diagnostic pop From ea208dfaa635054159f9dd19330624cad8cd12ab Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 9 Nov 2023 19:26:11 +0100 Subject: [PATCH 47/92] remove faulty memory tracking --- examples/build_bt.cpp | 8 -------- 1 file changed, 8 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 841a737..7d5b9af 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,10 +18,6 @@ * ******************************************************************************/ -// #ifdef BT_INSTRUMENT -#include "memphis.hpp" -// #endif - #include "pasta/block_tree/utils/MersenneHash.hpp" #include @@ -256,10 +252,6 @@ int main(int argc, char** argv) { std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); -#ifdef MEMPHIS_ENABLED - std::cout << " memory=" << memphis::peak_heap_usage; -#endif - std::cout << std::endl; #ifdef BT_INSTRUMENT From b893612f7c875f76cc7e80d57ebed3b3c6b99750 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 9 Nov 2023 19:26:24 +0100 Subject: [PATCH 48/92] remove faulty memory tracking --- examples/memphis.hpp | 118 ------------------------------------------- 1 file changed, 118 deletions(-) delete mode 100644 examples/memphis.hpp diff --git a/examples/memphis.hpp b/examples/memphis.hpp deleted file mode 100644 index 2543a5e..0000000 --- a/examples/memphis.hpp +++ /dev/null @@ -1,118 +0,0 @@ -// Be very careful when including this header! It will overwrite memory allocation operators! -// Only include it, if you want to track your heap memory. -// Source: https://en.cppreference.com/w/cpp/memory/new/operator_new - -#pragma once -#pragma GCC system_header - -#include -#include -#include -#include -#include - -#ifndef MEMPHIS_ENABLED -# define MEMPHIS_ENABLED -#endif // !MEMPHIS_ENABLED - -namespace memphis { -/// @brief The current heap usage in bytes. -std::atomic_size_t current_heap_usage = 0; -/// @brief The number of heap allocations since the start of the program. -std::atomic_size_t num_allocations = 0; -/// @brief The maximum heap usage over the course of the program's runtime in -/// bytes. -std::atomic_size_t peak_heap_usage = 0; - -/// -/// @brief Retrieves the size of the allocation of the given pointer in bytes. -/// -/// @param ptr A pointer to some allocation. -/// @return The size of the allocation in bytes. -/// -[[nodiscard("allocation size determined but discarded")]] inline size_t -allocation_size(const void* ptr); -} // namespace memphis - -#ifdef _WIN32 -# include -inline size_t memphis::allocation_size(const void* ptr) { - return _msize(const_cast(ptr)); -} -#elif defined unix -# include -inline size_t memphis::allocation_size(const void* ptr) { - return malloc_usable_size(const_cast(ptr)); -} -#elif defined __APPLE__ -# include -inline size_t memphis::allocation_size(const void* ptr) { - return malloc_size(ptr); -} - -#endif - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wmismatched-new-delete" -// no inline, required by [replacement.functions]/3 -void* operator new(std::size_t sz) { - if (sz == 0) - ++sz; // avoid std::malloc(0) which may return nullptr on success - - if (void* ptr = std::malloc(sz)) { - const std::size_t current = - memphis::current_heap_usage.fetch_add(sz, std::memory_order_relaxed) + - sz; - const std::size_t old_peak = - memphis::peak_heap_usage.load(std::memory_order_acquire); - memphis::peak_heap_usage.store(std::max(current, old_peak), - std::memory_order_release); - memphis::num_allocations.fetch_add(1, std::memory_order_relaxed); - return ptr; - } - - throw std::bad_alloc{}; // required by [new.delete.single]/3 -} - -// no inline, required by [replacement.functions]/3 -void* operator new[](std::size_t sz) { - if (sz == 0) - ++sz; // avoid std::malloc(0) which may return nullptr on success - - if (void* ptr = std::malloc(sz)) { - const std::size_t current = - memphis::current_heap_usage.fetch_add(sz, std::memory_order_relaxed) + - sz; - const std::size_t old_peak = - memphis::peak_heap_usage.load(std::memory_order_acquire); - memphis::peak_heap_usage.store(std::max(current, old_peak), - std::memory_order_release); - memphis::num_allocations.fetch_add(1, std::memory_order_relaxed); - return ptr; - } - - throw std::bad_alloc{}; // required by [new.delete.single]/3 -} - -void operator delete(void* ptr) noexcept { - std::size_t sz = memphis::allocation_size(ptr); - memphis::current_heap_usage.fetch_sub(sz, std::memory_order_relaxed); - std::free(ptr); -} - -void operator delete(void* ptr, std::size_t size) noexcept { - memphis::current_heap_usage.fetch_sub(size, std::memory_order_relaxed); - std::free(ptr); -} - -void operator delete[](void* ptr) noexcept { - std::size_t sz = memphis::allocation_size(ptr); - memphis::current_heap_usage.fetch_sub(sz, std::memory_order_relaxed); - std::free(ptr); -} - -void operator delete[](void* ptr, std::size_t size) noexcept { - memphis::current_heap_usage.fetch_sub(size, std::memory_order_relaxed); - std::free(ptr); -} -#pragma GCC diagnostic pop From 7384e38d8e2069880d2473e6fa58bf8b24e9bbc2 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 13 Nov 2023 13:30:12 +0100 Subject: [PATCH 49/92] use lower prime in hopes that fixes overflow errors --- examples/build_bt.cpp | 4 +- .../block_tree_fp_par_sync_sharded.hpp | 10 +- .../pasta/block_tree/utils/MersenneHash.hpp | 146 ++---------------- .../block_tree/utils/sync_sharded_map.hpp | 2 - 4 files changed, 18 insertions(+), 144 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 7d5b9af..4d2d2e8 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -259,9 +259,9 @@ int main(int argc, char** argv) { << ", equals: " << mersenne_hash_equals << ", collisions: " << mersenne_hash_collisions << ", percent equals: " - << mersenne_hash_equals / ((double)mersenne_hash_comparisons) + << 100 * mersenne_hash_equals / ((double)mersenne_hash_comparisons) << ", percent collisions: " - << mersenne_hash_collisions / ((double)mersenne_hash_comparisons) + << 100 * mersenne_hash_collisions / ((double)mersenne_hash_comparisons) << std::endl; #endif diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index a2f3f6f..42157fa 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -60,11 +60,13 @@ class BlockTreeFPParShardedSync : public BlockTree { constexpr static size_type SIGMA = 256; /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - //constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 107; constexpr static uint8_t PRIME_EXPONENT = 89; - //constexpr static uint8_t PRIME_EXPONENT = 61; + // constexpr static uint8_t PRIME_EXPONENT = 61; /// @brief A mersenne prime used for the Rabin-Karp hasher constexpr static uint128_t PRIME = pasta::primer(); + // constexpr static uint128_t PRIME = (static_cast(0x97009E545BB) + // << (14 * 4)) | static_cast(0x2DA8B4A8C9A82B); /// @brief A bit vector using BitVector = pasta::BitVector; @@ -74,8 +76,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_map>; - //std::unordered_map>; + robin_hood::unordered_node_map>; + // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 4f51cd9..50c8534 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -25,12 +25,12 @@ #include #include #include +#include #include #include #include #include #include -#include namespace pasta { @@ -74,147 +74,21 @@ class MersenneHash { if (hash_ != other.hash_) return false; -#define MH_MEMCMP - -#ifndef MH_NOCMP - const std::vector& text = *text_; - const std::vector& other_text = *other.text_; -#endif - -#ifdef MH_LOOP - for (uint64_t i = 0; i < length_; i++) { - if (text[start_ + i] != other_text[other.start_ + i]) { - return false; - } - } - return true; -#elif defined MH_PACKED_LOOP - const size_t num_blocks = length_ / 8; - for (size_t block = 0; block < num_blocks; ++block) { - uint64_t a = - *reinterpret_cast(text.data() + start_ + block * 8); - uint64_t b = *reinterpret_cast(other_text.data() + - other.start_ + block * 8); - if (a != b) { - return false; - } - } - return memcmp(text_->data() + start_ + num_blocks * 8, - other.text_->data() + other.start_ + num_blocks * 8, - length_ - num_blocks * 8) == 0; -#elif defined MH_PACKED_LOOP_UNROLL - const size_t num_blocks = length_ / 16; - for (size_t block = 0; block < num_blocks; ++block) { - uint64_t a1 = *reinterpret_cast(text.data() + start_ + - 2 * block * 8); - uint64_t a2 = *reinterpret_cast(text.data() + start_ + - (2 * block + 1) * 8); - uint64_t b1 = *reinterpret_cast( - other_text.data() + other.start_ + 2 * block * 8); - uint64_t b2 = *reinterpret_cast( - other_text.data() + other.start_ + (2 * block + 1) * 8); - if (a1 != b1 || a2 != b2) { - return false; - } - } - return memcmp(text_->data() + start_ + num_blocks * 16, - other.text_->data() + other.start_ + num_blocks * 16, - length_ - num_blocks * 16) == 0; -#elif defined MH_SSE - // Requires SSE2 - const size_t num_blocks = length_ / 16; - for (size_t block = 0; block < num_blocks; ++block) { - __m128i_u a = _mm_loadu_si128(reinterpret_cast( - text.data() + start_ + block * 16)); - __m128i_u b = _mm_loadu_si128(reinterpret_cast( - other_text.data() + other.start_ + block * 16)); - __m128i_u res = _mm_cmpeq_epi8(a, b); - if (0xFFFF != _mm_movemask_epi8(res)) { - return false; - } - } - return memcmp(text_->data() + start_ + num_blocks * 16, - other.text_->data() + other.start_ + num_blocks * 16, - length_ - num_blocks * 16) == 0; -#elif defined MH_AVX - // Requires AVX-2 - const size_t num_blocks = length_ / 32; - for (size_t block = 0; block < num_blocks; ++block) { - __m256i a = _mm256_loadu_si256( - reinterpret_cast(text.data() + start_ + block * 32)); - __m256i b = _mm256_loadu_si256(reinterpret_cast( - other_text.data() + other.start_ + block * 32)); - __m256i res = _mm256_cmpeq_epi8(a, b); - if (static_cast(0xFFFFFFFF) != _mm256_movemask_epi8(res)) { - return false; - } - } - return memcmp(text_->data() + start_ + num_blocks * 32, - other.text_->data() + other.start_ + num_blocks * 32, - length_ - num_blocks * 32) == 0; + const bool is_same = memcmp(text_->data() + start_, + other.text_->data() + other.start_, + length_) == 0; -#elif defined MH_MEMCMP - const bool b = memcmp(text.data() + start_, - other_text.data() + other.start_, - length_) == 0; - -# ifdef BT_INSTRUMENT - if (!b) { - mersenne_hash_collisions++; - } else { - mersenne_hash_equals++; - } -# endif - return b; - -#elif defined MH_SPARSECMP - // Just compare as much data as fits into a word, choosing the largest - // integer type that fits into the length of this window - static constexpr size_t bytes_per_t = sizeof(T); - const T* text_ptr = text.data() + start_; - const T* other_text_ptr = other_text.data() + other.start_; - bool b; - switch (length_ * bytes_per_t) { - case 1: - b = text[0] == other_text[0]; - break; - case 2: - case 3: { - const uint16_t* ptr = reinterpret_cast(text_ptr); - const uint16_t* other_ptr = - reinterpret_cast(other_text_ptr); - b = *ptr == *other_ptr; - break; - } - case 4: - case 5: - case 6: - case 7: { - const uint32_t* ptr = reinterpret_cast(text_ptr); - const uint32_t* other_ptr = - reinterpret_cast(other_text_ptr); - b = *ptr == *other_ptr; - break; - } - default: { - const uint64_t* ptr = reinterpret_cast(text_ptr); - const uint64_t* other_ptr = - reinterpret_cast(other_text_ptr); - b = *ptr == *other_ptr; - } - } -# ifdef BT_INSTRUMENT - if (!b) { +#ifdef BT_INSTRUMENT + if (!is_same) { + // The hash is the same but the substring isn't => collision mersenne_hash_collisions++; } else { + // The substrings are the same mersenne_hash_equals++; } -# endif - return b; -#elif defined MH_NOCMP - return true; #endif - } + return is_same; + }; }; } // namespace pasta diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index da7e270..4e52ca4 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -280,8 +280,6 @@ class SyncShardedMap { return; } - static unsigned char zeroed[sizeof(StoredValue)]; - memset(&zeroed, 0, sizeof(StoredValue)); // Handle all tasks in the queue for (size_t i = 0; i < num_tasks; ++i) { auto& entry = task_queue_[i]; From e1bf09cc6a1734661dd4ddfff8eea54639b0a2db Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 13 Nov 2023 19:15:27 +0100 Subject: [PATCH 50/92] variant that uses an identity hash for small blocks --- examples/build_bt.cpp | 46 +- .../block_tree_fp_par_sync_sharded_small.hpp | 1581 +++++++++++++++++ 2 files changed, 1623 insertions(+), 4 deletions(-) create mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 4d2d2e8..898b1f4 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -27,7 +27,7 @@ #include #include -#define PAR_SHARDED_SYNC +#define PAR_SHARDED_SYNC_SMALL #ifdef FP # include std::unique_ptr> @@ -108,6 +108,23 @@ make_bt(std::vector& text, queue_size); } # define ALGO_NAME "shard_sync" +#elif defined PAR_SHARDED_SYNC_SMALL +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + return std::make_unique< + pasta::BlockTreeFPParShardedSyncSmall>(text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "shard_sync_small" #elif defined PAR_PHMAP # include std::unique_ptr> @@ -161,6 +178,24 @@ make_bt(std::vector& text, #ifdef BT_MALLOC_COUNT # include #endif + +#if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL +# define USES_QUEUE true +# define IS_PARALLEL true +#else +# define USES_QUEUE false +#endif + +#ifndef IS_PARALLEL +# if defined PAR_SHARDED_SYNC_SMALL || defined PAR_SHARDED_SYNC || \ + defined PAR_SHARDED || defined PAR_PHMAP || defined LPF || \ + defined PAR_PHMAP +# define IS_PARALLEL true +# else +# define IS_PARALLEL false +# endif +#endif + #include #include @@ -194,6 +229,7 @@ int main(int argc, char** argv) { const size_t leaf_length = atoi(argv[3]); +#if defined IS_PARALLEL if (argc < 5) { std::cerr << "Please input number of threads (ignored if single threaded " "algorithm)" @@ -202,8 +238,9 @@ int main(int argc, char** argv) { } const size_t threads = atoi(argv[4]); +#endif -#ifdef PAR_SHARDED_SYNC +#if defined USES_QUEUE if (argc < 6) { std::cerr << "Please input queue size" << std::endl; exit(1); @@ -241,7 +278,7 @@ int main(int argc, char** argv) { arity, leaf_length, threads -#ifdef PAR_SHARDED_SYNC +#if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL , queue_size #endif @@ -261,7 +298,8 @@ int main(int argc, char** argv) { << ", percent equals: " << 100 * mersenne_hash_equals / ((double)mersenne_hash_comparisons) << ", percent collisions: " - << 100 * mersenne_hash_collisions / ((double)mersenne_hash_comparisons) + << 100 * mersenne_hash_collisions / + ((double)mersenne_hash_comparisons) << std::endl; #endif diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp new file mode 100644 index 0000000..5ab0eeb --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp @@ -0,0 +1,1581 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template +class BlockTreeFPParShardedSyncSmall : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief For some block size (in bytes) i, return the number of trailing + /// zeros in a 64 bit integer when zeroing out characters that are not part + /// of the block. + constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + + /// @brief Masks for identity hashes for a block size i (in bytes) + constexpr static uint64_t HASH_MASKS[9] = { + 0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + // constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); + // constexpr static uint128_t PRIME = (static_cast(0x97009E545BB) + // << (14 * 4)) | static_cast(0x2DA8B4A8C9A82B); + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + robin_hood::unordered_flat_map>; + // std::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + inline LevelData(int64_t level_index_, + int64_t block_size_, + int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + inline PairOccurrences(PairOccurrences&&) = default; + inline PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + [[gnu::noinline]] void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + [[gnu::noinline]] inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + [[gnu::noinline]] inline void update(size_type block_index, + size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size) > + 8 / sizeof(input_type)) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size) > 8 / sizeof(input_type)) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t TRAILING_ZEROS = + MASK_TRAILING_ZEROS[pair_size * sizeof(input_type)]; + const uint64_t HASH_MASK = HASH_MASKS[pair_size * sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr) & + HASH_MASK) >> + TRAILING_ZEROS; + RabinKarpHash hash(text, hash_value, block_start, block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::vector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t TRAILING_ZEROS = + MASK_TRAILING_ZEROS[pair_size / sizeof(input_type)]; + const uint64_t HASH_MASK = HASH_MASKS[pair_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr + offset) & + HASH_MASK) >> + TRAILING_ZEROS; + RabinKarpHash current_hash(text, + hash_value, + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t TRAILING_ZEROS = + sizeof(input_type) * 8 * (8 - block_size); + const uint64_t HASH_MASK = static_cast(~0) << TRAILING_ZEROS; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr) & + HASH_MASK) >> + TRAILING_ZEROS; + RabinKarpHash hash(text, hash_value, block_start, block_size); + + shard.insert(hash, {i, 0}); + } + } + const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::vector& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t TRAILING_ZEROS = + MASK_TRAILING_ZEROS[level_data.block_size / sizeof(input_type)]; + const uint64_t HASH_MASK = + HASH_MASKS[level_data.block_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr + offset) & + HASH_MASK) >> + TRAILING_ZEROS; + RabinKarpHash hash(text, + hash_value, + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParShardedSyncSmall(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParShardedSyncSmall() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +} // namespace pasta From 2c1c1456ab75ced5da1a39a71a5c509bf93f0c21 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 14 Nov 2023 17:46:42 +0100 Subject: [PATCH 51/92] sharded sync using identity hash for small substrings --- .gitmodules | 3 + CMakeLists.txt | 3 + extlib/unordered_dense | 1 + .../block_tree_fp_par_sync_sharded.hpp | 4 +- .../block_tree_fp_par_sync_sharded_small.hpp | 173 ++++++++++-------- .../pasta/block_tree/utils/MersenneHash.hpp | 24 +-- .../block_tree/utils/sync_sharded_map.hpp | 24 +-- 7 files changed, 132 insertions(+), 100 deletions(-) create mode 160000 extlib/unordered_dense diff --git a/.gitmodules b/.gitmodules index 77f8d8a..3a9ade8 100644 --- a/.gitmodules +++ b/.gitmodules @@ -41,3 +41,6 @@ [submodule "extlib/parlayhash"] path = extlib/parlayhash url = https://github.com/cmuparlay/parlayhash +[submodule "extlib/unordered_dense"] + path = extlib/unordered_dense + url = https://github.com/martinus/unordered_dense diff --git a/CMakeLists.txt b/CMakeLists.txt index 0afde89..2bb1cf8 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,6 +72,7 @@ if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) if (PASTA_BLOCK_TREE_BENCH) target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) target_compile_definitions(build_bt PRIVATE BT_BENCH) + #target_compile_definitions(build_bt PRIVATE ROBIN_HOOD_LOG_ENABLED) endif () if (PASTA_BLOCK_TREE_MALLOC_COUNT) @@ -152,5 +153,7 @@ target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parallel-hashmap/parallel_hashmap) target_include_directories(pasta_block_tree SYSTEM INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parlayhash/include) +target_include_directories(pasta_block_tree SYSTEM INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/unordered_dense/include) ################################################################################ diff --git a/extlib/unordered_dense b/extlib/unordered_dense new file mode 160000 index 0000000..729896c --- /dev/null +++ b/extlib/unordered_dense @@ -0,0 +1 @@ +Subproject commit 729896c7ba8bbd9da5573679270133086d05b5dd diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 42157fa..21a62c8 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -61,8 +61,8 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher // constexpr static uint8_t PRIME_EXPONENT = 107; - constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; + // constexpr static uint8_t PRIME_EXPONENT = 89; + constexpr static uint8_t PRIME_EXPONENT = 61; /// @brief A mersenne prime used for the Rabin-Karp hasher constexpr static uint128_t PRIME = pasta::primer(); // constexpr static uint128_t PRIME = (static_cast(0x97009E545BB) diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp index 5ab0eeb..31b5e38 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp @@ -26,6 +26,7 @@ #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" +#include #include #include #include @@ -54,7 +55,6 @@ enum class UseHash { /// blocks themselves. /// @tparam input_type The type of the characters in the input string /// @tparam size_type The type used for indices etc. (must be a signed integer) -/// @tparam queue_type The type of queue to use for communication /// in the sharded hash map. template class BlockTreeFPParShardedSyncSmall : public BlockTree { @@ -67,17 +67,33 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { constexpr static uint64_t MASK_TRAILING_ZEROS[9] = {64, 56, 48, 40, 32, 24, 16, 8, 0}; + /// @brief Masks used for the identity hash. These depend on endianness + constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } + } + /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static uint64_t HASH_MASKS[9] = { - 0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + constexpr static std::array HASH_MASKS = masks(); /// @brief A marker for a block that has no earlier occurrence constexpr static size_type NO_EARLIER_OCC = -1; @@ -88,13 +104,11 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { constexpr static size_type SIGMA = 256; /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - // constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - constexpr static uint8_t PRIME_EXPONENT = 61; + constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + // constexpr static uint8_t PRIME_EXPONENT = 61; /// @brief A mersenne prime used for the Rabin-Karp hasher constexpr static uint128_t PRIME = pasta::primer(); - // constexpr static uint128_t PRIME = (static_cast(0x97009E545BB) - // << (14 * 4)) | static_cast(0x2DA8B4A8C9A82B); /// @brief A bit vector using BitVector = pasta::BitVector; @@ -104,7 +118,9 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_flat_map>; + ankerl::unordered_dense::map>; + // robin_hood::unordered_flat_map>; // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template @@ -120,6 +136,18 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { using RabinKarpMap = SyncShardedMap; +#define MIX + static uint64_t mix_select(uint64_t key) { +#ifdef MIX + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); +#endif + return key; + } + #ifdef BT_INSTRUMENT public: size_t bp_hash_pairs_ns = 0; @@ -156,9 +184,9 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief The number of blocks on the current level int64_t num_blocks; - inline LevelData(int64_t level_index_, - int64_t block_size_, - int64_t num_blocks_) + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) : is_internal(nullptr), is_internal_rank(nullptr), pointers(new std::vector()), @@ -171,7 +199,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief Checks whether a block is adjacent in the text /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + [[nodiscard]] bool next_is_adjacent(size_t i) const { return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; } }; @@ -197,18 +225,18 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { : first_occ_block(first_occ_block_), occurrences() {} - inline PairOccurrences(PairOccurrences&&) = default; - inline PairOccurrences& operator=(PairOccurrences&&) = default; + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block_pair(size_type block_index) { + void add_block_pair(size_type block_index) { occurrences.push_back(block_index); } /// @brief If the given block index is an earlier occurrence, update it /// @param block_index The block index of an occurrence - [[gnu::noinline]] void update(size_type block_index) { + void update(size_type block_index) { first_occ_block = std::min(first_occ_block, block_index); } }; @@ -253,6 +281,8 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { : first_occ(other.first_occ.load()), occurrences(std::move(other.occurrences)) {} + ~BlockOccurrences() = default; + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { first_occ = other.first_occ.load(); occurrences = std::move(other.occurrences); @@ -261,16 +291,15 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block(size_type block_index) { + void add_block(size_type block_index) { occurrences.push_back(block_index); } /// @brief If the given block index and offset are an earlier occurrence, /// update them /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - [[gnu::noinline]] inline void update(size_type block_index, - size_type block_offset) { + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { FirstOccurrence prev_first_occ = this->first_occ.load(); FirstOccurrence set(block_index, block_offset); while (block_index < prev_first_occ.block && @@ -346,6 +375,9 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief Constructs the block tree. /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map void construct(const std::vector& text, const size_t threads, const size_t queue_size) { @@ -410,8 +442,8 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); - if (2 * static_cast(current.block_size) > - 8 / sizeof(input_type)) { + if (2 * static_cast(current.block_size * sizeof(input_type)) > + 8) { scan_block_pairs(text, current, is_padded, @@ -430,7 +462,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { .count(); now = Clock::now(); #endif - if (static_cast(current.block_size) > 8 / sizeof(input_type)) { + if (static_cast(current.block_size * sizeof(input_type)) > 8) { scan_blocks(text, current, is_padded, @@ -521,9 +553,10 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { return 1 + ((x - 1) / y); } - [[maybe_unused]] void print_aggregate(const char* name, - const tlx::Aggregate& agg, - size_t div = 1) { + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", name, static_cast(agg.min() / div), @@ -540,6 +573,9 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @param level The data for the current level. /// @param is_padded `true` iff the last block on this level *does not* end at /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or /// use the blocks' contents themselves as hashes. /// For block sizes greater than 4 bytes, use Rabin-Karp. @@ -602,7 +638,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { // Hash every window and determine for all block pairs whether // they have previous occurrences. - size_t segment_size = + const size_t segment_size = std::max(1, ceil_div(num_block_pairs, num_threads)); // Start and end index of the current thread's segment @@ -626,28 +662,25 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { shard.insert(hash, i); } } else { - const uint64_t TRAILING_ZEROS = - MASK_TRAILING_ZEROS[pair_size * sizeof(input_type)]; const uint64_t HASH_MASK = HASH_MASKS[pair_size * sizeof(input_type)]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & - HASH_MASK) >> - TRAILING_ZEROS; - RabinKarpHash hash(text, hash_value, block_start, block_size); + (*reinterpret_cast(block_start_ptr) & HASH_MASK); + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); // Try to find the hash in the map, insert a new entry if it // doesn't exist, and add the current block to the entry shard.insert(hash, i); } } - const size_t thread_order = - threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; - - const bool is_last_thread = thread_order == num_threads; - if (is_last_thread) { + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { last_done.store(true, std::memory_order_release); } @@ -855,17 +888,14 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { tlx::Aggregate& agg #endif ) { - const uint64_t TRAILING_ZEROS = - MASK_TRAILING_ZEROS[pair_size / sizeof(input_type)]; const uint64_t HASH_MASK = HASH_MASKS[pair_size / sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK) >> - TRAILING_ZEROS; + HASH_MASK); RabinKarpHash current_hash(text, - hash_value, + mix_select(hash_value), block_start + offset, pair_size); // Find the hash of the current window among the hashed block @@ -889,10 +919,12 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @brief Determine the positions for each block's earliest occurrence if /// there is any. /// - /// @param s The input text + /// @param text The input text /// @param level_data The data for the current level /// @param is_padded true, iff the last block of the level extends past the /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. /// @tparam use_hash Determines whether to use a rabin karp hash for hashing /// text windows or to use the block's content as a hash. For any window size /// greater than 8 bytes, use Rabin-Karp. @@ -916,7 +948,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size); + BlockMap links(threads, queue_size, num_blocks); // The number of threads finished with hashing blocks std::atomic_size_t num_done = 0; @@ -974,27 +1006,24 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { shard.insert(hash, {i, 0}); } } else { - const uint64_t TRAILING_ZEROS = - sizeof(input_type) * 8 * (8 - block_size); - const uint64_t HASH_MASK = static_cast(~0) << TRAILING_ZEROS; + const uint64_t HASH_MASK = HASH_MASKS[block_size / sizeof(input_type)]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & - HASH_MASK) >> - TRAILING_ZEROS; - RabinKarpHash hash(text, hash_value, block_start, block_size); + (*reinterpret_cast(block_start_ptr) & HASH_MASK); + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); shard.insert(hash, {i, 0}); } } - const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - const bool is_last_thread = thread_order == num_threads; - - if (is_last_thread) { + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { last_done.store(true, std::memory_order_release); } @@ -1174,18 +1203,15 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { tlx::Aggregate& hits #endif ) { - const uint64_t TRAILING_ZEROS = - MASK_TRAILING_ZEROS[level_data.block_size / sizeof(input_type)]; const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size / sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK) >> - TRAILING_ZEROS; + HASH_MASK); RabinKarpHash hash(text, - hash_value, + mix_select(hash_value), block_start + offset, level_data.block_size); // Find all blocks in the multimap that match our hash @@ -1266,7 +1292,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { for (size_t block = 0; block < levels[level].is_internal->size(); block++) { if ((*levels[level].is_internal)[block]) { - new_num_internal[level]++; + ++new_num_internal[level]; } } } @@ -1429,7 +1455,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; (*offsets)[num_back_blocks] = offset; - num_back_blocks++; + ++num_back_blocks; } sdsl::util::bit_compress(*pointers); @@ -1577,5 +1603,4 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { } } }; - } // namespace pasta diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 50c8534..6137bd9 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -25,11 +25,9 @@ #include #include #include -#include #include #include -#include -#include +#include #include namespace pasta { @@ -49,9 +47,9 @@ class MersenneHash { uint32_t start_; uint32_t length_; MersenneHash(std::vector const& text, - uint128_t hash, - uint64_t start, - uint64_t length) + const uint128_t hash, + const uint64_t start, + const uint64_t length) : text_(&text), hash_(hash), start_(start), @@ -62,15 +60,17 @@ class MersenneHash { constexpr MersenneHash(const MersenneHash& other) = default; constexpr MersenneHash(MersenneHash&& other) = default; - MersenneHash& operator=(const MersenneHash& other) = default; - MersenneHash& operator=(MersenneHash&& other) = default; + MersenneHash& operator=(const MersenneHash& other) = default; + MersenneHash& operator=(MersenneHash&& other) = default; bool operator==(const MersenneHash& other) const { #ifdef BT_INSTRUMENT - mersenne_hash_comparisons++; + ++mersenne_hash_comparisons; #endif // if (length_ != other.length_) // return false; + // std::cout << static_cast(hash_) << ", " + // << static_cast(other.hash_) << std::endl; if (hash_ != other.hash_) return false; @@ -81,10 +81,10 @@ class MersenneHash { #ifdef BT_INSTRUMENT if (!is_same) { // The hash is the same but the substring isn't => collision - mersenne_hash_collisions++; + ++mersenne_hash_collisions; } else { // The substrings are the same - mersenne_hash_equals++; + ++mersenne_hash_equals; } #endif return is_same; @@ -96,7 +96,7 @@ class MersenneHash { namespace std { template struct hash> { - pasta::MersenneHash::uint128_t + typename pasta::MersenneHash::uint128_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; } diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 4e52ca4..4ea03eb 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -87,9 +87,9 @@ class SyncShardedMap { /// The sequential backing hash map type using SeqHashMap = SeqHashMapType; /// The sequential hash map's hasher - using Hasher = SeqHashMap::hasher; + using Hasher = typename SeqHashMap::hasher; /// The type used for updates - using InputValue = UpdateFn::InputValue; + using InputValue = typename UpdateFn::InputValue; /// The actual pair of key and value stored in the map using StoredValue = std::pair; @@ -129,7 +129,7 @@ class SyncShardedMap { std::barrier barrier_; /// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html - inline uint64_t mix_select(uint64_t key) { + [[nodiscard]] uint64_t mix_select(uint64_t key) const { key ^= (key >> 31); key *= 0x7fb5d329728ea185; key ^= (key >> 27); @@ -144,12 +144,13 @@ class SyncShardedMap { // /// @brief Creates a new sharded map. /// - /// @param fill_threshold The fill percentage (between 0 and 1) above which - /// a thread is signaled to handle its own tasks. /// @param thread_count The exact number of threads working on this map. /// @param queue_capacity The maximum amount of tasks allowed in each queue. + /// @param map_capacity The initial capacity for each thread's local hash map. /// - SyncShardedMap(size_t thread_count, size_t queue_capacity) + SyncShardedMap(size_t thread_count, + size_t queue_capacity, + size_t map_capacity = 1024) : thread_count_(thread_count), map_(), task_queue_(), @@ -166,6 +167,7 @@ class SyncShardedMap { thread_count); for (size_t i = 0; i < thread_count; i++) { map_.emplace_back(); + map_.back().reserve(map_capacity); task_queue_.emplace_back(new StoredValue[queue_capacity], queue_capacity); task_count_[i] = 0; } @@ -229,7 +231,7 @@ class SyncShardedMap { } } - void handle_queue_sync(bool make_others_wait = true) { + void handle_queue_sync(const bool make_others_wait = true) { if (make_others_wait) { // If this value is >0 then other threads will also handle their queue // when trying to insert @@ -316,8 +318,6 @@ class SyncShardedMap { target_task_count.fetch_sub(1, mem::acq_rel); handle_queue_sync(); // Since the queue was handled, the task count is now 0 - // TODO It might be worth considering the recursive call again - // It might be the cause of some segfaults insert(std::move(pair)); return; } @@ -406,11 +406,11 @@ class SyncShardedMap { } } - SeqHashMap::iterator end() { + typename SeqHashMap::iterator end() { return map_.back().end(); } - SeqHashMap::iterator find(const K& key) { + typename SeqHashMap::iterator find(const K& key) { const size_t hash = Hasher{}(key); const size_t target_thread_id = mix_select(hash); SeqHashMap& map = map_[target_thread_id]; @@ -446,7 +446,7 @@ class SyncShardedMap { return loads; } - [[maybe_unused]] void print_ins_upd() { + [[maybe_unused]] void print_ins_upd() const { std::osyncstream(std::cout) << "Inserts: " << num_inserts_.load() << "\nUpdates: " << num_updates_.load() << std::endl; From 370861ab78ba887adfb620483e3675aed25c44bd Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 14 Nov 2023 18:01:27 +0100 Subject: [PATCH 52/92] remove map capacity parameter to reduce memory usage --- examples/build_bt.cpp | 33 ++++++++----------- .../block_tree_fp_par_sync_sharded_small.hpp | 2 +- .../block_tree/utils/sync_sharded_map.hpp | 5 +-- 3 files changed, 16 insertions(+), 24 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 898b1f4..8996240 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -27,7 +27,7 @@ #include #include -#define PAR_SHARDED_SYNC_SMALL +#define PAR_SHARDED_SYNC #ifdef FP # include std::unique_ptr> @@ -131,7 +131,8 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t) { ; return std::make_unique>( text, @@ -147,7 +148,8 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t) { ; return std::make_unique>( text, @@ -163,7 +165,8 @@ std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, - const size_t threads) { + const size_t threads, + const size_t) { ; return std::make_unique>( text, @@ -175,10 +178,6 @@ make_bt(std::vector& text, # define ALGO_NAME "par_phf" #endif -#ifdef BT_MALLOC_COUNT -# include -#endif - #if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL # define USES_QUEUE true # define IS_PARALLEL true @@ -229,7 +228,7 @@ int main(int argc, char** argv) { const size_t leaf_length = atoi(argv[3]); -#if defined IS_PARALLEL +#if IS_PARALLEL if (argc < 5) { std::cerr << "Please input number of threads (ignored if single threaded " "algorithm)" @@ -238,14 +237,18 @@ int main(int argc, char** argv) { } const size_t threads = atoi(argv[4]); +#else + const size_t threads = 1; #endif -#if defined USES_QUEUE +#if USES_QUEUE if (argc < 6) { std::cerr << "Please input queue size" << std::endl; exit(1); } const size_t queue_size = atoi(argv[5]); +#else + const size_t queue_size = 0; #endif std::stringstream ss; @@ -274,15 +277,7 @@ int main(int argc, char** argv) { << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); - auto bt = make_bt(text, - arity, - leaf_length, - threads -#if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL - , - queue_size -#endif - ); + auto bt = make_bt(text, arity, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp index 31b5e38..0fd471e 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp @@ -948,7 +948,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { } // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size, num_blocks); + BlockMap links(threads, queue_size); // The number of threads finished with hashing blocks std::atomic_size_t num_done = 0; diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 4ea03eb..9b225d3 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -148,9 +148,7 @@ class SyncShardedMap { /// @param queue_capacity The maximum amount of tasks allowed in each queue. /// @param map_capacity The initial capacity for each thread's local hash map. /// - SyncShardedMap(size_t thread_count, - size_t queue_capacity, - size_t map_capacity = 1024) + SyncShardedMap(size_t thread_count, size_t queue_capacity) : thread_count_(thread_count), map_(), task_queue_(), @@ -167,7 +165,6 @@ class SyncShardedMap { thread_count); for (size_t i = 0; i < thread_count; i++) { map_.emplace_back(); - map_.back().reserve(map_capacity); task_queue_.emplace_back(new StoredValue[queue_capacity], queue_capacity); task_count_[i] = 0; } From d5315ea06e87d044f4a991a8a56c60b0c8c16a56 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 20 Nov 2023 21:32:24 +0100 Subject: [PATCH 53/92] bit block tree access --- .gitmodules | 3 - CMakeLists.txt | 46 +- examples/build_bt.cpp | 83 +- extlib/BBHash | 1 - include/pasta/block_tree/bit_block_tree.hpp | 765 +++++++++ include/pasta/block_tree/block_tree.hpp | 109 +- ...par_phf.hpp => bit_block_tree_sharded.hpp} | 622 ++++--- .../construction/block_tree_fp_par_phf2.hpp | 1432 ----------------- .../block_tree_fp_par_sync_sharded_small.hpp | 2 + .../pasta/block_tree/utils/MersenneHash.hpp | 20 +- .../block_tree/utils/MersenneRabinKarp.hpp | 27 +- 11 files changed, 1328 insertions(+), 1782 deletions(-) delete mode 160000 extlib/BBHash create mode 100644 include/pasta/block_tree/bit_block_tree.hpp rename include/pasta/block_tree/construction/{block_tree_fp_par_phf.hpp => bit_block_tree_sharded.hpp} (74%) delete mode 100644 include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp diff --git a/.gitmodules b/.gitmodules index 3a9ade8..17c4003 100644 --- a/.gitmodules +++ b/.gitmodules @@ -35,9 +35,6 @@ [submodule "extlib/--force"] path = extlib/--force url = https://github.com/rizkg/BBHash -[submodule "extlib/BBHash"] - path = extlib/BBHash - url = https://github.com/rizkg/BBHash [submodule "extlib/parlayhash"] path = extlib/parlayhash url = https://github.com/cmuparlay/parlayhash diff --git a/CMakeLists.txt b/CMakeLists.txt index 2bb1cf8..c7dad0c 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -40,9 +40,9 @@ option(PASTA_BLOCK_TREE_BENCH "Enable outputting benchmark information" OFF) include(ExternalProject) +include(FetchContent) # Optional test if (PASTA_BLOCK_TREE_BUILD_TESTS) - include(FetchContent) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -74,29 +74,21 @@ if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) target_compile_definitions(build_bt PRIVATE BT_BENCH) #target_compile_definitions(build_bt PRIVATE ROBIN_HOOD_LOG_ENABLED) endif () +endif () - if (PASTA_BLOCK_TREE_MALLOC_COUNT) - message(DEBUG "malloc_count enabled") - # malloc_count - ExternalProject_Add(malloc_count - PREFIX ${CMAKE_CURRENT_BINARY_DIR} - GIT_REPOSITORY git@github.com:Skadic/malloc_count.git - BUILD_COMMAND gcc -c -fpic -ldl ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count/malloc_count.c - CONFIGURE_COMMAND "" - INSTALL_COMMAND "" - ) - - add_library(libmalloc_count INTERFACE) - target_include_directories(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) - target_link_libraries(libmalloc_count INTERFACE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count-build/malloc_count.o) - - target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/extlib/malloc_count) - target_include_directories(build_bt PRIVATE ${CMAKE_CURRENT_BINARY_DIR}/src/malloc_count) - #target_link_libraries(build_bt libmalloc_count) - #target_compile_definitions(build_bt PRIVATE BT_MALLOC_COUNT) - endif () +set(ZSTD_BUILD_STATIC ON) +set(ZSTD_BUILD_SHARED OFF) + +FetchContent_Declare( + zstd + URL "https://github.com/facebook/zstd/releases/download/v1.5.5/zstd-1.5.5.tar.gz" + DOWNLOAD_EXTRACT_TIMESTAMP TRUE + SOURCE_SUBDIR build/cmake +) + +FetchContent_MakeAvailable(zstd) +set_target_properties(libzstd_static PROPERTIES COMPILE_FLAGS "-w") -endif () set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) @@ -119,11 +111,6 @@ add_library(jiffy1 INTERFACE) target_include_directories(jiffy1 INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy-1) -add_library(bbhash STATIC ${CMAKE_CURRENT_SOURCE_DIR}/extlib/BBHash/example.cpp) -target_include_directories(bbhash SYSTEM PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/extlib/BBHash) -set_target_properties(bbhash PROPERTIES COMPILE_FLAGS "-w") - add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/include) @@ -145,8 +132,9 @@ target_link_libraries(pasta_block_tree INTERFACE waitfree-mpsc-queue sdsl #jiffy - bbhash - jiffy1) + jiffy1 + libzstd_static) + target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) target_include_directories(pasta_block_tree INTERFACE diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 8996240..d8c2d80 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -21,13 +21,16 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include +#include #include #include #include +#include #include +#include #include -#define PAR_SHARDED_SYNC +#define PAR_SHARDED_SYNC_SMALL #ifdef FP # include std::unique_ptr> @@ -159,23 +162,6 @@ make_bt(std::vector& text, threads); } # define ALGO_NAME "par_parlay" -#elif defined PAR_PHF -# include -std::unique_ptr> -make_bt(std::vector& text, - const size_t arity, - const size_t leaf_length, - const size_t threads, - const size_t) { - ; - return std::make_unique>( - text, - arity, - 1, - leaf_length, - threads); -} -# define ALGO_NAME "par_phf" #endif #if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL @@ -195,6 +181,8 @@ make_bt(std::vector& text, # endif #endif +#define BIT_BT + #include #include @@ -225,7 +213,6 @@ int main(int argc, char** argv) { std::cerr << "Please input max leaf length" << std::endl; exit(1); } - const size_t leaf_length = atoi(argv[3]); #if IS_PARALLEL @@ -262,22 +249,44 @@ int main(int argc, char** argv) { << std::endl; #endif +#ifdef BIT_BT + pasta::BitVector bv; +#else std::vector text; +#endif { std::string input; std::ifstream t(argv[1]); std::stringstream buffer; buffer << t.rdbuf(); input = buffer.str(); +#ifdef BIT_BT + new (&bv) pasta::BitVector(input.size()); + for (size_t i = 0; i < input.size(); ++i) { + bv[i] = input[i] == 'G' || input[i] == 'A'; + } +#else text = std::vector(input.begin(), input.end()); +#endif } std::cout << "RESULT algo=" << ALGO_NAME << " file=" << std::filesystem::path(argv[1]).filename().string() +#ifdef BIT_BT + << " bv_size=" << bv.size() +#else + << " file_size=" << text.size() +#endif << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); - auto bt = make_bt(text, arity, leaf_length, threads, queue_size); + // auto bt = make_bt(text, arity, leaf_length, threads, queue_size); + auto bt = std::make_unique>(bv, + arity, + 20, + leaf_length, + threads, + queue_size); auto elapsed = std::chrono::duration_cast(Clock::now() - now) .count(); @@ -300,17 +309,45 @@ int main(int argc, char** argv) { // std::ofstream ot(out_path); // bt->serialize(ot); -#pragma omp parallel for +#ifdef BIT_BT +#else +# pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { std::osyncstream(std::cerr) - << "Error at position " << i << "\nExpected: " << (char)text[i] - << "\nActual: " << (char)c << std::endl; + << "Error at position " << i + << "\nExpected: " << static_cast(text[i]) + << "\nActual: " << static_cast(c) << std::endl; exit(1); } } +#endif // ot.close(); + /* + for (size_t i = 0; i < bv.size() / 8; i++) { + uint8_t b = 0; + for (char j = 0; j < 8; j++) { + b |= bt->access(i * 8 + j) << j; + } + std::cout << i << ": "; + std::cout << std::flush + << std::bitset<8>{static_cast( + std::as_bytes(bv.data())[i])} + << " "; + std::cout << std::bitset<8>{b} << std::endl; + } + */ +#pragma omp parallel for + for (size_t i = 0; i < bv.size(); ++i) { + const bool c = bt->access(i); + if (c != bv[i]) { + std::osyncstream(std::cerr) + << "Error at position " << i << "\nExpected: " << std::boolalpha + << bv[i] << "\nActual: " << c << std::noboolalpha << std::endl; + exit(1); + } + } return 0; } diff --git a/extlib/BBHash b/extlib/BBHash deleted file mode 160000 index 1803c23..0000000 --- a/extlib/BBHash +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 1803c2325afab8ad045c94ef7872a319bc44a5e5 diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp new file mode 100644 index 0000000..68c68fe --- /dev/null +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -0,0 +1,765 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class BitBlockTree { +public: + /// @brief If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + using BVStoreType = pasta::BitVector::RawDataType; + constexpr static size_t BV_STORE_TYPE_BITS = sizeof(BVStoreType) * 8; + constexpr static size_t BV_STORE_TYPE_BYTES = sizeof(BVStoreType); + + size_type tau_; + size_type max_leaf_length_; + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits; + bool rank_support = false; + /// \brief Bit vectors for each level determining whether a block is internal + /// (=1) or not (=0) + std::vector block_tree_types_; + std::vector*> + block_tree_types_rs_; + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + ankerl::unordered_dense::map chars_index_; + std::vector chars_; + size_type u_chars_; + std::vector>> c_ranks_; + std::vector>> pointer_c_ranks_; + + bool access(const size_type bit_index) { + // FIXME: As of now this works on little endian systems only + const int64_t byte_index = bit_index / 8; + const int64_t bit_offset = bit_index % 8; + + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = byte_index / block_size_lvl_[0]; + int64_t off = byte_index % block_size_lvl_[0]; + int64_t child; + for (size_type i = 0; static_cast(i) < block_tree_types_.size(); + i++) { + auto& lvl = *block_tree_types_[i]; + auto& lvl_rs = *block_tree_types_rs_[i]; + auto& lvl_ptr = *block_tree_pointers_[i]; + auto& lvl_off = *block_tree_offsets_[i]; + if (lvl[blk_pointer] == 0) { + size_type blk = lvl_rs.rank0(blk_pointer); + off = off + lvl_off[blk]; + blk_pointer = lvl_ptr[blk]; + if (off >= block_size) { + blk_pointer++; + off -= block_size; + } + } + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; + } + const uint8_t byte = + decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; + //std::cout << std::bitset<8>(byte) << std::endl; + return ((1 << bit_offset) & byte) != 0; + }; + + int64_t select(uint8_t c, size_type j) { + auto c_index = chars_index_[c]; + auto& top_level = *block_tree_types_[0]; + + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + size_type current_block = (j - 1) / block_size_lvl_[0]; + size_type end_block = c_ranks_[c_index][0].size() - 1; + int64_t block_size = block_size_lvl_[0]; + // find first level block containing the jth occurrence of c with a bin + // search + while (current_block != end_block) { + size_type m = current_block + (end_block - current_block) / 2; + + size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; + if (f < j) { + if (end_block - current_block == 1) { + if (c_ranks_[c_index][0][m] < static_cast(j)) { + current_block = m + 1; + } + break; + } + current_block = m; + } else { + end_block = m - 1; + } + } + + // accumulator + int64_t s = current_block * block_size - 1; + // index that indicates how many c's are still unaccounted for + j -= (current_block == 0) ? 0 : c_ranks_[c_index][0][current_block - 1]; + // we translate unmarked blocks on the top level independently as it differs + // from the other levels + if (!top_level[current_block]) { + int64_t blk = top_level_rs.rank0(current_block); + current_block = top_level_ptr[blk]; + int64_t g = top_level_off[blk]; + int64_t rank_d = (current_block == 0) ? + c_ranks_[c_index][0][0] : + c_ranks_[c_index][0][current_block] - + c_ranks_[c_index][0][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][0][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][0][blk]; + s -= g; + } + } + uint64_t i = 1; + while (i < block_tree_types_.size()) { + auto& current_level = *block_tree_types_[i]; + auto& current_level_rs = *block_tree_types_rs_[i]; + auto& current_level_ptr = *block_tree_pointers_[i]; + auto& current_level_off = *block_tree_offsets_[i]; + auto& prev_level_rs = *block_tree_types_rs_[i - 1]; + current_block = prev_level_rs.rank1(current_block) * tau_; + block_size /= tau_; + int64_t k = current_block; + while ((int64_t)c_ranks_[c_index][i][current_block] < j) { + current_block++; + } + j -= (current_block == k) ? 0 : c_ranks_[c_index][i][current_block - 1]; + s += (current_block - k) * block_size; + if (!current_level[current_block]) { + int64_t blk = current_level_rs.rank0(current_block); + current_block = current_level_ptr[blk]; + int64_t g = current_level_off[blk]; + int64_t rank_d = (current_block % tau_ == 0) ? + c_ranks_[c_index][i][current_block] : + c_ranks_[c_index][i][current_block] - + c_ranks_[c_index][i][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][i][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][i][blk]; + s -= g; + } + } + i++; + } + + current_block = (*block_tree_types_rs_[i - 1]).rank1(current_block) * tau_; + int64_t l = 0; + while (j > 0) { + if (compressed_leaves_[current_block * leaf_size + l] == compress_map_[c]) + j--; + l++; + } + return s + l; + } + + int64_t rank_base(uint8_t c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + size_type to = off + top_level_off[blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (to >= block_size) { + int64_t adder = (child == 0) ? + c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + rank += adder; + blk_pointer++; + off = to - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + size_type to = off + ptr_off; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + + if (to >= block_size) { + auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + rank += adder; + blk_pointer++; + child = blk_pointer % tau_; + off = to - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + } + + int64_t rank(uint8_t c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + blk_pointer++; + off = off - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + blk_pointer++; + child = blk_pointer % tau_; + off = off - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + }; + + int64_t print_space_usage() { + int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + for (auto bv : block_tree_types_) { + space_usage += bv->size() / 8; + } + for (auto rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + } + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + for (const auto iv : block_tree_offsets_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + if (rank_support) { + for (auto c : chars_) { + int64_t sum = 0; + for (auto lvl : pointer_c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + for (auto lvl : c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + space_usage += sum; + } + } + + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + // space_usage += leaves_.size() * sizeof(uint8_t); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); + + return space_usage; + }; + + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + + int32_t add_rank_support() { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + int32_t add_rank_support_omp(int32_t threads) { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + omp_set_num_threads(threads); + +#pragma omp parallel for default(none) + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + inline size_type leading_zeros(int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + inline size_type leading_zeros(int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type rank_block(uint8_t c, size_type i, size_type j) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_block(c, i + 1, rank_blk * tau_ + k); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + size_type rank_g = 0; + rank_c += c_ranks_[chars_index_[c]][i][ptr]; + if (off != 0) { + rank_g = part_rank_block(c, i, ptr, off); + size_type rank_2nd = part_rank_block(c, i, ptr + 1, off); + rank_c -= rank_g; + rank_c += rank_2nd; + } + pointer_c_ranks_[chars_index_[c]][i][rank_0] = rank_g; + } + c_ranks_[chars_index_[c]][i][j] = rank_c; + return rank_c; + } + size_type part_rank_block(uint8_t c, size_type i, size_type j, size_type g) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + block_size_lvl_[i + 1] <= g; k++) { + rank_c += c_ranks_[chars_index_[c]][i + 1][rank_blk * tau_ + k]; + k_sum += block_size_lvl_[i + 1]; + } + + if (k_sum != g) { + rank_c += part_rank_block(c, i + 1, rank_blk * tau_ + k, g - k_sum); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + leaf_size <= g; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + k_sum += leaf_size; + } + + if (k_sum != g) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, g % leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + if (g + off >= block_size_lvl_[i]) { + rank_c += c_ranks_[chars_index_[c]][i][ptr] - + pointer_c_ranks_[chars_index_[c]][i][rank_0] + + part_rank_block(c, i, ptr + 1, g + off - block_size_lvl_[i]); + } else { + rank_c += part_rank_block(c, i, ptr, g + off) - + pointer_c_ranks_[chars_index_[c]][i][rank_0]; + } + } + return rank_c; + } + size_type rank_leaf(uint8_t c, size_type leaf_index, size_type i) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + // size_type x = leaves_.size() - leaf_index * this->tau_; + // i = std::min(i, x); + size_type result = 0; + for (size_type ind = 0; ind < i; ind++) { + if (compressed_leaves_[leaf_index * leaf_size + ind] == + compress_map_[c]) { + result++; + } + } + return result; + } + + size_type map_unique_chars(const std::vector& text) { + this->u_chars_ = 0; + uint8_t i = 0; + for (auto a : text) { + if (chars_index_.find(a) == chars_index_.end()) { + chars_index_[a] = i; + i++; + chars_.push_back(a); + } + } + this->u_chars_ = i; + return 0; + }; + size_type + find_next_smallest_index_binary_search(size_type i, + std::vector& pVector) { + int64_t l = 0; + int64_t r = pVector.size(); + while (l < r) { + int64_t m = std::floor((l + r) / 2); + if (i < pVector[m]) { + r = m; + } else { + l = m + 1; + } + } + return r - 1; + }; + int64_t + find_next_smallest_index_linear_scan(size_type i, + std::vector& pVector) { + int64_t b = 0; + while (b < pVector.size() && i >= pVector[b]) { + b++; + } + return b - 1; + }; + size_type find_next_smallest_index_block_tree(size_type index) { + size_type block_size = this->block_size_lvl_[0]; + size_type blk_pointer = index / block_size; + size_type off = index % block_size; + size_type child = 0; + for (size_type i = 0; i < this->block_tree_types_.size(); i++) { + if ((*this->block_tree_types_[i])[blk_pointer] == 0) { + return -1; + } + if (off > 0 && (*this->block_tree_types_[i])[blk_pointer + 1] == 0) { + return -1; + } + size_type rank_blk = this->block_tree_types_rs_[i]->rank1(blk_pointer); + blk_pointer = rank_blk * this->tau_; + block_size /= this->tau_; + child = off / block_size; + off = off % block_size; + blk_pointer += child; + } + return blk_pointer; + }; +}; + +} // namespace pasta + +/******************************************************************************/ diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index de69f2a..41a4510 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -36,7 +36,8 @@ namespace pasta { -template class BlockTree { +template +class BlockTree { public: /// @brief If this is true, then the only levels of the tree start to be /// included starting at the first level that contains a back block @@ -50,11 +51,11 @@ template class BlockTree { size_type leaf_size = 0; size_type amount_of_leaves = 0; bool rank_support = false; - std::vector block_tree_types_; - std::vector *> + std::vector block_tree_types_; + std::vector*> block_tree_types_rs_; - std::vector *> block_tree_pointers_; - std::vector *> block_tree_offsets_; + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; // std::vector*> block_tree_encoded_; std::vector block_size_lvl_; std::vector block_per_lvl_; @@ -64,7 +65,7 @@ template class BlockTree { std::vector decompress_map_; sdsl::int_vector<> compressed_leaves_; - std::unordered_map chars_index_; + ankerl::unordered_dense::map chars_index_; std::vector chars_; size_type u_chars_; std::vector> top_level_c_ranks_; @@ -78,10 +79,10 @@ template class BlockTree { int64_t child; for (size_type i = 0; static_cast(i) < block_tree_types_.size(); i++) { - auto &lvl = *block_tree_types_[i]; - auto &lvl_rs = *block_tree_types_rs_[i]; - auto &lvl_ptr = *block_tree_pointers_[i]; - auto &lvl_off = *block_tree_offsets_[i]; + auto& lvl = *block_tree_types_[i]; + auto& lvl_rs = *block_tree_types_rs_[i]; + auto& lvl_ptr = *block_tree_pointers_[i]; + auto& lvl_off = *block_tree_offsets_[i]; if (lvl[blk_pointer] == 0) { size_type blk = lvl_rs.rank0(blk_pointer); off = off + lvl_off[blk]; @@ -101,18 +102,17 @@ template class BlockTree { int64_t select(input_type c, size_type j) { auto c_index = chars_index_[c]; - auto &top_level = *block_tree_types_[0]; + auto& top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; size_type current_block = (j - 1) / block_size_lvl_[0]; size_type end_block = c_ranks_[c_index][0].size() - 1; int64_t block_size = block_size_lvl_[0]; // find first level block containing the jth occurrence of c with a bin // search while (current_block != end_block) { - size_type m = current_block + (end_block - current_block) / 2; size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; @@ -139,10 +139,10 @@ template class BlockTree { int64_t blk = top_level_rs.rank0(current_block); current_block = top_level_ptr[blk]; int64_t g = top_level_off[blk]; - int64_t rank_d = (current_block == 0) - ? c_ranks_[c_index][0][0] - : c_ranks_[c_index][0][current_block] - - c_ranks_[c_index][0][current_block - 1]; + int64_t rank_d = (current_block == 0) ? + c_ranks_[c_index][0][0] : + c_ranks_[c_index][0][current_block] - + c_ranks_[c_index][0][current_block - 1]; rank_d -= pointer_c_ranks_[c_index][0][blk]; if (rank_d < j) { j -= rank_d; @@ -155,11 +155,11 @@ template class BlockTree { } uint64_t i = 1; while (i < block_tree_types_.size()) { - auto ¤t_level = *block_tree_types_[i]; - auto ¤t_level_rs = *block_tree_types_rs_[i]; - auto ¤t_level_ptr = *block_tree_pointers_[i]; - auto ¤t_level_off = *block_tree_offsets_[i]; - auto &prev_level_rs = *block_tree_types_rs_[i - 1]; + auto& current_level = *block_tree_types_[i]; + auto& current_level_rs = *block_tree_types_rs_[i]; + auto& current_level_ptr = *block_tree_pointers_[i]; + auto& current_level_off = *block_tree_offsets_[i]; + auto& prev_level_rs = *block_tree_types_rs_[i - 1]; current_block = prev_level_rs.rank1(current_block) * tau_; block_size /= tau_; int64_t k = current_block; @@ -172,10 +172,10 @@ template class BlockTree { int64_t blk = current_level_rs.rank0(current_block); current_block = current_level_ptr[blk]; int64_t g = current_level_off[blk]; - int64_t rank_d = (current_block % tau_ == 0) - ? c_ranks_[c_index][i][current_block] - : c_ranks_[c_index][i][current_block] - - c_ranks_[c_index][i][current_block - 1]; + int64_t rank_d = (current_block % tau_ == 0) ? + c_ranks_[c_index][i][current_block] : + c_ranks_[c_index][i][current_block] - + c_ranks_[c_index][i][current_block - 1]; rank_d -= pointer_c_ranks_[c_index][i][blk]; if (rank_d < j) { j -= rank_d; @@ -200,10 +200,10 @@ template class BlockTree { } int64_t rank_base(input_type c, size_type index) { - pasta::BitVector &top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; int64_t c_index = chars_index_[c]; int64_t block_size = block_size_lvl_[0]; int64_t blk_pointer = index / block_size; @@ -224,10 +224,10 @@ template class BlockTree { blk_pointer = top_level_ptr[blk]; child = blk_pointer; if (to >= block_size) { - int64_t adder = (child == 0) - ? c_ranks_[c_index][0][blk_pointer] - : c_ranks_[c_index][0][blk_pointer] - - c_ranks_[c_index][0][blk_pointer - 1]; + int64_t adder = (child == 0) ? + c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; rank += adder; blk_pointer++; off = to - block_size; @@ -258,8 +258,8 @@ template class BlockTree { child = blk_pointer % tau_; if (to >= block_size) { - auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] - : c_ranks_[c_index][i][blk_pointer] - + auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - c_ranks_[c_index][i][blk_pointer - 1]; rank += adder; blk_pointer++; @@ -285,10 +285,10 @@ template class BlockTree { } int64_t rank(input_type c, size_type index) { - pasta::BitVector &top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; int64_t c_index = chars_index_[c]; int64_t block_size = block_size_lvl_[0]; int64_t blk_pointer = index / block_size; @@ -308,8 +308,8 @@ template class BlockTree { blk_pointer = top_level_ptr[blk]; child = blk_pointer; if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] - : c_ranks_[c_index][0][blk_pointer] - + rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - c_ranks_[c_index][0][blk_pointer - 1]; blk_pointer++; off = off - block_size; @@ -338,8 +338,8 @@ template class BlockTree { blk_pointer = (*block_tree_pointers_[i])[blk]; child = blk_pointer % tau_; if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] - : c_ranks_[c_index][i][blk_pointer] - + rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - c_ranks_[c_index][i][blk_pointer - 1]; blk_pointer++; child = blk_pointer % tau_; @@ -562,8 +562,10 @@ template class BlockTree { /// @param[out] height The number of levels in the tree. /// @param[out] blk_size The size of blocks on the first level of the tree. /// - void calculate_padding(int64_t &padding, int64_t text_length, int64_t &height, - int64_t &blk_size) { + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { // This is the number of characters occupied by a tree with s*tau^h levels // and leaves of size tau. At the start, we only have a tree with the first // level with s leaf blocks which each have size tau. If we insert another @@ -621,8 +623,8 @@ template class BlockTree { c_ranks_[chars_index_[c]][i][j] = rank_c; return rank_c; } - size_type part_rank_block(input_type c, size_type i, size_type j, - size_type g) { + size_type + part_rank_block(input_type c, size_type i, size_type j, size_type g) { if (static_cast(j) >= block_tree_types_[i]->size()) { return 0; } @@ -669,7 +671,6 @@ template class BlockTree { return rank_c; } size_type rank_leaf(input_type c, size_type leaf_index, size_type i) { - if (static_cast(leaf_index * leaf_size) >= compressed_leaves_.size()) { return 0; @@ -686,7 +687,7 @@ template class BlockTree { return result; } - size_type map_unique_chars(const std::vector &text) { + size_type map_unique_chars(const std::vector& text) { this->u_chars_ = 0; input_type i = 0; for (auto a : text) { @@ -701,7 +702,7 @@ template class BlockTree { }; size_type find_next_smallest_index_binary_search(size_type i, - std::vector &pVector) { + std::vector& pVector) { int64_t l = 0; int64_t r = pVector.size(); while (l < r) { @@ -716,7 +717,7 @@ template class BlockTree { }; int64_t find_next_smallest_index_linear_scan(size_type i, - std::vector &pVector) { + std::vector& pVector) { int64_t b = 0; while (b < pVector.size() && i >= pVector[b]) { b++; diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp similarity index 74% rename from include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp rename to include/pasta/block_tree/construction/bit_block_tree_sharded.hpp index 8ed82fd..9c7b94b 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phf.hpp +++ b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp @@ -21,17 +21,12 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/bit_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#include -#pragma GCC diagnostic pop - +#include #include #include #include @@ -46,17 +41,59 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes -/// and a sharded hash map. -/// @tparam input_type The type of the characters in the input string +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. /// @tparam size_type The type used for indices etc. (must be a signed integer) -/// @tparam queue_type The type of queue to use for communication /// in the sharded hash map. -template -class BlockTreeFPParPHF : public BlockTree { +template +class BitBlockTreeSharded : public BitBlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; + /// @brief For some block size (in bytes) i, return the number of trailing + /// zeros in a 64 bit integer when zeroing out characters that are not part + /// of the block. + constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + + /// @brief Masks used for the identity hash. These depend on endianness + constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } + } + + /// @brief Masks for identity hashes for a block size i (in bytes) + constexpr static std::array HASH_MASKS = masks(); + /// @brief A marker for a block that has no earlier occurrence constexpr static size_type NO_EARLIER_OCC = -1; /// @brief A marker for a block that has been pruned @@ -64,10 +101,13 @@ class BlockTreeFPParPHF : public BlockTree { /// @brief Base of the polynomial used for the Rabin-Karp hasher constexpr static size_type SIGMA = 256; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = 2305843009213693951ULL; + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 61; + constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + // constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); /// @brief A bit vector using BitVector = pasta::BitVector; @@ -77,19 +117,35 @@ class BlockTreeFPParPHF : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_map>; + ankerl::unordered_dense::map>; + // robin_hood::unordered_flat_map>; + // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher - using RabinKarpHash = MersenneHash; + using RabinKarpHash = MersenneHash; /// @brief A hash map with rabin karp hashes as keys template update_fn_type> + UpdateFunction update_fn_type, + template typename seq_map_type = SeqHashMap> using RabinKarpMap = - SyncShardedMap; + SyncShardedMap; + +#define MIX + static uint64_t mix_select(uint64_t key) { +#ifdef MIX + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); +#endif + return key; + } #ifdef BT_INSTRUMENT public: @@ -120,16 +176,16 @@ class BlockTreeFPParPHF : public BlockTree { /// @brief Block start indices std::unique_ptr> block_starts; /// @brief The block size on this level - size_type block_size; - /// @brief The index of the current level. First level is 0, second level is - /// 1 etc. - size_type level_index; + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; /// @brief The number of blocks on the current level - size_type num_blocks; + int64_t num_blocks; - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) : is_internal(nullptr), is_internal_rank(nullptr), pointers(new std::vector()), @@ -142,9 +198,8 @@ class BlockTreeFPParPHF : public BlockTree { /// @brief Checks whether a block is adjacent in the text /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; } }; @@ -160,15 +215,6 @@ class BlockTreeFPParPHF : public BlockTree { /// another thread tries to access the vector during reallocation. std::list occurrences; - inline PairOccurrences() - : first_occ_block{std::numeric_limits::max()}, - occurrences{} {} - - bool operator==(const PairOccurrences& other) const { - return first_occ_block == other.first_occ_block && - occurrences == other.occurrences; - }; - /// @brief Initialize the occurrences of a hashed block pair. /// /// Note, that this only sets the first occurrence to the given block index, @@ -178,21 +224,20 @@ class BlockTreeFPParPHF : public BlockTree { : first_occ_block(first_occ_block_), occurrences() {} + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block_pair(size_type block_index) { + void add_block_pair(size_type block_index) { occurrences.push_back(block_index); } /// @brief If the given block index is an earlier occurrence, update it /// @param block_index The block index of an occurrence - [[gnu::noinline]] void update(size_type block_index) { + void update(size_type block_index) { first_occ_block = std::min(first_occ_block, block_index); } - - void invalidate() { - first_occ_block = std::numeric_limits::max(); - } }; /// @brief Contains data about the occurrences of a hashed block @@ -235,18 +280,25 @@ class BlockTreeFPParPHF : public BlockTree { : first_occ(other.first_occ.load()), occurrences(std::move(other.occurrences)) {} + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + /// @brief Add a block index to the occurrences. /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block(size_type block_index) { + void add_block(size_type block_index) { occurrences.push_back(block_index); } /// @brief If the given block index and offset are an earlier occurrence, /// update them /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - [[gnu::noinline]] inline void update(size_type block_index, - size_type block_offset) { + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { FirstOccurrence prev_first_occ = this->first_occ.load(); FirstOccurrence set(block_index, block_offset); while (block_index < prev_first_occ.block && @@ -322,7 +374,10 @@ class BlockTreeFPParPHF : public BlockTree { /// @brief Constructs the block tree. /// @param text The input text. - void construct(const std::vector& text, + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::span text, const size_t threads, const size_t queue_size) { #ifdef BT_INSTRUMENT @@ -386,14 +441,38 @@ class BlockTreeFPParPHF : public BlockTree { TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); - scan_block_pairs(text, current, is_padded, threads, queue_size); + if (2 * static_cast(current.block_size) > 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); #endif - scan_blocks(text, current, is_padded, threads, queue_size); + if (static_cast(current.block_size) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( Clock::now() - now) @@ -472,9 +551,10 @@ class BlockTreeFPParPHF : public BlockTree { return 1 + ((x - 1) / y); } - [[maybe_unused]] void print_aggregate(const char* name, - const tlx::Aggregate& agg, - size_t div = 1) { + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", name, static_cast(agg.min() / div), @@ -484,17 +564,6 @@ class BlockTreeFPParPHF : public BlockTree { static_cast(agg.count())); } - struct SHash { - uint64_t operator()(const uint64_t& key, - uint64_t = 0xAAAAAAAA55555555ULL) const { - return key; - } - }; - - // using hasher_t = boomphf::SingleHashFunctor; - using hasher_t = SHash; - using phf_t = boomphf::mphf; - /// @brief Scan through the blocks pairwise in order to identify which blocks /// should be replaced with back blocks. /// @@ -502,9 +571,15 @@ class BlockTreeFPParPHF : public BlockTree { /// @param level The data for the current level. /// @param is_padded `true` iff the last block on this level *does not* end at /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. /// - /// @return The block start indices for the next level of the tree - void scan_block_pairs(const std::vector& text, + template + void scan_block_pairs(const std::span text, LevelData& level, const bool is_padded, const size_t threads, @@ -518,23 +593,10 @@ class BlockTreeFPParPHF : public BlockTree { // A map containing hashed block pairs mapped to their indices of the // pairs' first block respectively BlockPairMap map(threads, queue_size); - std::vector hashes(level.num_blocks - 1 - is_padded); - std::vector> values(level.num_blocks - 1 - - is_padded); - hashes.resize(level.num_blocks - 1 - is_padded, - std::numeric_limits::max()); - values.resize(level.num_blocks - 1 - is_padded, - {std::numeric_limits::max(), - std::numeric_limits::max()}); - - std::vector table; std::atomic_size_t threads_done = 0; std::atomic_bool last_done = false; auto& barrier = map.barrier(); - - phf_t* phf; - #ifdef BT_INSTRUMENT TimePoint now = Clock::now(); tlx::Aggregate scan_hits; @@ -558,11 +620,7 @@ class BlockTreeFPParPHF : public BlockTree { handle_queue_ns, \ scan_hits, \ threads, \ - std::cout, \ - hashes, \ - values, \ - phf, \ - table) + std::cout) #else # pragma omp parallel default(none) num_threads(threads) \ shared(level, map, text, is_padded, threads_done, last_done, barrier) @@ -578,7 +636,7 @@ class BlockTreeFPParPHF : public BlockTree { // Hash every window and determine for all block pairs whether // they have previous occurrences. - size_t segment_size = + const size_t segment_size = std::max(1, ceil_div(num_block_pairs, num_threads)); // Start and end index of the current thread's segment @@ -586,21 +644,51 @@ class BlockTreeFPParPHF : public BlockTree { const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block - if (!level.next_is_adjacent(i)) { - continue; + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr) & HASH_MASK); + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); } - rk.restart(block_starts[i]); - // Move the hasher to the current block pair - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - hashes[i] = hash.hash_; - values[i] = {hash.hash_, i}; } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); #pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT @@ -617,71 +705,43 @@ class BlockTreeFPParPHF : public BlockTree { } #endif -#pragma omp single - { - std::sort(hashes.begin(), hashes.end()); - size_t num_distinct = 1; - { - uint64_t last = hashes[0]; - for (size_t i = 1; i < hashes.size(); ++i) { - if (hashes[i] > last) { - hashes[num_distinct++] = hashes[i]; - last = hashes[i]; - } - } - } - values.resize(num_distinct); - phf = new boomphf::mphf(num_distinct, - hashes, - 1, - 3.0, - false, - false, - 0.1); - // std::cout << "number keys for level " << level.level_index << ": " - // << phf->nbKeys() << " while num_distinct is " << - // num_distinct - // << std::endl; - table.resize(phf->nbKeys() + 1, PairOccurrences()); - - // TODO Could be parallel - for (auto& [hash, occ] : values) { - if (hash == std::numeric_limits::max()) { - continue; - } - volatile uint64_t pos = phf->lookup(hash); - if (pos >= table.size()) { - table.resize(pos + 1, PairOccurrences()); - } - PairOccurrences& occs = table[pos]; - // std::cout << "yo hash " << hash << " occ " << occ << " -> " << pos - // << std::endl; - occs.add_block_pair(occ); - occs.update(occ); - } - now = Clock::now(); - } -#pragma omp barrier - if (start < static_cast(num_block_pairs)) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); } - scan_windows_in_block_pair(rk, - {table}, - *phf, - block_size, - i + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); + ); + } } } @@ -722,9 +782,8 @@ class BlockTreeFPParPHF : public BlockTree { #endif level.is_internal = std::make_unique(level.num_blocks); - fill_is_internal(*level.is_internal, std::span(table)); + fill_is_internal(*level.is_internal, map); level.is_internal_rank = std::make_unique(*level.is_internal); - delete phf; } /// @brief Fills the bit vector `is_internal` based on the values in the @@ -733,8 +792,7 @@ class BlockTreeFPParPHF : public BlockTree { /// this level. /// @param map A map, mapping hashed block pairs to their first occurrence's /// block index. - void fill_is_internal(BitVector& is_internal, - std::span map) { + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { const size_type num_blocks = is_internal.size(); #ifdef BT_INSTRUMENT TimePoint now = Clock::now(); @@ -745,17 +803,15 @@ class BlockTreeFPParPHF : public BlockTree { // occurrence. The LSB is 1 iff the block and its predecessor // have a prior occurrence. sdsl::int_vector<2> markings(num_blocks, 0); - for (const PairOccurrences& pair_occs : map) { - if (pair_occs.first_occ_block == ~0) { - continue; - } - for (const size_type occ : pair_occs.occurrences) { - if (pair_occs.first_occ_block < occ) { - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; - } - } - } + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); #ifdef BT_INSTRUMENT bp_markings_ns += std::chrono::duration_cast(Clock::now() - now) @@ -790,8 +846,7 @@ class BlockTreeFPParPHF : public BlockTree { /// hashed. static inline void scan_windows_in_block_pair(RabinKarp& rk, - std::span map, - phf_t& phf, + BlockPairMap& map, const size_t num_iterations, const size_type current_block_index #ifdef BT_INSTRUMENT @@ -803,10 +858,48 @@ class BlockTreeFPParPHF : public BlockTree { RabinKarpHash current_hash = rk.current_hash(); // Find the hash of the current window among the hashed block // pairs. - const uint64_t idx = phf.lookup(current_hash.hash_); - PairOccurrences* occurrences; - if (idx >= map.size() || - (occurrences = &map[idx])->first_occ_block == ~0) { + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::span& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr + offset) & + HASH_MASK); + RabinKarpHash current_hash(text, + mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { #ifdef BT_INSTRUMENT agg.add(0); continue; @@ -816,18 +909,25 @@ class BlockTreeFPParPHF : public BlockTree { continue; #endif } - occurrences->update(current_block_index); + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); } } /// @brief Determine the positions for each block's earliest occurrence if /// there is any. /// - /// @param s The input text + /// @param text The input text /// @param level_data The data for the current level /// @param is_padded true, iff the last block of the level extends past the /// end of the text - void scan_blocks(const std::vector& text, + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(std::span text, LevelData& level_data, const bool is_padded, const size_t threads, @@ -895,19 +995,33 @@ class BlockTreeFPParPHF : public BlockTree { const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); // Hash each block and store their hashes in the map - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[block_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr) & HASH_MASK); + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } } - const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - - const bool is_last_thread = thread_order == num_threads; - if (is_last_thread) { + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { last_done.store(true, std::memory_order_release); } @@ -936,23 +1050,41 @@ class BlockTreeFPParPHF : public BlockTree { // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); } - scan_windows_in_block(rk, - links, - level_data, - i + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); + ); + } } } #ifdef BT_INSTRUMENT @@ -1040,7 +1172,45 @@ class BlockTreeFPParPHF : public BlockTree { ) { for (size_type offset = 0; offset < level_data.block_size; ++offset, rk.next()) { - const RabinKarpHash hash = rk.current_hash(); + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::span& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + (*reinterpret_cast(block_start_ptr + offset) & + HASH_MASK); + RabinKarpHash hash(text, + mix_select(hash_value), + block_start + offset, + level_data.block_size); // Find all blocks in the multimap that match our hash auto found = links.find(hash); if (found == links.end()) { @@ -1068,7 +1238,7 @@ class BlockTreeFPParPHF : public BlockTree { /// @param level The level data of the current level. /// @return The level data of the next level. [[nodiscard]] LevelData - generate_next_level(const std::vector& text, + generate_next_level(const std::span text, const LevelData& level) const { const size_t block_size = level.block_size; const size_t num_blocks = level.num_blocks; @@ -1108,9 +1278,9 @@ class BlockTreeFPParPHF : public BlockTree { /// @param[in] levels A vector containing data for each level, with the /// first entry corresponding to the topmost level. /// - void make_tree(const std::vector& text, + void make_tree(const std::span text, std::vector& levels, - int64_t padding) { + const int64_t padding) { const bool is_padded = padding > 0; // Count the current number of internal blocks per level @@ -1119,7 +1289,7 @@ class BlockTreeFPParPHF : public BlockTree { for (size_t block = 0; block < levels[level].is_internal->size(); block++) { if ((*levels[level].is_internal)[block]) { - new_num_internal[level]++; + ++new_num_internal[level]; } } } @@ -1282,7 +1452,7 @@ class BlockTreeFPParPHF : public BlockTree { (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; (*offsets)[num_back_blocks] = offset; - num_back_blocks++; + ++num_back_blocks; } sdsl::util::bit_compress(*pointers); @@ -1396,11 +1566,12 @@ class BlockTreeFPParPHF : public BlockTree { } public: - BlockTreeFPParPHF(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads) { + BitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { const auto old = omp_get_max_threads(); const auto old_dynamic = omp_get_dynamic(); omp_set_dynamic(0); @@ -1408,13 +1579,15 @@ class BlockTreeFPParPHF : public BlockTree { this->tau_ = arity; this->s_ = root_arity; this->max_leaf_length_ = max_leaf_length; - this->map_unique_chars(text); - construct(text, threads, 1000); + this->num_bits = text.size(); + const std::span bytes(reinterpret_cast(text.data().data()), + ceil_div(text.size(), 8ULL)); + construct(bytes, threads, queue_size); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } - ~BlockTreeFPParPHF() { + ~BitBlockTreeSharded() { for (auto& rank : this->block_tree_types_rs_) { delete rank; } @@ -1429,5 +1602,4 @@ class BlockTreeFPParPHF : public BlockTree { } } }; - } // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp deleted file mode 100644 index 4cee5bd..0000000 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phf2.hpp +++ /dev/null @@ -1,1432 +0,0 @@ -/******************************************************************************* - * This file is part of pasta::block_tree - * - * Copyright (C) 2023 Etienne Palanga - * - * pasta::block_tree is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * pasta::block_tree is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with pasta::block_tree. If not, see . - * - ******************************************************************************/ - -#pragma once - -#include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/block_tree.hpp" -#include "pasta/block_tree/utils/MersenneHash.hpp" -#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include "pasta/block_tree/utils/sync_sharded_map.hpp" - -#pragma GCC diagnostic push -#pragma GCC diagnostic ignored "-Wunknown-pragmas" -#pragma GCC diagnostic ignored "-Wmaybe-uninitialized" -#include -#pragma GCC diagnostic pop - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -__extension__ typedef unsigned __int128 uint128_t; - -namespace pasta { - -/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes -/// and a sharded hash map. -/// @tparam input_type The type of the characters in the input string -/// @tparam size_type The type used for indices etc. (must be a signed integer) -/// @tparam queue_type The type of queue to use for communication -/// in the sharded hash map. -template -class BlockTreeFPParPHF : public BlockTree { - using Clock = std::chrono::high_resolution_clock; - using TimePoint = Clock::time_point; - - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = 2305843009213693951ULL; - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 61; - - /// @brief A bit vector - using BitVector = pasta::BitVector; - /// @brief A rank data structure for a bit vector - using Rank = pasta::RankSelect; - - /// @brief A sequential hash map used as backing for the sharded hash map. - template - using SeqHashMap = - robin_hood::unordered_map>; - - /// @brief A rabin karp hasher preconfigured for the current template - /// parameters - using RabinKarp = MersenneRabinKarp; - /// @brief A rabin karp hash for the preconfigured rabin karp hasher - using RabinKarpHash = MersenneHash; - - /// @brief A hash map with rabin karp hashes as keys - template update_fn_type> - using RabinKarpMap = - SyncShardedMap; - -#ifdef BT_INSTRUMENT -public: - size_t bp_hash_pairs_ns = 0; - size_t bp_scan_pairs_ns = 0; - size_t bp_markings_ns = 0; - size_t bp_bitvec_ns = 0; - - size_t b_hash_blocks_ns = 0; - size_t b_scan_blocks_ns = 0; - size_t b_update_blocks_ns = 0; -#endif - -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - size_type block_size; - /// @brief The index of the current level. First level is 0, second level is - /// 1 etc. - size_type level_index; - /// @brief The number of blocks on the current level - size_type num_blocks; - - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - inline PairOccurrences() - : first_occ_block{std::numeric_limits::max()}, - occurrences{} {} - - bool operator==(const PairOccurrences& other) const { - return first_occ_block == other.first_occ_block && - occurrences == other.occurrences; - }; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - [[gnu::noinline]] void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - - void invalidate() { - first_occ_block = std::numeric_limits::max(); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - [[gnu::noinline]] inline void update(size_type block_index, - size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - - /// @brief Constructs the block tree. - /// @param text The input text. - void construct(const std::vector& text, - const size_t threads, - const size_t queue_size) { -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - const size_type text_len = text.size(); - /// The number of characters a block tree with s top-level blocks and arity - /// of strictly tau would exceed over the text size - int64_t padding; - /// The height of the tree - int64_t tree_height; - /// The size of the largest blocks (i.e. the top level blocks) - int64_t top_block_size; - - this->calculate_padding(padding, text_len, tree_height, top_block_size); - - const bool is_padded = padding > 0; - - std::vector levels; - - // Prepare the top level - levels.emplace_back(0, top_block_size, text_len / top_block_size); - LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); - for (size_type i = 0; i < text_len; i += top_level.block_size) { - top_level.block_starts->push_back(i); - } - top_level.block_size = top_block_size; - top_level.num_blocks = top_level.block_starts->size(); - -#ifdef BT_INSTRUMENT - - const size_t setup_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_BENCH - std::cout << " setup=" << setup_ns; -# endif - - size_t pairs_ns = 0; - size_t blocks_ns = 0; - size_t generate_ns = 0; -#endif -#ifdef BT_DBG - std::cout << "using " << threads << " threads" << std::endl; -#endif - -#ifdef BT_BENCH - std::cout << " queue_capacity=" << queue_size; -#endif - - // Construct the pre-pruned tree level by level - for (size_t level = 0; level < static_cast(tree_height); level++) { -#ifdef BT_DBG - std::cout << "----------------- level " << level << " -----------------" - << std::endl; -#endif - -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - LevelData& current = levels.back(); - scan_block_pairs(text, current, is_padded, threads, queue_size); -#ifdef BT_INSTRUMENT - pairs_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - scan_blocks(text, current, is_padded, threads, queue_size); -#ifdef BT_INSTRUMENT - blocks_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { - levels.push_back(std::move(generate_next_level(text, current))); - } -#ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); -#endif - } -#ifdef BT_INSTRUMENT -# if defined(BT_DBG) - std::cout << "pairs: " << (pairs_ns / 1'000'000) - << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) - << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) - << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) - << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) - << "ms,\nblocks: " << (blocks_ns / 1'000'000) - << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) - << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) - << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) - << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," - << std::endl; -# elif defined(BT_BENCH) - std::cout << " pairs=" << (pairs_ns / 1'000'000) - << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) - << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) - << " markings=" << (bp_markings_ns / 1'000'000) - << " bitvec=" << (bp_bitvec_ns / 1'000'000) - << " blocks=" << (blocks_ns / 1'000'000) - << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) - << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) - << " update_blocks=" << (b_update_blocks_ns / 1'000'000) - << " generate_next=" << (generate_ns / 1'000'000); - -# endif - now = Clock::now(); -#endif - prune(levels); -#ifdef BT_INSTRUMENT - size_t prune_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -# ifdef BT_DBG - std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; -# elif defined BT_BENCH - std::cout << " prune=" << (prune_ns / 1'000'000); -# endif -#endif - - make_tree(text, levels, padding); -#ifdef BT_INSTRUMENT - size_t make_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); -# ifdef BT_DBG - std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; -# elif defined BT_BENCH - std::cout << " make=" << (make_ns / 1'000'000); -# endif -#endif - } - - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - [[maybe_unused]] void print_aggregate(const char* name, - const tlx::Aggregate& agg, - size_t div = 1) { - printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", - name, - static_cast(agg.min() / div), - static_cast(agg.max() / div), - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - static_cast(agg.count())); - } - - struct SHash { - uint64_t operator()(const uint64_t& key, - uint64_t = 0xAAAAAAAA55555555ULL) const { - return key; - } - }; - - // using hasher_t = boomphf::SingleHashFunctor; - using hasher_t = SHash; - using phf_t = boomphf::mphf; - - /// @brief Scan through the blocks pairwise in order to identify which blocks - /// should be replaced with back blocks. - /// - /// @param text The input string. - /// @param level The data for the current level. - /// @param is_padded `true` iff the last block on this level *does not* end at - /// the exact end of the text. - /// - /// @return The block start indices for the next level of the tree - void scan_block_pairs(const std::vector& text, - LevelData& level, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - if (level.num_blocks < 4) { - level.is_internal = std::make_unique(level.num_blocks, true); - level.is_internal_rank = std::make_unique(*level.is_internal); - return; - } - - // A map containing hashed block pairs mapped to their indices of the - // pairs' first block respectively - BlockPairMap map(threads, queue_size); - std::vector hashes(level.num_blocks - 1 - is_padded); - std::vector> values(level.num_blocks - 1 - - is_padded); - hashes.resize(level.num_blocks - 1 - is_padded, - std::numeric_limits::max()); - values.resize(level.num_blocks - 1 - is_padded, - {std::numeric_limits::max(), - std::numeric_limits::max()}); - - std::vector table; - - std::atomic_size_t threads_done = 0; - std::atomic_bool last_done = false; - auto& barrier = map.barrier(); - - phf_t* phf; - -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, \ - map, \ - text, \ - now, \ - is_padded, \ - threads_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits, \ - threads, \ - std::cout, \ - hashes, \ - values, \ - phf, \ - table) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) -#endif - { - const size_t thread_id = omp_get_thread_num(); - typename BlockPairMap::Shard shard = map.get_shard(thread_id); - const size_t num_threads = omp_get_num_threads(); - const size_t num_block_pairs = level.num_blocks - 1 - is_padded; - const size_t block_size = level.block_size; - const size_t pair_size = 2 * block_size; - const auto& block_starts = *level.block_starts; - - // Hash every window and determine for all block pairs whether - // they have previous occurrences. - size_t segment_size = - std::max(1, ceil_div(num_block_pairs, num_threads)); - - // Start and end index of the current thread's segment - const auto start = thread_id * segment_size; - const auto end = - std::min(num_block_pairs, (thread_id + 1) * segment_size); - - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block - if (!level.next_is_adjacent(i)) { - continue; - } - rk.restart(block_starts[i]); - // Move the hasher to the current block pair - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - hashes[i] = hash.hash_; - values[i] = {hash.hash_, i}; - } -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - { - bp_hash_pairs_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - -#pragma omp single - { - std::sort(hashes.begin(), hashes.end()); - size_t num_distinct = 1; - { - uint64_t last = hashes[0]; - for (size_t i = 1; i < hashes.size(); ++i) { - if (hashes[i] > last) { - hashes[num_distinct++] = hashes[i]; - last = hashes[i]; - } - } - } - values.resize(num_distinct); - phf = new boomphf::mphf(num_distinct, - hashes, - 1, - 3.0, - false, - false, - 0.1); - // std::cout << "number keys for level " << level.level_index << ": " - // << phf->nbKeys() << " while num_distinct is " << - // num_distinct - // << std::endl; - table.resize(phf->nbKeys() + 1, PairOccurrences()); - - // TODO Could be parallel - for (auto& [hash, occ] : values) { - if (hash == std::numeric_limits::max()) { - continue; - } - volatile uint64_t pos = phf->lookup(hash); - if (pos >= table.size()) { - table.resize(pos + 1, PairOccurrences()); - } - PairOccurrences& occs = table[pos]; - // std::cout << "yo hash " << hash << " occ " << occ << " -> " << pos - // << std::endl; - occs.add_block_pair(occ); - occs.update(occ); - } - } -#pragma omp barrier - - if (start < static_cast(num_block_pairs)) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); - } - scan_windows_in_block_pair(rk, - {table}, - *phf, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } - -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - bp_scan_pairs_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : map.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Pair Map Loads ", map_loads); - print_aggregate("Pair Map Hits ", scan_hits); - print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); - - BT_ASSERT(map.num_inserts_.load() == map.size()); -# endif -#endif - - level.is_internal = std::make_unique(level.num_blocks); - fill_is_internal(*level.is_internal, std::span(table)); - level.is_internal_rank = std::make_unique(*level.is_internal); - delete phf; - } - - /// @brief Fills the bit vector `is_internal` based on the values in the - /// given map. - /// @param is_internal An unfilled bit vector with a bit for each block on - /// this level. - /// @param map A map, mapping hashed block pairs to their first occurrence's - /// block index. - void fill_is_internal(BitVector& is_internal, - std::span map) { - const size_type num_blocks = is_internal.size(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - // Set up the packed array holding the markings for each block. - // Each mark is a 2-bit number. - // The MSB is 1 iff the block and its successor have a prior - // occurrence. The LSB is 1 iff the block and its predecessor - // have a prior occurrence. - sdsl::int_vector<2> markings(num_blocks, 0); - for (const PairOccurrences& pair_occs : map) { - if (pair_occs.first_occ_block == ~0) { - continue; - } - for (const size_type occ : pair_occs.occurrences) { - if (pair_occs.first_occ_block < occ) { - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; - } - } - } -#ifdef BT_INSTRUMENT - bp_markings_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the bit vector indicating which blocks are internal - is_internal[0] = true; - is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; - for (size_type i = 0; i < num_blocks - 1; ++i) { - const bool block_is_internal = markings[i] != 0b11; - is_internal[i] = block_is_internal; - } -#ifdef BT_INSTRUMENT - bp_bitvec_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scan through the windows starting in a block and mark - /// them accordingly if they represent the earliest occurrence of some - /// block hash. - /// - /// The supplied `RabinKarp` hasher must be at the start of the block. - /// @param rk A Rabin-Karp hasher whose state is at the start of the block. - /// @param map The map containing the hashes of block pairs mapped to their - /// block indexes at which they occur. - /// @param num_iterations The number of contiguous windows to hash. - /// @param current_block_index The index of the block being currently - /// hashed. - static inline void - scan_windows_in_block_pair(RabinKarp& rk, - std::span map, - phf_t& phf, - const size_t num_iterations, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg -#endif - ) { - for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { - RabinKarpHash current_hash = rk.current_hash(); - // Find the hash of the current window among the hashed block - // pairs. - const uint64_t idx = phf.lookup(current_hash.hash_); - PairOccurrences* occurrences; - if (idx >= map.size() || - (occurrences = &map[idx])->first_occ_block == ~0) { -#ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); -#else - continue; -#endif - } - occurrences->update(current_block_index); - } - } - - /// @brief Determine the positions for each block's earliest occurrence if - /// there is any. - /// - /// @param s The input text - /// @param level_data The data for the current level - /// @param is_padded true, iff the last block of the level extends past the - /// end of the text - void scan_blocks(const std::vector& text, - LevelData& level_data, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - const size_t num_blocks = level_data.num_blocks; - - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); - level_data.offsets = - std::make_unique>(num_blocks, 0); - level_data.counters = - std::make_unique>(num_blocks, 0); - - if (num_blocks <= 2) { - return; - } - - // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size); - - // The number of threads finished with hashing blocks - std::atomic_size_t num_done = 0; - // Whether the last thread is done - std::atomic_bool last_done = false; - auto& barrier = links.barrier(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, \ - text, \ - links, \ - now, \ - is_padded, \ - num_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) -#endif - { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); - // Hash each block and store their hashes in the map - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - - const bool is_last_thread = thread_order == num_threads; - - if (is_last_thread) { - last_done.store(true, std::memory_order_release); - } - - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - - { - b_hash_blocks_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : links.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - - BT_ASSERT(links.num_inserts_.load() == links.size()); -# endif -#endif - - // By this point, the map should contain the first occurrences of - // every respective block's content. We then fill the pointers - // and offsets with this data and increment counters accordingly - links.for_each( - [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; - } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); - -#ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scans through block-sized windows starting inside one block and - /// tries to find blocks with matching hashes in the map. Such blocks - /// will have their earliest occurrence update. - /// @param rk A Rabin-Karp hasher whose current state is at a block start. - /// @param links A map whose keys are hashed blocks and the values - /// are all block indices of blocks matching the hash in ascending order. - /// @param level_data The data for the current level. - /// @param current_block_index The index of the block which the - /// Rabin-Karp hasher is situated in. - static void scan_windows_in_block(RabinKarp& rk, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits -#endif - ) { - for (size_type offset = 0; offset < level_data.block_size; - ++offset, rk.next()) { - const RabinKarpHash hash = rk.current_hash(); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { -#ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); -#else - continue; -#endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - - /// @brief Generate the block size, number of block and block start indices - /// for the next level. - /// - /// This depends on the current level's block size, number of blocks and - /// is_internal bit vector being filled. - /// - /// @param text The input text. - /// @param level The level data of the current level. - /// @return The level data of the next level. - [[nodiscard]] LevelData - generate_next_level(const std::vector& text, - const LevelData& level) const { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const auto& is_internal = *level.is_internal; - const size_t next_block_size = block_size / this->tau_; - - std::vector new_block_starts; - new_block_starts.reserve(num_blocks * this->tau_); - for (size_t i = 0; i < num_blocks; ++i) { - if (!is_internal[i]) { - continue; - } - - // We generate up to tau new blocks for each internal block, - // excluding blocks that start past the end of the text - const auto parent_block_start = (*level.block_starts)[i]; - for (size_t j = 0, current_block_start = parent_block_start; - j < static_cast(this->tau_) && - current_block_start < text.size(); - ++j, current_block_start += next_block_size) { - new_block_starts.push_back(current_block_start); - } - } - - LevelData next_level(level.level_index + 1, - next_block_size, - new_block_starts.size()); - next_level.block_starts = - std::make_unique>(std::move(new_block_starts)); - return next_level; - } - - /// - /// @brief Takes a vector of levels and fills the block tree fields with - /// them. - /// - /// @param[in] levels A vector containing data for each level, with the - /// first entry corresponding to the topmost level. - /// - void make_tree(const std::vector& text, - std::vector& levels, - int64_t padding) { - const bool is_padded = padding > 0; - - // Count the current number of internal blocks per level - std::vector new_num_internal(levels.size(), 0); - for (size_t level = 0; level < levels.size(); level++) { - for (size_t block = 0; block < levels[level].is_internal->size(); - block++) { - if ((*levels[level].is_internal)[block]) { - new_num_internal[level]++; - } - } - } - - // Create first level - bool found_back_block = levels[0].is_internal->size() > - static_cast(new_num_internal[0]) || - !this->CUT_FIRST_LEVELS; - LevelData& top_level = levels.front(); - if (found_back_block) { - const size_t n = top_level.num_blocks; - const size_t num_internal = new_num_internal[0]; - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_back_blocks = 0; - for (size_t i = 0; i < n; i++) { - // if a back block is found, add its pointer and offset - if (!(*top_level.is_internal)[i]) { - (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; - (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; - num_back_blocks++; - } - } - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(top_level.is_internal.release()); - this->block_tree_types_rs_.push_back( - new Rank(*this->block_tree_types_.back())); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(top_level.block_size); - } - top_level.pointers.reset(); - top_level.offsets.reset(); - top_level.counters.reset(); - - // Add level data to the tree - for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData& level = levels[level_index]; - LevelData& previous_level = levels[level_index - 1]; - found_back_block |= static_cast(new_num_internal[level_index]) < - levels[level_index].is_internal->size(); - if (!found_back_block) { - level.is_internal.reset(); - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - continue; - } - - make_tree_level(levels, - new_num_internal, - level_index, - is_padded, - text.size()); - - // We don't need these anymore - if (level_index < levels.size() - 1) { - level.is_internal.reset(); - } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - } - - this->leaf_size = levels.back().block_size / this->tau_; - // Construct the leaf string - int64_t leaf_count = 0; - auto& last_is_internal = *levels.back().is_internal; - std::vector& last_block_starts = *levels.back().block_starts; - for (size_t block = 0; block < last_is_internal.size(); block++) { - if (!last_is_internal[block]) { - continue; - } - const size_type block_start = last_block_starts[block]; - // For every leaf on the last level, we have tau leaf blocks - leaf_count += this->tau_; - // Iterate through all characters in this child and - // add them to the leaf string - for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); - b++) { - if (static_cast(block_start + b) < text.size()) { - this->leaves_.push_back(text[block_start + b]); - } - } - } - this->amount_of_leaves = leaf_count; - this->compress_leaves(); - } - - /// @brief Generates a level and adds the relevant data to the block tree. - /// - /// @param levels The vector of levels of the tree. - /// @param level_index The index of the level to generate. This must be - /// strictly greater than 0. - /// @param is_padded Whether there is padding in the last block of the tree - void make_tree_level(std::vector& levels, - const std::vector& new_num_internal, - const size_t level_index, - const bool is_padded, - const size_t text_len) { - LevelData& previous_level = levels[level_index - 1]; - LevelData& level = levels[level_index]; - - size_type new_size = - (new_num_internal[level_index - 1] - is_padded) * this->tau_; - // Determine the number of children the last block generated - if (is_padded) { - const size_type last_block_parent_start = - previous_level.block_starts->back(); - const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); - } - previous_level.block_starts.reset(); - const size_type num_internal = new_num_internal[level_index]; - - // Allocate new vectors for the tree - auto* is_internal = new BitVector(new_size); - auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); - auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); - - // Number of non-pruned blocks before the current block - size_type num_non_pruned = 0; - // Number of back blocks before the current block - size_type num_back_blocks = 0; - // Number of pruned blocks before the current block - size_type num_pruned = 0; - - // We will reuse the allocated memory of the pointers vector to - // store the number of pruned blocks before the block. The - // invariant is that all values up to i are overwritten while all - // values starting after i will still be valid pointers - // This contains the number of pruned blocks before the block i - std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { - const size_type ptr = (*level.pointers)[i]; - prefix_pruned_blocks[i] = num_pruned; - - // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { - num_pruned++; - continue; - } - - // Add it to the is_internal bit vector - const bool block_is_internal = (*level.is_internal)[i]; - (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; - - if (block_is_internal) { - continue; - } - - // If it is a back block, add its pointer and offset - const size_type offset = (*level.offsets)[i]; - - (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; - (*offsets)[num_back_blocks] = offset; - num_back_blocks++; - } - - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(is_internal); - this->block_tree_types_rs_.push_back(new Rank(*is_internal)); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(level.block_size); - } - - /// @brief Prunes the tree of unnecessary nodes. - /// @param levels The levels of the tre represented as a vector of levels. - void prune(std::vector& levels) { - // We need to traverse the block tree in post order, - // handling children from right to left - for (int block_index = levels[0].num_blocks - 1; block_index >= 0; - --block_index) { - prune_block(levels, 0, block_index); - } - } - - /// @brief Prunes a block and its descendants of unnecessary internal nodes. - /// @param levels The WIP levels of the tree. - /// @param level_index The level of the block to prune. - /// @param block_index The index of the block to prune. - /// @return Whether this block is/stays internal after the pruning process - bool prune_block(std::vector& levels, - const size_t level_index, - const size_t block_index) const { - LevelData& level = levels[level_index]; - BitVector& is_internal = *level.is_internal; - - // If the current block is a back block already, there is nothing - // to prune - if (!is_internal[block_index]) { - return false; - } - - const size_type first_child = - level.is_internal_rank->rank1(block_index) * this->tau_; - - bool has_internal_children = false; - - // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are - // not on the last level. - if (level_index < levels.size() - 1) { - const size_type last_child = - std::min(first_child + this->tau_ - 1, - levels[level_index + 1].is_internal->size() - 1); - // Iterate through children in reverse - for (size_type child = last_child; child >= first_child; --child) { - has_internal_children |= prune_block(levels, level_index + 1, child); - } - } - - // If any of the children is internal, this block stays internal - // as well - if (has_internal_children) { - return true; - } - - const size_type pointer = (*level.pointers)[block_index]; - const size_type offset = (*level.offsets)[block_index]; - const size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing - // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { - return true; - } - - // Now we know that there is an earlier occurrence, - // and nothing is pointing here. - // We will make this block here into a back block... - is_internal[block_index] = false; - (*level.counters)[pointer] += 1; - (*level.counters)[pointer + 1] += offset > 0; - - if (level_index == levels.size() - 1) { - return false; - } - - // ...and mark the children as pruned - LevelData& child_level = levels[level_index + 1]; - const size_type last_child = - std::min(first_child + this->tau_ - 1, - child_level.is_internal->size() - 1); - for (size_type child = last_child; child >= first_child; --child) { - const size_type child_pointer = (*child_level.pointers)[child]; - const size_type child_offset = (*child_level.offsets)[child]; -#ifdef BT_DBG - if (!(*child_level.is_internal)[child] && child_pointer < 0) { - std::cout << "non-internal node missing pointer" << std::endl; - std::cout << level_index << ", " << block_index << " / " - << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { - std::cout << "pruned node missing pointer" << std::endl; - } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - BT_ASSERT(child_pointer >= 0); -#endif - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; - } - - return false; - } - -public: - BlockTreeFPParPHF(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads) { - const auto old = omp_get_max_threads(); - const auto old_dynamic = omp_get_dynamic(); - omp_set_dynamic(0); - omp_set_num_threads(static_cast(threads)); - this->tau_ = arity; - this->s_ = root_arity; - this->max_leaf_length_ = max_leaf_length; - this->map_unique_chars(text); - construct(text, threads, 1000); - omp_set_dynamic(old_dynamic); - omp_set_num_threads(old); - } - - ~BlockTreeFPParPHF() { - for (auto& rank : this->block_tree_types_rs_) { - delete rank; - } - for (auto& bv : this->block_tree_types_) { - delete bv; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; - } - } -}; - -} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp index 0fd471e..e8eead5 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp @@ -41,6 +41,7 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { +/* /// @brief Determine whether to use a Rabin-Karp hash for hashing text windows /// or just use the block's content itself as a hash, stored in an integer. enum class UseHash { @@ -49,6 +50,7 @@ enum class UseHash { /// @brief Use the block's content as a hash IDENTITY }; +*/ /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes /// and a sharded hash map. Small blocks are not RK-hashed but rather use the diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 6137bd9..f845be7 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -28,6 +28,7 @@ #include #include #include +#include #include namespace pasta { @@ -42,7 +43,7 @@ template class MersenneHash { public: __extension__ typedef unsigned __int128 uint128_t; - const std::vector* text_; + std::span text_; uint128_t hash_; uint32_t start_; uint32_t length_; @@ -50,12 +51,21 @@ class MersenneHash { const uint128_t hash, const uint64_t start, const uint64_t length) - : text_(&text), + : text_(text), hash_(hash), start_(start), length_(length){}; - constexpr MersenneHash() : text_(nullptr), hash_(0), start_(0), length_(0){}; + MersenneHash(const std::span text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : text_(text), + hash_(hash), + start_(start), + length_(length){}; + + constexpr MersenneHash() : text_(), hash_(0), start_(0), length_(0){}; constexpr MersenneHash(const MersenneHash& other) = default; constexpr MersenneHash(MersenneHash&& other) = default; @@ -74,8 +84,8 @@ class MersenneHash { if (hash_ != other.hash_) return false; - const bool is_same = memcmp(text_->data() + start_, - other.text_->data() + other.start_, + const bool is_same = memcmp(text_.data() + start_, + other.text_.data() + other.start_, length_) == 0; #ifdef BT_INSTRUMENT diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index a16a174..2e99807 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -52,7 +52,7 @@ template class MersenneRabinKarp { public: /// The text being hashed - std::vector const& text_; + std::span text_; uint128_t sigma_; /// The start index of the currently hashed window uint64_t init_; @@ -65,17 +65,17 @@ class MersenneRabinKarp { uint128_t max_sigma_; /// @brief Construct a new Rabin Karp hasher. - /// @param text The text to hash. + /// @param text The text to hash. (not just the window but the entire text) /// @param sigma The alphabet size. /// @param init The start index of the first hashed window in the text. /// @param length The window size. /// @param prime A large prime used for modulus operations /// iff not using mersenne_exponent. - MersenneRabinKarp(std::vector const& text, - uint64_t sigma, - uint64_t init, - uint64_t length, - uint128_t prime) + MersenneRabinKarp(const std::span text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) : text_(text), sigma_(sigma), init_(init), @@ -86,7 +86,7 @@ class MersenneRabinKarp { uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { fp = fp * sigma; - fp = mersenneModulo(fp + text_.at(i)); + fp = mersenneModulo(fp + text_[i]); } for (uint64_t i = 0; i < length_ - 1; i++) { sigma_c = mersenneModulo(sigma_c * sigma_); @@ -95,9 +95,16 @@ class MersenneRabinKarp { max_sigma_ = sigma_c; }; + MersenneRabinKarp(const std::vector& text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : MersenneRabinKarp(std::span(text), sigma, init, length, prime) {} + /// @brief Moves the hasher to the specified start index in the backing /// vector. - void restart(uint64_t index) { + void restart(const uint64_t index) { if (index + length_ >= text_.size()) { return; } @@ -110,7 +117,7 @@ class MersenneRabinKarp { hash_ = fp; }; - inline uint128_t mersenneModulo(uint128_t k) { + inline uint128_t mersenneModulo(uint128_t k) const { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { From e279c823adfd8d3e5cf5b2ae9836e6e0b4540f83 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 20 Nov 2023 23:17:47 +0100 Subject: [PATCH 54/92] make build_bt use a command line parser --- examples/build_bt.cpp | 236 +++++++++--------- .../pasta/block_tree/utils/MersenneHash.hpp | 17 ++ 2 files changed, 137 insertions(+), 116 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index d8c2d80..a4ab274 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -185,6 +185,7 @@ make_bt(std::vector& text, #include #include +#include using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -192,55 +193,60 @@ using Duration = Clock::duration; int main(int argc, char** argv) { using namespace pasta; - if (argc < 2) { - std::cerr << "Please input file" << std::endl; - exit(1); - } - if (!std::filesystem::exists(argv[1])) { - std::cerr << "File " << argv[1] << " does not exist" << std::endl; - exit(1); - } + tlx::CmdlineParser cp; + cp.set_description( + "Build a block tree for a given input text or bit vector."); - if (argc < 3) { - std::cerr << "Please input tree arity (tau)" << std::endl; - exit(1); - } + std::string file; + cp.add_param_string("file", + file, + "The path to the file which to build a block tree from"); - const size_t arity = atoi(argv[2]); + size_t arity = 0; + cp.add_param_size_t("arity", arity, "The arity of the block tree"); + size_t leaf_length = 0; + cp.add_param_size_t( + "leaf", + arity, + "The maximum number of characters saved verbatim per leaf block."); - if (argc < 4) { - std::cerr << "Please input max leaf length" << std::endl; - exit(1); - } - const size_t leaf_length = atoi(argv[3]); + size_t threads = 1; + cp.add_size_t('t', + "threads", + threads, + "The number of threads to use for parallel algorithms (ignored " + "for sequential algorithms)"); + size_t queue_size = 1024; + cp.add_size_t( + 'q', + "queue_size", + queue_size, + "The size of each thread's queue used for sharded hash map algorithms"); -#if IS_PARALLEL - if (argc < 5) { - std::cerr << "Please input number of threads (ignored if single threaded " - "algorithm)" - << std::endl; - exit(1); - } + bool make_bv = false; + cp.add_bool('b', + "bitvec", + make_bv, + "Whether to interpret the input as a bitvector. If \"-o\" is not " + "used, each byte of the input file will be interpreted as 8 bits " + "of the bit vector respectively. In each byte, the least " + "significant bit is index 0."); - const size_t threads = atoi(argv[4]); -#else - const size_t threads = 1; -#endif + std::string one_chars = ""; + cp.add_string( + 'o', + "one_chars", + one_chars, + "A string to be used with the \"-b\" flag. If used, each character of " + "the input represents one bit of the bit vector. It will be a 1 if this " + "parameter string contains the respective character, 0 otherwise."); -#if USES_QUEUE - if (argc < 6) { - std::cerr << "Please input queue size" << std::endl; - exit(1); + if (!cp.process(argc, argv)) { + return 1; } - const size_t queue_size = atoi(argv[5]); -#else - const size_t queue_size = 0; -#endif - std::stringstream ss; - ss << argv[1] << "_arit" << arity << "_leaf" << leaf_length << "_new.bt"; - std::string out_path = ss.str(); + std::cout << one_chars << std::endl; #ifdef BT_DBG std::cout << "building block tree with parameters:" @@ -249,103 +255,101 @@ int main(int argc, char** argv) { << std::endl; #endif -#ifdef BIT_BT pasta::BitVector bv; -#else std::vector text; -#endif { std::string input; std::ifstream t(argv[1]); std::stringstream buffer; buffer << t.rdbuf(); input = buffer.str(); -#ifdef BIT_BT - new (&bv) pasta::BitVector(input.size()); - for (size_t i = 0; i < input.size(); ++i) { - bv[i] = input[i] == 'G' || input[i] == 'A'; + if (make_bv) { + if (one_chars.empty()) { + // Interpret each character as 8 bits + new (&bv) pasta::BitVector(input.size() * 8); + std::span bytes = std::as_writable_bytes(bv.data()); + for (size_t i = 0; i < input.size(); ++i) { + bytes[i] = std::byte{static_cast(input[i])}; + } + } else { + // Interpret each character as a bit + new (&bv) pasta::BitVector(input.size()); + std::array is_one; + for (char c : one_chars) { + is_one[static_cast(c)] = true; + } + for (size_t i = 0; i < input.size(); ++i) { + bv[i] = is_one[static_cast(input[i])]; + } + } + } else { + text = std::vector(input.begin(), input.end()); } -#else - text = std::vector(input.begin(), input.end()); -#endif } std::cout << "RESULT algo=" << ALGO_NAME - << " file=" << std::filesystem::path(argv[1]).filename().string() -#ifdef BIT_BT - << " bv_size=" << bv.size() -#else - << " file_size=" << text.size() -#endif - << " threads=" << threads << " arity=" << arity + << " file=" << std::filesystem::path(file).filename().string(); + if (make_bv) { + std::cout << " bv_size=" << bv.size(); + } else { + std::cout << " file_size=" << text.size(); + } + std::cout << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); - // auto bt = make_bt(text, arity, leaf_length, threads, queue_size); - auto bt = std::make_unique>(bv, - arity, - 20, - leaf_length, - threads, - queue_size); - auto elapsed = - std::chrono::duration_cast(Clock::now() - now) - .count(); - - std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); - std::cout << std::endl; + if (make_bv) { + // Make bit vector block tree + auto bt = std::make_unique>(bv, + arity, + 1, + leaf_length, + threads, + queue_size); + auto elapsed = std::chrono::duration_cast( + Clock::now() - now) + .count(); + std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); + std::cout << std::endl; -#ifdef BT_INSTRUMENT - std::cout << "comparisons: " << mersenne_hash_comparisons - << ", equals: " << mersenne_hash_equals - << ", collisions: " << mersenne_hash_collisions - << ", percent equals: " - << 100 * mersenne_hash_equals / ((double)mersenne_hash_comparisons) - << ", percent collisions: " - << 100 * mersenne_hash_collisions / - ((double)mersenne_hash_comparisons) - << std::endl; +#if defined BT_INSTRUMENT && defined BT_DBG + pasta::print_hash_data(); #endif - // std::ofstream ot(out_path); - // bt->serialize(ot); -#ifdef BIT_BT -#else -# pragma omp parallel for - for (size_t i = 0; i < text.size(); ++i) { - const auto c = bt->access(i); - if (c != text[i]) { - std::osyncstream(std::cerr) - << "Error at position " << i - << "\nExpected: " << static_cast(text[i]) - << "\nActual: " << static_cast(c) << std::endl; - exit(1); +#pragma omp parallel for + for (size_t i = 0; i < bv.size(); ++i) { + const bool c = bt->access(i); + if (c != bv[i]) { + std::osyncstream(std::cerr) + << "Error at position " << i << "\nExpected: " << std::boolalpha + << bv[i] << "\nActual: " << c << std::noboolalpha << std::endl; + exit(1); + } } - } + } else { + // Make text block tree + auto bt = make_bt(text, arity, leaf_length, threads, queue_size); + auto elapsed = std::chrono::duration_cast( + Clock::now() - now) + .count(); + + std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); + std::cout << std::endl; + +#if defined BT_INSTRUMENT && defined BT_DBG + pasta::print_hash_data(); #endif - // ot.close(); - /* - for (size_t i = 0; i < bv.size() / 8; i++) { - uint8_t b = 0; - for (char j = 0; j < 8; j++) { - b |= bt->access(i * 8 + j) << j; - } - std::cout << i << ": "; - std::cout << std::flush - << std::bitset<8>{static_cast( - std::as_bytes(bv.data())[i])} - << " "; - std::cout << std::bitset<8>{b} << std::endl; - } - */ + #pragma omp parallel for - for (size_t i = 0; i < bv.size(); ++i) { - const bool c = bt->access(i); - if (c != bv[i]) { - std::osyncstream(std::cerr) - << "Error at position " << i << "\nExpected: " << std::boolalpha - << bv[i] << "\nActual: " << c << std::noboolalpha << std::endl; - exit(1); + for (size_t i = 0; i < text.size(); ++i) { + const auto c = bt->access(i); + if (c != text[i]) { + std::osyncstream(std::cerr) + << "Error at position " << i + << "\nExpected: " << static_cast(text[i]) + << "\nActual: " << static_cast(c) << std::endl; + exit(1); + } } } diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index f845be7..2ff6962 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -31,14 +31,31 @@ #include #include +#ifdef BT_INSTRUMENT +# include +#endif + namespace pasta { #ifdef BT_INSTRUMENT static std::atomic_size_t mersenne_hash_comparisons = 0; static std::atomic_size_t mersenne_hash_equals = 0; static std::atomic_size_t mersenne_hash_collisions = 0; + +void print_hash_data() { + std::cout << "comparisons: " << mersenne_hash_comparisons + << ", equals: " << mersenne_hash_equals + << ", collisions: " << mersenne_hash_collisions + << ", percent equals: " + << 100 * mersenne_hash_equals / ((double)mersenne_hash_comparisons) + << ", percent collisions: " + << 100 * mersenne_hash_collisions / + ((double)mersenne_hash_comparisons) + << std::endl; +} #endif + template class MersenneHash { public: From f8a2ab4851a72e8970c510e4a9b0d6c7e841b65b Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 24 Nov 2023 01:29:06 +0100 Subject: [PATCH 55/92] bit rank queries --- examples/build_bt.cpp | 28 +- include/pasta/block_tree/bit_block_tree.hpp | 659 +++++++++++------- .../construction/bit_block_tree_sharded.hpp | 17 +- include/pasta/block_tree/utils/byteread.hpp | 57 ++ include/pasta/block_tree/utils/concepts.hpp | 30 +- 5 files changed, 503 insertions(+), 288 deletions(-) create mode 100644 include/pasta/block_tree/utils/byteread.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index a4ab274..b0ab593 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -20,7 +20,6 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" -#include #include #include #include @@ -208,7 +207,7 @@ int main(int argc, char** argv) { size_t leaf_length = 0; cp.add_param_size_t( "leaf", - arity, + leaf_length, "The maximum number of characters saved verbatim per leaf block."); size_t threads = 1; @@ -246,8 +245,6 @@ int main(int argc, char** argv) { return 1; } - std::cout << one_chars << std::endl; - #ifdef BT_DBG std::cout << "building block tree with parameters:" << "\narity: " << arity << "\nmax leaf length: " << leaf_length @@ -274,7 +271,7 @@ int main(int argc, char** argv) { } else { // Interpret each character as a bit new (&bv) pasta::BitVector(input.size()); - std::array is_one; + std::array is_one{}; for (char c : one_chars) { is_one[static_cast(c)] = true; } @@ -288,14 +285,14 @@ int main(int argc, char** argv) { } std::cout << "RESULT algo=" << ALGO_NAME - << " file=" << std::filesystem::path(file).filename().string(); + << " file=" << std::filesystem::path(file).filename().string() + << " threads=" << threads << " arity=" << arity + << " leaf_length=" << leaf_length; if (make_bv) { std::cout << " bv_size=" << bv.size(); } else { std::cout << " file_size=" << text.size(); } - std::cout << " threads=" << threads << " arity=" << arity - << " leaf_length=" << leaf_length; TimePoint now = Clock::now(); if (make_bv) { @@ -326,6 +323,21 @@ int main(int argc, char** argv) { exit(1); } } + + bt->add_bit_rank_support(); + size_t num_ones = 0; + for (size_t i = 0; i < bv.size(); i++) { + const size_t rank = bt->rank1(i); + if (num_ones != rank) { + std::osyncstream(std::cerr) + << "Error at position " << i << "\nExpected: " << num_ones + << "\nActual: " << rank << std::endl; + throw std::runtime_error("oof"); + } + + num_ones += bv[i]; + } + } else { // Make text block tree auto bt = make_bt(text, arity, leaf_length, threads, queue_size); diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 68c68fe..7dc7765 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -22,6 +22,8 @@ #pragma once #include +#include +#include #include #include #include @@ -31,14 +33,13 @@ #include #include #include -#include namespace pasta { template class BitBlockTree { public: - /// @brief If this is true, then the only levels of the tree start to be + /// If this is true, then the only levels of the tree start to be /// included starting at the first level that contains a back block /// /// For example, if levels 0 to 5 do not contain any back blocks, then the @@ -46,21 +47,23 @@ class BitBlockTree { bool CUT_FIRST_LEVELS = true; using BVStoreType = pasta::BitVector::RawDataType; - constexpr static size_t BV_STORE_TYPE_BITS = sizeof(BVStoreType) * 8; - constexpr static size_t BV_STORE_TYPE_BYTES = sizeof(BVStoreType); + /// The arity of the tree size_type tau_; size_type max_leaf_length_; + /// The arity of the tree's root size_type s_ = 1; size_type leaf_size = 0; size_type amount_of_leaves = 0; size_type num_bits; bool rank_support = false; - /// \brief Bit vectors for each level determining whether a block is internal + /// Bit vectors for each level determining whether a block is internal /// (=1) or not (=0) std::vector block_tree_types_; std::vector*> block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source std::vector*> block_tree_pointers_; std::vector*> block_tree_offsets_; // std::vector*> block_tree_encoded_; @@ -74,42 +77,51 @@ class BitBlockTree { ankerl::unordered_dense::map chars_index_; std::vector chars_; - size_type u_chars_; std::vector>> c_ranks_; std::vector>> pointer_c_ranks_; - bool access(const size_type bit_index) { + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + bool access(const size_type bit_index) const { // FIXME: As of now this works on little endian systems only const int64_t byte_index = bit_index / 8; const int64_t bit_offset = bit_index % 8; int64_t block_size = block_size_lvl_[0]; - int64_t blk_pointer = byte_index / block_size_lvl_[0]; - int64_t off = byte_index % block_size_lvl_[0]; - int64_t child; - for (size_type i = 0; static_cast(i) < block_tree_types_.size(); - i++) { - auto& lvl = *block_tree_types_[i]; - auto& lvl_rs = *block_tree_types_rs_[i]; - auto& lvl_ptr = *block_tree_pointers_[i]; - auto& lvl_off = *block_tree_offsets_[i]; - if (lvl[blk_pointer] == 0) { - size_type blk = lvl_rs.rank0(blk_pointer); - off = off + lvl_off[blk]; - blk_pointer = lvl_ptr[blk]; + int64_t block_index = byte_index / block_size; + int64_t off = byte_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; if (off >= block_size) { - blk_pointer++; + ++block_index; off -= block_size; } } block_size /= tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; } const uint8_t byte = - decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; - //std::cout << std::bitset<8>(byte) << std::endl; + decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; return ((1 << bit_offset) & byte) != 0; }; @@ -167,7 +179,7 @@ class BitBlockTree { } } uint64_t i = 1; - while (i < block_tree_types_.size()) { + while (i < height()) { auto& current_level = *block_tree_types_[i]; auto& current_level_rs = *block_tree_types_rs_[i]; auto& current_level_ptr = *block_tree_pointers_[i]; @@ -212,169 +224,116 @@ class BitBlockTree { return s + l; } - int64_t rank_base(uint8_t c, size_type index) { - pasta::BitVector& top_level = *block_tree_types_[0]; - auto& top_level_rs = *block_tree_types_rs_[0]; - auto& top_level_ptr = *block_tree_pointers_[0]; - auto& top_level_off = *block_tree_offsets_[0]; - int64_t c_index = chars_index_[c]; - int64_t block_size = block_size_lvl_[0]; - int64_t blk_pointer = index / block_size; - int64_t off = index % block_size; - int64_t rank = - (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; - int64_t child = 0; - if (top_level[blk_pointer]) { - block_size /= tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; - } else { - size_type blk = top_level_rs.rank0(blk_pointer); - rank -= pointer_c_ranks_[c_index][0][blk]; - size_type to = off + top_level_off[blk]; - off = off + top_level_off[blk]; - blk_pointer = top_level_ptr[blk]; - child = blk_pointer; - if (to >= block_size) { - int64_t adder = (child == 0) ? - c_ranks_[c_index][0][blk_pointer] : - c_ranks_[c_index][0][blk_pointer] - - c_ranks_[c_index][0][blk_pointer - 1]; - rank += adder; - blk_pointer++; - off = to - block_size; - } - block_size = block_size / tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; - } - // we first calculate the - uint64_t i = 1; - while (i < block_tree_types_.size()) { - rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; - if ((*block_tree_types_[i])[blk_pointer]) { - size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + /// @brief Counts the number of 1-bits up to (and excluding) an index. + size_t rank1(const size_type bit_index) const { + const size_t byte_index = bit_index / 8; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = byte_index / block_size; + size_t block_offset = byte_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { block_size /= tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = rank_blk * tau_ + child; - i++; - } else { - size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); - rank -= pointer_c_ranks_[c_index][i][blk]; - size_type ptr_off = (*block_tree_offsets_[i])[blk]; - size_type to = off + ptr_off; - off = off + ptr_off; - blk_pointer = (*block_tree_pointers_[i])[blk]; - child = blk_pointer % tau_; - - if (to >= block_size) { - auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : - c_ranks_[c_index][i][blk_pointer] - - c_ranks_[c_index][i][blk_pointer - 1]; - rank += adder; - blk_pointer++; - child = blk_pointer % tau_; - off = to - block_size; - } - auto remove_prefix = - (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; - rank -= remove_prefix; - } - } - size_type prefix_leaves = blk_pointer - child; - for (int j = 0; j < child * leaf_size; j++) { - if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == - compress_map_[c]) - rank++; - } - for (int j = 0; j <= off; j++) { - if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) - rank++; - } + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + + block]]; + rank += std::popcount(byte); + } + for (size_t block = 0; block < block_offset; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; + rank += std::popcount(byte); + } + + // Masks to remove bits from the last byte, + // that aren't part of the ran query + static constexpr std::array MASKS = { + 0b0000'0000, + 0b0000'0001, + 0b0000'0011, + 0b0000'0111, + 0b0000'1111, + 0b0001'1111, + 0b0011'1111, + 0b0111'1111, + }; + rank += std::popcount( + decompress_map_[compressed_leaves_[block_index * leaf_size + + block_offset]] & + MASKS[bit_index % 8]); return rank; } - int64_t rank(uint8_t c, size_type index) { - pasta::BitVector& top_level = *block_tree_types_[0]; - auto& top_level_rs = *block_tree_types_rs_[0]; - auto& top_level_ptr = *block_tree_pointers_[0]; - auto& top_level_off = *block_tree_offsets_[0]; - int64_t c_index = chars_index_[c]; - int64_t block_size = block_size_lvl_[0]; - int64_t blk_pointer = index / block_size; - int64_t off = index % block_size; - int64_t rank = - (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; - int64_t child = 0; - if (top_level[blk_pointer]) { - block_size /= tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; - } else { - size_type blk = top_level_rs.rank0(blk_pointer); - rank -= pointer_c_ranks_[c_index][0][blk]; - off = off + top_level_off[blk]; - blk_pointer = top_level_ptr[blk]; - child = blk_pointer; - if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : - c_ranks_[c_index][0][blk_pointer] - - c_ranks_[c_index][0][blk_pointer - 1]; - blk_pointer++; - off = off - block_size; - } - block_size = block_size / tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; - } - // we first calculate the - uint64_t i = 1; - while (i < block_tree_types_.size()) { - rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; - if ((*block_tree_types_[i])[blk_pointer]) { - size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); - block_size /= tau_; - child = off / block_size; - off = off % block_size; - blk_pointer = rank_blk * tau_ + child; - i++; - } else { - size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); - rank -= pointer_c_ranks_[c_index][i][blk]; - size_type ptr_off = (*block_tree_offsets_[i])[blk]; - off = off + ptr_off; - blk_pointer = (*block_tree_pointers_[i])[blk]; - child = blk_pointer % tau_; - if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : - c_ranks_[c_index][i][blk_pointer] - - c_ranks_[c_index][i][blk_pointer - 1]; - blk_pointer++; - child = blk_pointer % tau_; - off = off - block_size; - } - auto remove_prefix = - (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; - rank -= remove_prefix; - } - } - size_type prefix_leaves = blk_pointer - child; - for (int j = 0; j < child * leaf_size; j++) { - if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == - compress_map_[c]) - rank++; - } - for (int j = 0; j <= off; j++) { - if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) - rank++; - } - return rank; - }; + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } int64_t print_space_usage() { int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + @@ -417,27 +376,49 @@ class BitBlockTree { return space_usage; }; - void compress_leaves() { - // Holds a 1 on every char that exists - compress_map_.resize(256, 0); - decompress_map_.resize(256, 0); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compress_map_[this->leaves_[i]] = 1; + int32_t add_bit_rank_support() { + rank_support = true; + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); } - for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { - const size_t tmp = compress_map_[c]; - compress_map_[c] = cur_val; - decompress_map_[cur_val] = c; - cur_val += tmp; + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); } - compressed_leaves_.resize(this->leaves_.size()); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); } - sdsl::util::bit_compress(this->compressed_leaves_); - leaves_.resize(0); - leaves_.shrink_to_fit(); + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + return 0; } int32_t add_rank_support() { @@ -445,7 +426,7 @@ class BitBlockTree { c_ranks_.resize(chars_.size(), std::vector>()); pointer_c_ranks_.resize(chars_.size(), std::vector>()); for (uint64_t i = 0; i < c_ranks_.size(); i++) { - c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + c_ranks_[i].resize(height(), sdsl::int_vector<0>()); for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { c_ranks_[i][j].resize(block_tree_types_[j]->size()); } @@ -468,7 +449,7 @@ class BitBlockTree { max = c_ranks_[chars_index_[c]][0][i]; } } - for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + for (uint64_t i = 1; i < height(); i++) { size_type counter = tau_; size_type acc = 0; for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { @@ -497,7 +478,7 @@ class BitBlockTree { c_ranks_.resize(chars_.size(), std::vector>()); pointer_c_ranks_.resize(chars_.size(), std::vector>()); for (uint64_t i = 0; i < c_ranks_.size(); i++) { - c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + c_ranks_[i].resize(height(), sdsl::int_vector<0>()); for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { c_ranks_[i][j].resize(block_tree_types_[j]->size()); } @@ -523,7 +504,7 @@ class BitBlockTree { max = c_ranks_[chars_index_[c]][0][i]; } } - for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + for (uint64_t i = 1; i < height(); i++) { size_type counter = tau_; size_type acc = 0; for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { @@ -547,15 +528,38 @@ class BitBlockTree { return 0; } +protected: + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } /// @brief Calculate the number of leading zeros for a 32-bit integer. /// This value is capped at 31. - inline size_type leading_zeros(int32_t val) { + static size_type leading_zeros(const int32_t val) { return __builtin_clz(static_cast(val) | 1); } /// @brief Calculate the number of leading zeros for a 64-bit integer. /// This value is capped at 64. - inline size_type leading_zeros(int64_t val) { + static size_type leading_zeros(const int64_t val) { return __builtin_clzll(static_cast(val) | 1); } @@ -603,60 +607,187 @@ class BitBlockTree { padding = tmp_padding - text_length; } - size_type rank_block(uint8_t c, size_type i, size_type j) { - if (static_cast(j) >= block_tree_types_[i]->size()) { + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + // TODO: Handle the case where blocks are not internal + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + /// + /// @brief Generates rank information for a block and all its children + /// recursively. + /// + /// @param c The character to generate rank information for. + /// @param level The level index the block is on. + /// @param block_index The index of the block on this level + size_type rank_block(uint8_t c, size_type level, size_type block_index) { + if (static_cast(block_index) >= + block_tree_types_[level]->size()) { return 0; } size_type rank_c = 0; - if ((*block_tree_types_[i])[j] == 1) { - if (static_cast(i) != block_tree_types_.size() - 1) { - size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + if ((*block_tree_types_[level])[block_index] == 1) { + if (static_cast(level) != height() - 1) { + size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); for (size_type k = 0; k < tau_; k++) { - rank_c += rank_block(c, i + 1, rank_blk * tau_ + k); + rank_c += rank_block(c, level + 1, rank_blk * tau_ + k); } } else { - size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); for (size_type k = 0; k < tau_; k++) { rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); } } } else { - size_type rank_0 = block_tree_types_rs_[i]->rank0(j); - size_type ptr = (*block_tree_pointers_[i])[rank_0]; - size_type off = (*block_tree_offsets_[i])[rank_0]; + size_type rank_0 = block_tree_types_rs_[level]->rank0(block_index); + size_type ptr = (*block_tree_pointers_[level])[rank_0]; + size_type off = (*block_tree_offsets_[level])[rank_0]; size_type rank_g = 0; - rank_c += c_ranks_[chars_index_[c]][i][ptr]; + rank_c += c_ranks_[chars_index_[c]][level][ptr]; if (off != 0) { - rank_g = part_rank_block(c, i, ptr, off); - size_type rank_2nd = part_rank_block(c, i, ptr + 1, off); + rank_g = part_rank_block(c, level, ptr, off); + size_type rank_2nd = part_rank_block(c, level, ptr + 1, off); rank_c -= rank_g; rank_c += rank_2nd; } - pointer_c_ranks_[chars_index_[c]][i][rank_0] = rank_g; + pointer_c_ranks_[chars_index_[c]][level][rank_0] = rank_g; } - c_ranks_[chars_index_[c]][i][j] = rank_c; + c_ranks_[chars_index_[c]][level][block_index] = rank_c; return rank_c; } - size_type part_rank_block(uint8_t c, size_type i, size_type j, size_type g) { - if (static_cast(j) >= block_tree_types_[i]->size()) { + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + k++) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + size_type part_rank_block(uint8_t c, + size_type level, + size_type block_index, + size_type g) { + if (static_cast(block_index) >= + block_tree_types_[level]->size()) { return 0; } size_type rank_c = 0; - if ((*block_tree_types_[i])[j] == 1) { - if (static_cast(i) != block_tree_types_.size() - 1) { - size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + if ((*block_tree_types_[level])[block_index] == 1) { + if (static_cast(level) != height() - 1) { + size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); size_type k = 0; size_type k_sum = 0; - for (k = 0; k < tau_ && k_sum + block_size_lvl_[i + 1] <= g; k++) { - rank_c += c_ranks_[chars_index_[c]][i + 1][rank_blk * tau_ + k]; - k_sum += block_size_lvl_[i + 1]; + for (k = 0; k < tau_ && k_sum + block_size_lvl_[level + 1] <= g; k++) { + rank_c += c_ranks_[chars_index_[c]][level + 1][rank_blk * tau_ + k]; + k_sum += block_size_lvl_[level + 1]; } if (k_sum != g) { - rank_c += part_rank_block(c, i + 1, rank_blk * tau_ + k, g - k_sum); + rank_c += + part_rank_block(c, level + 1, rank_blk * tau_ + k, g - k_sum); } } else { - size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); size_type k = 0; size_type k_sum = 0; for (k = 0; k < tau_ && k_sum + leaf_size <= g; k++) { @@ -669,20 +800,47 @@ class BitBlockTree { } } } else { - size_type rank_0 = block_tree_types_rs_[i]->rank0(j); - size_type ptr = (*block_tree_pointers_[i])[rank_0]; - size_type off = (*block_tree_offsets_[i])[rank_0]; - if (g + off >= block_size_lvl_[i]) { - rank_c += c_ranks_[chars_index_[c]][i][ptr] - - pointer_c_ranks_[chars_index_[c]][i][rank_0] + - part_rank_block(c, i, ptr + 1, g + off - block_size_lvl_[i]); + size_type rank_0 = block_tree_types_rs_[level]->rank0(block_index); + size_type ptr = (*block_tree_pointers_[level])[rank_0]; + size_type off = (*block_tree_offsets_[level])[rank_0]; + if (g + off >= block_size_lvl_[level]) { + rank_c += c_ranks_[chars_index_[c]][level][ptr] - + pointer_c_ranks_[chars_index_[c]][level][rank_0] + + part_rank_block(c, + level, + ptr + 1, + g + off - block_size_lvl_[level]); } else { - rank_c += part_rank_block(c, i, ptr, g + off) - - pointer_c_ranks_[chars_index_[c]][i][rank_0]; + rank_c += part_rank_block(c, level, ptr, g + off) - + pointer_c_ranks_[chars_index_[c]][level][rank_0]; } } return rank_c; } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + const uint8_t byte = + decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; + result += std::popcount(byte); + } + return result; + } + size_type rank_leaf(uint8_t c, size_type leaf_index, size_type i) { if (static_cast(leaf_index * leaf_size) >= compressed_leaves_.size()) { @@ -700,19 +858,6 @@ class BitBlockTree { return result; } - size_type map_unique_chars(const std::vector& text) { - this->u_chars_ = 0; - uint8_t i = 0; - for (auto a : text) { - if (chars_index_.find(a) == chars_index_.end()) { - chars_index_[a] = i; - i++; - chars_.push_back(a); - } - } - this->u_chars_ = i; - return 0; - }; size_type find_next_smallest_index_binary_search(size_type i, std::vector& pVector) { @@ -742,7 +887,7 @@ class BitBlockTree { size_type blk_pointer = index / block_size; size_type off = index % block_size; size_type child = 0; - for (size_type i = 0; i < this->block_tree_types_.size(); i++) { + for (size_type i = 0; i < this->height(); i++) { if ((*this->block_tree_types_[i])[blk_pointer] == 0) { return -1; } diff --git a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp index 9c7b94b..258023e 100644 --- a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp @@ -24,6 +24,7 @@ #include "pasta/block_tree/bit_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include @@ -665,7 +666,7 @@ class BitBlockTreeSharded : public BitBlockTree { const size_t block_start = block_starts[i]; const uint8_t* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & HASH_MASK); + pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start, @@ -890,8 +891,7 @@ class BitBlockTreeSharded : public BitBlockTree { const uint8_t* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK); + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash current_hash(text, mix_select(hash_value), block_start + offset, @@ -1009,7 +1009,7 @@ class BitBlockTreeSharded : public BitBlockTree { const size_t block_start = block_starts[i]; const uint8_t* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & HASH_MASK); + pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start, @@ -1205,8 +1205,7 @@ class BitBlockTreeSharded : public BitBlockTree { const uint8_t* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK); + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start + offset, @@ -1332,8 +1331,10 @@ class BitBlockTreeSharded : public BitBlockTree { LevelData& previous_level = levels[level_index - 1]; found_back_block |= static_cast(new_num_internal[level_index]) < levels[level_index].is_internal->size(); - if (!found_back_block) { - level.is_internal.reset(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); diff --git a/include/pasta/block_tree/utils/byteread.hpp b/include/pasta/block_tree/utils/byteread.hpp new file mode 100644 index 0000000..52da16e --- /dev/null +++ b/include/pasta/block_tree/utils/byteread.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "concepts.hpp" + +namespace pasta { + +/// @brief Swaps the bytes in an integer. +template +Int byteswap(Int& i) { + switch (sizeof(Int)) { + case 16: + __bswap_16(i); + break; + case 32: + __bswap_32(i); + break; + case 64: + __bswap_64(i); + break; + default: { + } + } + return i; +} + +/// @brief Copies an integer from a pointer using the architecture's native +/// endianness. +/// @param ptr The pointer to copy from. +template +Int copy_ne(const void* const ptr) { + Int i; + memcpy(&i, ptr, sizeof(Int)); + return i; +} + +/// @brief Copies a little endian integer from a pointer. +/// @param ptr The pointer to copy from. +template +Int copy_le(const void* const ptr) { + Int i = copy_ne(ptr); +#if __BYTE_ORDER == __BIG_ENDIAN + byteswap(i); +#endif + return i; +} + +/// @brief Copies a big endian integer from a pointer. +/// @param ptr The pointer to copy from. +template +Int copy_be(const void* const ptr) { + Int i = copy_ne(ptr); +#if __BYTE_ORDER == __LITTLE_ENDIAN + byteswap(i); +#endif + return i; +} +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/concepts.hpp b/include/pasta/block_tree/utils/concepts.hpp index 8ebd2a7..a008698 100644 --- a/include/pasta/block_tree/utils/concepts.hpp +++ b/include/pasta/block_tree/utils/concepts.hpp @@ -1,3 +1,4 @@ +#pragma once #include #include @@ -11,23 +12,22 @@ namespace pasta { /// @tparam Fn The type of the update function. /// @tparam K The key type saved in the hash map. /// @tparam V The value type saved in the hash map. -/// @tparam InputV The type that the update function accepts. This is not -/// required to be the same as the map's value type. /// template -concept UpdateFunction = requires(const K& k, V& v_lv, Fn::InputValue in_v_rv) { - typename Fn::InputValue; - // Updates a pre-existing value in the map. - // Arguments are the key, the value in the map, - // and the input value used to update the value in - // the map - { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; - // Initialize a value from an input value - // Arguments are the key, and the value used to - // initialize the value in the map. This returns the - // value to be inserted into the map - { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; -}; +concept UpdateFunction = + requires(const K& k, V& v_lv, typename Fn::InputValue in_v_rv) { + typename Fn::InputValue; + // Updates a pre-existing value in the map. + // Arguments are the key, the value in the map, + // and the input value used to update the value in + // the map + { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; + // Initialize a value from an input value + // Arguments are the key, and the value used to + // initialize the value in the map. This returns the + // value to be inserted into the map + { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; + }; namespace internal { using Capacity = size_t; From 0660f3128a660d95877888eaefeb980b1854f63c Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sun, 26 Nov 2023 02:11:39 +0100 Subject: [PATCH 56/92] WIP bit select queries --- examples/build_bt.cpp | 32 +++-- include/pasta/block_tree/bit_block_tree.hpp | 130 ++++++++++++++++++++ 2 files changed, 154 insertions(+), 8 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index b0ab593..d452496 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -318,24 +318,40 @@ int main(int argc, char** argv) { const bool c = bt->access(i); if (c != bv[i]) { std::osyncstream(std::cerr) - << "Error at position " << i << "\nExpected: " << std::boolalpha - << bv[i] << "\nActual: " << c << std::noboolalpha << std::endl; + << "Access error at position " << i + << "\nExpected: " << std::boolalpha << bv[i] << "\nActual: " << c + << std::noboolalpha << std::endl; exit(1); } } bt->add_bit_rank_support(); - size_t num_ones = 0; + FlatRankSelect<> frs(bv); +#pragma omp parallel for for (size_t i = 0; i < bv.size(); i++) { - const size_t rank = bt->rank1(i); - if (num_ones != rank) { + const size_t bt_rank = bt->rank1(i); + const size_t bv_rank = frs.rank1(i); + if (bv_rank != bt_rank) { std::osyncstream(std::cerr) - << "Error at position " << i << "\nExpected: " << num_ones - << "\nActual: " << rank << std::endl; + << "Rank error at position " << i << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; throw std::runtime_error("oof"); } + } - num_ones += bv[i]; + const size_t num_ones = frs.rank1(bv.size()); + std::cout << "num ones: " << num_ones << std::endl; + +#pragma omp parallel for + for (size_t i = 1; i <= num_ones; i++) { + const size_t bv_rank = frs.select1(i); + const size_t bt_rank = bt->select_one(i); + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) + << "Select error at position " << i << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } } } else { diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 7dc7765..f9dfcee 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -125,6 +125,136 @@ class BitBlockTree { return ((1 << bit_offset) & byte) != 0; }; +private: + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + const size_t current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + if (current_rank < rank) { + if (start + 1 == end) { + // If there is only one block left, it's either the current or the + // next block + if (top_one_ranks[middle] < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + [[nodiscard("select result discarded")]] ssize_t + select_one(size_t rank) const { + const size_t start_rank = rank; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += block_size - offset; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + size_t k = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == k) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - k) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t l = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + l / 8]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + l += 8; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + l++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + (void)start_rank; + // const auto s_signed = static_cast(s); + // std::cout << start_rank << " -> s: " << s_signed << ", l: " << l + // << std::endl; + // return static_cast((s_signed >= 0 ? s_signed * 8 : s_signed) + + // l); + return pos + l; + } + int64_t select(uint8_t c, size_type j) { auto c_index = chars_index_[c]; auto& top_level = *block_tree_types_[0]; From 15735a72b7389105845f376db02f4f9fd27aa18a Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 27 Nov 2023 18:20:06 +0100 Subject: [PATCH 57/92] fix bit select queries --- examples/build_bt.cpp | 3 +-- include/pasta/block_tree/bit_block_tree.hpp | 7 ++++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index d452496..f5792d1 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -324,13 +324,13 @@ int main(int argc, char** argv) { exit(1); } } - bt->add_bit_rank_support(); FlatRankSelect<> frs(bv); #pragma omp parallel for for (size_t i = 0; i < bv.size(); i++) { const size_t bt_rank = bt->rank1(i); const size_t bv_rank = frs.rank1(i); + if (bv_rank != bt_rank) { std::osyncstream(std::cerr) << "Rank error at position " << i << "\nExpected: " << bv_rank @@ -340,7 +340,6 @@ int main(int argc, char** argv) { } const size_t num_ones = frs.rank1(bv.size()); - std::cout << "num ones: " << num_ones << std::endl; #pragma omp parallel for for (size_t i = 1; i <= num_ones; i++) { diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index f9dfcee..299a700 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -181,11 +181,11 @@ class BitBlockTree { rank_d -= pointer_prefix_one_counts_[0][back_block_index]; if (rank > rank_d) { rank -= rank_d; - pos += block_size - offset; + pos += (block_size - offset) * 8; ++current_block; } else { rank += pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset; + pos -= offset * 8; } } @@ -355,7 +355,8 @@ class BitBlockTree { } /// @brief Counts the number of 1-bits up to (and excluding) an index. - size_t rank1(const size_type bit_index) const { + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { const size_t byte_index = bit_index / 8; const auto& top_is_internal = *block_tree_types_[0]; const auto& top_is_internal_rank = *block_tree_types_rs_[0]; From d7d13813ddb98187e1fec4993cfacb51feda1e01 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 27 Nov 2023 20:11:30 +0100 Subject: [PATCH 58/92] select 0 queries --- examples/build_bt.cpp | 22 ++- include/pasta/block_tree/bit_block_tree.hpp | 142 ++++++++++++++++++-- 2 files changed, 147 insertions(+), 17 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index f5792d1..fc88a95 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -308,11 +308,12 @@ int main(int argc, char** argv) { .count(); std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); std::cout << std::endl; + bt->add_bit_rank_support(); + FlatRankSelect<> frs(bv); #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif - #pragma omp parallel for for (size_t i = 0; i < bv.size(); ++i) { const bool c = bt->access(i); @@ -324,8 +325,6 @@ int main(int argc, char** argv) { exit(1); } } - bt->add_bit_rank_support(); - FlatRankSelect<> frs(bv); #pragma omp parallel for for (size_t i = 0; i < bv.size(); i++) { const size_t bt_rank = bt->rank1(i); @@ -338,16 +337,29 @@ int main(int argc, char** argv) { throw std::runtime_error("oof"); } } + const size_t num_zeros = frs.rank0(bv.size()); + +#pragma omp parallel for + for (size_t i = 1; i <= num_zeros; i++) { + const size_t bv_rank = frs.select0(i); + const size_t bt_rank = bt->select0(i); + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "Select zero error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } const size_t num_ones = frs.rank1(bv.size()); #pragma omp parallel for for (size_t i = 1; i <= num_ones; i++) { const size_t bv_rank = frs.select1(i); - const size_t bt_rank = bt->select_one(i); + const size_t bt_rank = bt->select1(i); if (bv_rank != bt_rank) { std::osyncstream(std::cerr) - << "Select error at position " << i << "\nExpected: " << bv_rank + << "Select one error at position " << i << "\nExpected: " << bv_rank << "\nActual: " << bt_rank << std::endl; throw std::runtime_error("oof"); } diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 299a700..410238e 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -126,6 +126,7 @@ class BitBlockTree { }; private: + template [[nodiscard]] size_t find_initial_block(const size_t rank) const { const auto& top_one_ranks = one_ranks_[0]; const size_t block_size = block_size_lvl_[0]; @@ -133,12 +134,25 @@ class BitBlockTree { size_t end = top_one_ranks.size() - 1; while (start != end) { const size_t middle = start + (end - start) / 2; - const size_t current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } if (current_rank < rank) { if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } // If there is only one block left, it's either the current or the // next block - if (top_one_ranks[middle] < rank) { + if (bits < rank) { start = middle + 1; } break; @@ -152,9 +166,7 @@ class BitBlockTree { } public: - [[nodiscard("select result discarded")]] ssize_t - select_one(size_t rank) const { - const size_t start_rank = rank; + [[nodiscard("select result discarded")]] ssize_t select1(size_t rank) const { const auto& top_is_internal = *block_tree_types_[0]; const auto& top_is_internal_rank = *block_tree_types_rs_[0]; const auto& top_pointers = *block_tree_pointers_[0]; @@ -163,7 +175,7 @@ class BitBlockTree { size_t block_size = block_size_lvl_[0]; // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); + size_t current_block = find_initial_block(rank); size_t pos = (current_block * block_size * 8) - 1; // ReSharper disable once CppDFAUnreachableCode @@ -246,15 +258,121 @@ class BitBlockTree { } } } - (void)start_rank; - // const auto s_signed = static_cast(s); - // std::cout << start_rank << " -> s: " << s_signed << ", l: " << l - // << std::endl; - // return static_cast((s_signed >= 0 ? s_signed * 8 : s_signed) + - // l); return pos + l; } + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t k = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= (current_block == k) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - k) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_zeros = 8 - std::popcount(byte); + if (rank > num_zeros) { + rank -= num_zeros; + pos += 8; + byte_offset++; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) == 0; + } + } + } + return pos; + } + int64_t select(uint8_t c, size_type j) { auto c_index = chars_index_[c]; auto& top_level = *block_tree_types_[0]; From f0296f1421794fd244f77e50c107af3d52727f94 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 27 Nov 2023 20:26:06 +0100 Subject: [PATCH 59/92] code cleanup --- examples/build_bt.cpp | 5 +- include/pasta/block_tree/bit_block_tree.hpp | 438 ++------------------ 2 files changed, 34 insertions(+), 409 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index fc88a95..3fdbc15 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -307,9 +307,10 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); - std::cout << std::endl; - bt->add_bit_rank_support(); FlatRankSelect<> frs(bv); + bt->add_bit_rank_support(); + std::cout << " space_rs=" << bt->print_space_usage(); + std::cout << std::endl; #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 410238e..11ea710 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -46,8 +46,6 @@ class BitBlockTree { /// tree will only contain levels 6 and below. bool CUT_FIRST_LEVELS = true; - using BVStoreType = pasta::BitVector::RawDataType; - /// The arity of the tree size_type tau_; size_type max_leaf_length_; @@ -75,11 +73,6 @@ class BitBlockTree { std::vector decompress_map_; sdsl::int_vector<> compressed_leaves_; - ankerl::unordered_dense::map chars_index_; - std::vector chars_; - std::vector>> c_ranks_; - std::vector>> pointer_c_ranks_; - /// @brief For each level and each block, contains the number of 1s up to (and /// including) the block. std::vector> one_ranks_; @@ -166,7 +159,7 @@ class BitBlockTree { } public: - [[nodiscard("select result discarded")]] ssize_t select1(size_t rank) const { + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { const auto& top_is_internal = *block_tree_types_[0]; const auto& top_is_internal_rank = *block_tree_types_rs_[0]; const auto& top_pointers = *block_tree_pointers_[0]; @@ -213,12 +206,12 @@ class BitBlockTree { current_block = prev_is_internal_rank.rank1(current_block) * tau_; block_size /= tau_; - size_t k = current_block; + const size_t start_block = current_block; while (one_ranks[current_block] < rank) { ++current_block; } - rank -= (current_block == k) ? 0 : one_ranks[current_block - 1]; - pos += (current_block - k) * block_size * 8; + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; if (!is_internal[current_block]) { size_t back_block_index = is_internal_rank.rank0(current_block); current_block = pointers[back_block_index]; @@ -242,23 +235,24 @@ class BitBlockTree { current_block = block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t l = 0; + size_t byte_offset = 0; while (rank > 0) { const uint8_t byte = decompress_map_[compressed_leaves_[current_block * leaf_size + - l / 8]]; + byte_offset]]; const uint8_t num_ones = std::popcount(byte); if (rank > num_ones) { rank -= num_ones; - l += 8; + pos += 8; + ++byte_offset; } else { for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - l++; + pos++; rank -= ((1 << bit) & byte) > 0; } } } - return pos + l; + return pos; } [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { @@ -323,12 +317,12 @@ class BitBlockTree { const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; return rnk; }; - const size_t k = current_block; + const size_t start_block = current_block; while (zero_ranks(current_block) < rank) { ++current_block; } - rank -= (current_block == k) ? 0 : zero_ranks(current_block - 1); - pos += (current_block - k) * block_size * 8; + rank -= (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; if (!is_internal[current_block]) { size_t back_block_index = is_internal_rank.rank0(current_block); current_block = pointers[back_block_index]; @@ -373,105 +367,6 @@ class BitBlockTree { return pos; } - int64_t select(uint8_t c, size_type j) { - auto c_index = chars_index_[c]; - auto& top_level = *block_tree_types_[0]; - - auto& top_level_rs = *block_tree_types_rs_[0]; - auto& top_level_ptr = *block_tree_pointers_[0]; - auto& top_level_off = *block_tree_offsets_[0]; - size_type current_block = (j - 1) / block_size_lvl_[0]; - size_type end_block = c_ranks_[c_index][0].size() - 1; - int64_t block_size = block_size_lvl_[0]; - // find first level block containing the jth occurrence of c with a bin - // search - while (current_block != end_block) { - size_type m = current_block + (end_block - current_block) / 2; - - size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; - if (f < j) { - if (end_block - current_block == 1) { - if (c_ranks_[c_index][0][m] < static_cast(j)) { - current_block = m + 1; - } - break; - } - current_block = m; - } else { - end_block = m - 1; - } - } - - // accumulator - int64_t s = current_block * block_size - 1; - // index that indicates how many c's are still unaccounted for - j -= (current_block == 0) ? 0 : c_ranks_[c_index][0][current_block - 1]; - // we translate unmarked blocks on the top level independently as it differs - // from the other levels - if (!top_level[current_block]) { - int64_t blk = top_level_rs.rank0(current_block); - current_block = top_level_ptr[blk]; - int64_t g = top_level_off[blk]; - int64_t rank_d = (current_block == 0) ? - c_ranks_[c_index][0][0] : - c_ranks_[c_index][0][current_block] - - c_ranks_[c_index][0][current_block - 1]; - rank_d -= pointer_c_ranks_[c_index][0][blk]; - if (rank_d < j) { - j -= rank_d; - s += (block_size - g); - current_block++; - } else { - j += pointer_c_ranks_[c_index][0][blk]; - s -= g; - } - } - uint64_t i = 1; - while (i < height()) { - auto& current_level = *block_tree_types_[i]; - auto& current_level_rs = *block_tree_types_rs_[i]; - auto& current_level_ptr = *block_tree_pointers_[i]; - auto& current_level_off = *block_tree_offsets_[i]; - auto& prev_level_rs = *block_tree_types_rs_[i - 1]; - current_block = prev_level_rs.rank1(current_block) * tau_; - block_size /= tau_; - int64_t k = current_block; - while ((int64_t)c_ranks_[c_index][i][current_block] < j) { - current_block++; - } - j -= (current_block == k) ? 0 : c_ranks_[c_index][i][current_block - 1]; - s += (current_block - k) * block_size; - if (!current_level[current_block]) { - int64_t blk = current_level_rs.rank0(current_block); - current_block = current_level_ptr[blk]; - int64_t g = current_level_off[blk]; - int64_t rank_d = (current_block % tau_ == 0) ? - c_ranks_[c_index][i][current_block] : - c_ranks_[c_index][i][current_block] - - c_ranks_[c_index][i][current_block - 1]; - rank_d -= pointer_c_ranks_[c_index][i][blk]; - if (rank_d < j) { - j -= rank_d; - s += (block_size - g); - current_block++; - } else { - j += pointer_c_ranks_[c_index][i][blk]; - s -= g; - } - } - i++; - } - - current_block = (*block_tree_types_rs_[i - 1]).rank1(current_block) * tau_; - int64_t l = 0; - while (j > 0) { - if (compressed_leaves_[current_block * leaf_size + l] == compress_map_[c]) - j--; - l++; - } - return s + l; - } - /// @brief Counts the number of 1-bits up to (and excluding) an index. [[nodiscard("rank result discarded")]] size_t rank1(const size_type bit_index) const { @@ -584,40 +479,38 @@ class BitBlockTree { return bit_index - rank1(bit_index); } - int64_t print_space_usage() { - int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + - sizeof(leaf_size); - for (auto bv : block_tree_types_) { + size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + for (const auto bv : block_tree_types_) { space_usage += bv->size() / 8; } - for (auto rs : block_tree_types_rs_) { + for (const auto rs : block_tree_types_rs_) { space_usage += rs->space_usage(); } for (const auto iv : block_tree_pointers_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); } for (const auto iv : block_tree_offsets_) { - space_usage += (int64_t)sdsl::size_in_bytes(*iv); + space_usage += sdsl::size_in_bytes(*iv); } if (rank_support) { - for (auto c : chars_) { - int64_t sum = 0; - for (auto lvl : pointer_c_ranks_[chars_index_[c]]) { - sum += sdsl::size_in_bytes(lvl); - } - for (auto lvl : c_ranks_[chars_index_[c]]) { - sum += sdsl::size_in_bytes(lvl); - } - space_usage += sum; + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); } } - for (auto v : block_size_lvl_) { - space_usage += sizeof(v); + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); } - for (auto v : block_per_lvl_) { - space_usage += sizeof(v); + + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); } + // space_usage += leaves_.size() * sizeof(uint8_t); space_usage += sdsl::size_in_bytes(compressed_leaves_); space_usage += compress_map_.size(); @@ -670,113 +563,6 @@ class BitBlockTree { return 0; } - int32_t add_rank_support() { - rank_support = true; - c_ranks_.resize(chars_.size(), std::vector>()); - pointer_c_ranks_.resize(chars_.size(), std::vector>()); - for (uint64_t i = 0; i < c_ranks_.size(); i++) { - c_ranks_[i].resize(height(), sdsl::int_vector<0>()); - for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { - c_ranks_[i][j].resize(block_tree_types_[j]->size()); - } - } - for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { - pointer_c_ranks_[i].resize(block_tree_pointers_.size(), - sdsl::int_vector<0>()); - for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { - pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); - } - } - for (auto c : chars_) { - for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { - rank_block(c, 0, i); - } - size_type max = 0; - for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { - c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; - if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { - max = c_ranks_[chars_index_[c]][0][i]; - } - } - for (uint64_t i = 1; i < height(); i++) { - size_type counter = tau_; - size_type acc = 0; - for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { - size_type temp = c_ranks_[chars_index_[c]][i][j]; - c_ranks_[chars_index_[c]][i][j] += acc; - acc += temp; - counter--; - if (counter == 0) { - acc = 0; - counter = tau_; - } - } - } - for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { - sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); - } - for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { - sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); - } - } - return 0; - } - - int32_t add_rank_support_omp(int32_t threads) { - rank_support = true; - c_ranks_.resize(chars_.size(), std::vector>()); - pointer_c_ranks_.resize(chars_.size(), std::vector>()); - for (uint64_t i = 0; i < c_ranks_.size(); i++) { - c_ranks_[i].resize(height(), sdsl::int_vector<0>()); - for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { - c_ranks_[i][j].resize(block_tree_types_[j]->size()); - } - } - for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { - pointer_c_ranks_[i].resize(block_tree_pointers_.size(), - sdsl::int_vector<0>()); - for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { - pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); - } - } - omp_set_num_threads(threads); - -#pragma omp parallel for default(none) - for (auto c : chars_) { - for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { - rank_block(c, 0, i); - } - size_type max = 0; - for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { - c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; - if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { - max = c_ranks_[chars_index_[c]][0][i]; - } - } - for (uint64_t i = 1; i < height(); i++) { - size_type counter = tau_; - size_type acc = 0; - for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { - size_type temp = c_ranks_[chars_index_[c]][i][j]; - c_ranks_[chars_index_[c]][i][j] += acc; - acc += temp; - counter--; - if (counter == 0) { - acc = 0; - counter = tau_; - } - } - } - for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { - sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); - } - for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { - sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); - } - } - return 0; - } - protected: void compress_leaves() { // Holds a 1 on every char that exists @@ -878,7 +664,6 @@ class BitBlockTree { } } } else { - // TODO: Handle the case where blocks are not internal const size_type back_block_index = is_internal_rank.rank0(block_index); const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; const size_type off = (*block_tree_offsets_[level])[back_block_index]; @@ -897,49 +682,6 @@ class BitBlockTree { return num_ones; } - /// - /// @brief Generates rank information for a block and all its children - /// recursively. - /// - /// @param c The character to generate rank information for. - /// @param level The level index the block is on. - /// @param block_index The index of the block on this level - size_type rank_block(uint8_t c, size_type level, size_type block_index) { - if (static_cast(block_index) >= - block_tree_types_[level]->size()) { - return 0; - } - size_type rank_c = 0; - if ((*block_tree_types_[level])[block_index] == 1) { - if (static_cast(level) != height() - 1) { - size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); - for (size_type k = 0; k < tau_; k++) { - rank_c += rank_block(c, level + 1, rank_blk * tau_ + k); - } - } else { - size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); - for (size_type k = 0; k < tau_; k++) { - rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); - } - } - } else { - size_type rank_0 = block_tree_types_rs_[level]->rank0(block_index); - size_type ptr = (*block_tree_pointers_[level])[rank_0]; - size_type off = (*block_tree_offsets_[level])[rank_0]; - size_type rank_g = 0; - rank_c += c_ranks_[chars_index_[c]][level][ptr]; - if (off != 0) { - rank_g = part_rank_block(c, level, ptr, off); - size_type rank_2nd = part_rank_block(c, level, ptr + 1, off); - rank_c -= rank_g; - rank_c += rank_2nd; - } - pointer_c_ranks_[chars_index_[c]][level][rank_0] = rank_g; - } - c_ranks_[chars_index_[c]][level][block_index] = rank_c; - return rank_c; - } - size_type part_bit_rank_block(const size_type level, const size_type block_index, const size_type chars_to_process) { @@ -976,7 +718,7 @@ class BitBlockTree { } else { // We're on the last level for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; - k++) { + ++k) { num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); processed_chars += leaf_size; } @@ -1012,61 +754,6 @@ class BitBlockTree { return num_ones; } - size_type part_rank_block(uint8_t c, - size_type level, - size_type block_index, - size_type g) { - if (static_cast(block_index) >= - block_tree_types_[level]->size()) { - return 0; - } - size_type rank_c = 0; - if ((*block_tree_types_[level])[block_index] == 1) { - if (static_cast(level) != height() - 1) { - size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); - size_type k = 0; - size_type k_sum = 0; - for (k = 0; k < tau_ && k_sum + block_size_lvl_[level + 1] <= g; k++) { - rank_c += c_ranks_[chars_index_[c]][level + 1][rank_blk * tau_ + k]; - k_sum += block_size_lvl_[level + 1]; - } - - if (k_sum != g) { - rank_c += - part_rank_block(c, level + 1, rank_blk * tau_ + k, g - k_sum); - } - } else { - size_type rank_blk = block_tree_types_rs_[level]->rank1(block_index); - size_type k = 0; - size_type k_sum = 0; - for (k = 0; k < tau_ && k_sum + leaf_size <= g; k++) { - rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); - k_sum += leaf_size; - } - - if (k_sum != g) { - rank_c += rank_leaf(c, rank_blk * tau_ + k, g % leaf_size); - } - } - } else { - size_type rank_0 = block_tree_types_rs_[level]->rank0(block_index); - size_type ptr = (*block_tree_pointers_[level])[rank_0]; - size_type off = (*block_tree_offsets_[level])[rank_0]; - if (g + off >= block_size_lvl_[level]) { - rank_c += c_ranks_[chars_index_[c]][level][ptr] - - pointer_c_ranks_[chars_index_[c]][level][rank_0] + - part_rank_block(c, - level, - ptr + 1, - g + off - block_size_lvl_[level]); - } else { - rank_c += part_rank_block(c, level, ptr, g + off) - - pointer_c_ranks_[chars_index_[c]][level][rank_0]; - } - } - return rank_c; - } - /// /// @brief Count ones in leaf block. /// @@ -1089,69 +776,6 @@ class BitBlockTree { } return result; } - - size_type rank_leaf(uint8_t c, size_type leaf_index, size_type i) { - if (static_cast(leaf_index * leaf_size) >= - compressed_leaves_.size()) { - return 0; - } - // size_type x = leaves_.size() - leaf_index * this->tau_; - // i = std::min(i, x); - size_type result = 0; - for (size_type ind = 0; ind < i; ind++) { - if (compressed_leaves_[leaf_index * leaf_size + ind] == - compress_map_[c]) { - result++; - } - } - return result; - } - - size_type - find_next_smallest_index_binary_search(size_type i, - std::vector& pVector) { - int64_t l = 0; - int64_t r = pVector.size(); - while (l < r) { - int64_t m = std::floor((l + r) / 2); - if (i < pVector[m]) { - r = m; - } else { - l = m + 1; - } - } - return r - 1; - }; - int64_t - find_next_smallest_index_linear_scan(size_type i, - std::vector& pVector) { - int64_t b = 0; - while (b < pVector.size() && i >= pVector[b]) { - b++; - } - return b - 1; - }; - size_type find_next_smallest_index_block_tree(size_type index) { - size_type block_size = this->block_size_lvl_[0]; - size_type blk_pointer = index / block_size; - size_type off = index % block_size; - size_type child = 0; - for (size_type i = 0; i < this->height(); i++) { - if ((*this->block_tree_types_[i])[blk_pointer] == 0) { - return -1; - } - if (off > 0 && (*this->block_tree_types_[i])[blk_pointer + 1] == 0) { - return -1; - } - size_type rank_blk = this->block_tree_types_rs_[i]->rank1(blk_pointer); - blk_pointer = rank_blk * this->tau_; - block_size /= this->tau_; - child = off / block_size; - off = off % block_size; - blk_pointer += child; - } - return blk_pointer; - }; }; } // namespace pasta From 9f087f8a0d830e18315bc7f2d13f2983b88eb9fe Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 29 Nov 2023 15:49:23 +0100 Subject: [PATCH 60/92] measure rank/select construction time and space --- examples/build_bt.cpp | 10 +++++++--- include/pasta/block_tree/bit_block_tree.hpp | 3 ++- 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 3fdbc15..5b65439 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -306,11 +306,15 @@ int main(int argc, char** argv) { auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); - std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); - FlatRankSelect<> frs(bv); + const size_t no_rs_space = bt->print_space_usage(); bt->add_bit_rank_support(); - std::cout << " space_rs=" << bt->print_space_usage(); + auto elapsed_rs = std::chrono::duration_cast( + Clock::now() - now) + .count(); + const size_t rs_space = bt->print_space_usage(); + std::cout << " time=" << elapsed << " space=" << no_rs_space << " time_rs=" << elapsed_rs << " space_rs=" << rs_space; std::cout << std::endl; + FlatRankSelect<> frs(bv); #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 11ea710..15ad8fb 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -321,7 +321,8 @@ class BitBlockTree { while (zero_ranks(current_block) < rank) { ++current_block; } - rank -= (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); pos += (current_block - start_block) * block_size * 8; if (!is_internal[current_block]) { size_t back_block_index = is_internal_rank.rank0(current_block); From 3b65fd01547834d114e19784d1fc93bf3c284a6d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 30 Nov 2023 16:04:43 +0100 Subject: [PATCH 61/92] recursive bit block tree --- examples/build_bt.cpp | 94 +- ...arded_small.hpp => block_tree_sharded.hpp} | 90 +- .../rec_bit_block_tree_sharded.hpp | 3179 +++++++++++++++++ .../pasta/block_tree/rec_bit_block_tree.hpp | 1531 ++++++++ 4 files changed, 4811 insertions(+), 83 deletions(-) rename include/pasta/block_tree/construction/{block_tree_fp_par_sync_sharded_small.hpp => block_tree_sharded.hpp} (96%) create mode 100644 include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp create mode 100644 include/pasta/block_tree/rec_bit_block_tree.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 5b65439..dd3d8fc 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -26,7 +26,7 @@ #include #include #include -#include +#include #include #define PAR_SHARDED_SYNC_SMALL @@ -111,20 +111,20 @@ make_bt(std::vector& text, } # define ALGO_NAME "shard_sync" #elif defined PAR_SHARDED_SYNC_SMALL -# include -std::unique_ptr> +# include +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { - return std::make_unique< - pasta::BlockTreeFPParShardedSyncSmall>(text, - arity, - 1, - leaf_length, - threads, - queue_size); + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads, + queue_size); } # define ALGO_NAME "shard_sync_small" #elif defined PAR_PHMAP @@ -144,6 +144,23 @@ make_bt(std::vector& text, threads); } # define ALGO_NAME "par_map" +#elif defined REC_BIT +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "rec_bit_shard" #elif defined PAR_PARLAY # include std::unique_ptr> @@ -163,25 +180,6 @@ make_bt(std::vector& text, # define ALGO_NAME "par_parlay" #endif -#if defined PAR_SHARDED_SYNC || defined PAR_SHARDED_SYNC_SMALL -# define USES_QUEUE true -# define IS_PARALLEL true -#else -# define USES_QUEUE false -#endif - -#ifndef IS_PARALLEL -# if defined PAR_SHARDED_SYNC_SMALL || defined PAR_SHARDED_SYNC || \ - defined PAR_SHARDED || defined PAR_PHMAP || defined LPF || \ - defined PAR_PHMAP -# define IS_PARALLEL true -# else -# define IS_PARALLEL false -# endif -#endif - -#define BIT_BT - #include #include #include @@ -232,7 +230,7 @@ int main(int argc, char** argv) { "of the bit vector respectively. In each byte, the least " "significant bit is index 0."); - std::string one_chars = ""; + std::string one_chars; cp.add_string( 'o', "one_chars", @@ -241,6 +239,13 @@ int main(int argc, char** argv) { "the input represents one bit of the bit vector. It will be a 1 if this " "parameter string contains the respective character, 0 otherwise."); + bool verify = false; + cp.add_bool( + 'v', + "verify", + verify, + "Verify whether all queries on the produced block tree are correct."); + if (!cp.process(argc, argv)) { return 1; } @@ -297,23 +302,30 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree - auto bt = std::make_unique>(bv, - arity, - 1, - leaf_length, - threads, - queue_size); + auto bt = + std::make_unique>(bv, + arity, + 1, + leaf_length, + threads, + queue_size); auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); bt->add_bit_rank_support(); auto elapsed_rs = std::chrono::duration_cast( - Clock::now() - now) - .count(); + Clock::now() - now) + .count(); const size_t rs_space = bt->print_space_usage(); - std::cout << " time=" << elapsed << " space=" << no_rs_space << " time_rs=" << elapsed_rs << " space_rs=" << rs_space; + std::cout << " time=" << elapsed << " space=" << no_rs_space + << " time_rs=" << elapsed_rs << " space_rs=" << rs_space; std::cout << std::endl; + + if (!verify) { + return 0; + } + FlatRankSelect<> frs(bv); #if defined BT_INSTRUMENT && defined BT_DBG @@ -380,6 +392,10 @@ int main(int argc, char** argv) { std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); std::cout << std::endl; + if (!verify) { + return 0; + } + #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp similarity index 96% rename from include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp rename to include/pasta/block_tree/construction/block_tree_sharded.hpp index e8eead5..480b9b7 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded_small.hpp +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -41,7 +41,8 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { -/* +namespace sharded { + /// @brief Determine whether to use a Rabin-Karp hash for hashing text windows /// or just use the block's content itself as a hash, stored in an integer. enum class UseHash { @@ -50,7 +51,8 @@ enum class UseHash { /// @brief Use the block's content as a hash IDENTITY }; -*/ + +} // namespace sharded /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes /// and a sharded hash map. Small blocks are not RK-hashed but rather use the @@ -59,7 +61,7 @@ enum class UseHash { /// @tparam size_type The type used for indices etc. (must be a signed integer) /// in the sharded hash map. template -class BlockTreeFPParShardedSyncSmall : public BlockTree { +class BlockTreeSharded : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -446,17 +448,17 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { LevelData& current = levels.back(); if (2 * static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( @@ -465,17 +467,17 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { now = Clock::now(); #endif if (static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( @@ -582,7 +584,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// use the blocks' contents themselves as hashes. /// For block sizes greater than 4 bytes, use Rabin-Karp. /// - template + template void scan_block_pairs(const std::vector& text, LevelData& level, const bool is_padded, @@ -648,7 +650,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); - if constexpr (use_hash == UseHash::RABIN_KARP) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair @@ -669,7 +671,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & HASH_MASK); + pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start, @@ -710,7 +712,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { #endif if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == UseHash::RABIN_KARP) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { @@ -894,8 +896,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { const input_type* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK); + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash current_hash(text, mix_select(hash_value), block_start + offset, @@ -930,7 +931,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { /// @tparam use_hash Determines whether to use a rabin karp hash for hashing /// text windows or to use the block's content as a hash. For any window size /// greater than 8 bytes, use Rabin-Karp. - template + template void scan_blocks(const std::vector& text, LevelData& level_data, const bool is_padded, @@ -1000,7 +1001,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { (thread_id + 1) * segment_size); // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); for (size_t i = start; i < end; ++i) { rk.restart(block_starts[i]); @@ -1013,7 +1014,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr) & HASH_MASK); + pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start, @@ -1054,7 +1055,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { @@ -1210,8 +1211,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { const input_type* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = - (*reinterpret_cast(block_start_ptr + offset) & - HASH_MASK); + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash hash(text, mix_select(hash_value), block_start + offset, @@ -1337,8 +1337,10 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { LevelData& previous_level = levels[level_index - 1]; found_back_block |= static_cast(new_num_internal[level_index]) < levels[level_index].is_internal->size(); - if (!found_back_block) { - level.is_internal.reset(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); @@ -1571,12 +1573,12 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { } public: - BlockTreeFPParShardedSyncSmall(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads, - const size_t queue_size) { + BlockTreeSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { const auto old = omp_get_max_threads(); const auto old_dynamic = omp_get_dynamic(); omp_set_dynamic(0); @@ -1590,7 +1592,7 @@ class BlockTreeFPParShardedSyncSmall : public BlockTree { omp_set_num_threads(old); } - ~BlockTreeFPParShardedSyncSmall() { + ~BlockTreeSharded() { for (auto& rank : this->block_tree_types_rs_) { delete rank; } diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp new file mode 100644 index 0000000..b3b6444 --- /dev/null +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -0,0 +1,3179 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_bit_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveBitBlockTreeSharded + : public RecursiveBitBlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief For some block size (in bytes) i, return the number of trailing + /// zeros in a 64 bit integer when zeroing out characters that are not part + /// of the block. + constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + + /// @brief Masks used for the identity hash. These depend on endianness + constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } + } + + /// @brief Masks for identity hashes for a block size i (in bytes) + constexpr static std::array HASH_MASKS = masks(); + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + // constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + // robin_hood::unordered_flat_map>; + // std::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + +#define MIX + static uint64_t mix_select(uint64_t key) { +#ifdef MIX + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); +#endif + return key; + } + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::span text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size) > 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::span text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::span& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(std::span text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[block_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::span& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::span text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::span text, + std::vector& levels, + const int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels.size() <= 1 || + levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + auto* bt = + new RecursiveBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + BitVector is_internal(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + is_internal[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + auto* bt = new RecursiveBitBlockTreeSharded( + is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + auto& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + RecursiveBitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->num_bits_ = text.size(); + const std::span bytes(reinterpret_cast(text.data().data()), + ceil_div(text.size(), 8ULL)); + construct(bytes, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~RecursiveBitBlockTreeSharded() { + for (auto& bt : this->block_tree_types_) { + delete bt; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +template +class RecursiveBitBlockTreeSharded + : public RecursiveBitBlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief For some block size (in bytes) i, return the number of trailing + /// zeros in a 64 bit integer when zeroing out characters that are not part + /// of the block. + constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + + /// @brief Masks used for the identity hash. These depend on endianness + constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } + } + + /// @brief Masks for identity hashes for a block size i (in bytes) + constexpr static std::array HASH_MASKS = masks(); + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + // constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + // robin_hood::unordered_flat_map>; + // std::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + +#define MIX + static uint64_t mix_select(uint64_t key) { +#ifdef MIX + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); +#endif + return key; + } + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::span text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size) > 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::span text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::span& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(std::span text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[block_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::span& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::span text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::span text, + std::vector& levels, + const int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels.size() <= 1 || + levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + // We cannot "release()" is_internal yet. + // If this is the only level of the tree, we need it later when + // constructing the leaf string + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + // Release the top-level bitvector, + // since now we definitiely don't need it anymore + levels[0].is_internal.release(); + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + auto& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + RecursiveBitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->num_bits_ = text.size(); + const std::span bytes(reinterpret_cast(text.data().data()), + ceil_div(text.size(), 8ULL)); + construct(bytes, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~RecursiveBitBlockTreeSharded() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp new file mode 100644 index 0000000..51ea222 --- /dev/null +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -0,0 +1,1531 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class RecursiveBitBlockTree { +public: + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits_; + bool rank_support = false; + /// Recursively compress the bit vectors of the tree + std::vector*> + block_tree_types_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + [[nodiscard]] size_t size() const { + return num_bits_; + } + + bool operator[](const size_type bit_index) const { + return access(bit_index); + } + + bool access(const size_type bit_index) const { + // FIXME: As of now this works on little endian systems only + const int64_t byte_index = bit_index / 8; + const int64_t bit_offset = bit_index % 8; + + int64_t block_size = block_size_lvl_[0]; + int64_t block_index = byte_index / block_size; + int64_t off = byte_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal.access(block_index)) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal.rank1(block_index) * tau_ + child; + } + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; + return ((1 << bit_offset) & byte) != 0; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& prev_is_internal = *block_tree_types_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; + if (!is_internal.access(current_block)) { + size_t back_block_index = is_internal.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = block_tree_types_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + pos += 8; + ++byte_offset; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + return pos; + } + + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& prev_is_internal = *block_tree_types_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = block_tree_types_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_zeros = 8 - std::popcount(byte); + if (rank > num_zeros) { + rank -= num_zeros; + pos += 8; + byte_offset++; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) == 0; + } + } + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const size_t byte_index = bit_index / 8; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = byte_index / block_size; + size_t block_offset = byte_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + + block]]; + rank += std::popcount(byte); + } + for (size_t block = 0; block < block_offset; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; + rank += std::popcount(byte); + } + + // Masks to remove bits from the last byte, + // that aren't part of the ran query + static constexpr std::array MASKS = { + 0b0000'0000, + 0b0000'0001, + 0b0000'0011, + 0b0000'0111, + 0b0000'1111, + 0b0001'1111, + 0b0011'1111, + 0b0111'1111, + }; + rank += std::popcount( + decompress_map_[compressed_leaves_[block_index * leaf_size + + block_offset]] & + MASKS[bit_index % 8]); + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + for (const auto bt : block_tree_types_) { + space_usage += bt->print_space_usage(); + } + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + } + if (rank_support) { + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + } + + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + + // space_usage += leaves_.size() * sizeof(uint8_t); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); + + return space_usage; + }; + + int32_t add_bit_rank_support() { + rank_support = true; + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + return 0; + } + +protected: + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs + const auto& is_internal = *block_tree_types_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal.access(block_index)) { + const size_type internal_index = is_internal.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + const uint8_t byte = + decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; + result += std::popcount(byte); + } + return result; + } +}; + +template +class RecursiveBitBlockTree { +public: + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits_; + bool rank_support = false; + /// Bit vectors for each level determining whether a block is internal + /// (=1) or not (=0) + std::vector block_tree_types_; + std::vector*> + block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + [[nodiscard]] size_t size() const { + return num_bits_; + } + + bool operator[](const size_type bit_index) const { + return access(bit_index); + } + + bool access(const size_type bit_index) const { + // FIXME: As of now this works on little endian systems only + const int64_t byte_index = bit_index / 8; + const int64_t bit_offset = bit_index % 8; + + int64_t block_size = height() == 0 ? leaf_size : block_size_lvl_[0]; + int64_t block_index = byte_index / block_size; + int64_t off = byte_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + } + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; + return ((1 << bit_offset) & byte) != 0; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + pos += 8; + ++byte_offset; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + return pos; + } + + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_zeros = 8 - std::popcount(byte); + if (rank > num_zeros) { + rank -= num_zeros; + pos += 8; + byte_offset++; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) == 0; + } + } + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const size_t byte_index = bit_index / 8; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = byte_index / block_size; + size_t block_offset = byte_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + + block]]; + rank += std::popcount(byte); + } + for (size_t block = 0; block < block_offset; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; + rank += std::popcount(byte); + } + + // Masks to remove bits from the last byte, + // that aren't part of the ran query + static constexpr std::array MASKS = { + 0b0000'0000, + 0b0000'0001, + 0b0000'0011, + 0b0000'0111, + 0b0000'1111, + 0b0001'1111, + 0b0011'1111, + 0b0111'1111, + }; + rank += std::popcount( + decompress_map_[compressed_leaves_[block_index * leaf_size + + block_offset]] & + MASKS[bit_index % 8]); + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + for (const auto bv : block_tree_types_) { + space_usage += bv->size() / 8; + } + for (const auto rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + } + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + } + if (rank_support) { + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + } + + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + + // space_usage += leaves_.size() * sizeof(uint8_t); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); + + return space_usage; + }; + + int32_t add_bit_rank_support() { + rank_support = true; + if (height() == 0) { + return 0; + } + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + return 0; + } + +protected: + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + const uint8_t byte = + decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; + result += std::popcount(byte); + } + return result; + } +}; + +} // namespace pasta + +/******************************************************************************/ From 82f481117b8186f94f9781fcd1cfbf44a9f41a2d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 30 Nov 2023 21:20:35 +0100 Subject: [PATCH 62/92] reduce code duplication --- examples/build_bt.cpp | 30 +- .../rec_bit_block_tree_sharded.hpp | 1632 +---------------- .../pasta/block_tree/rec_bit_block_tree.hpp | 827 +-------- 3 files changed, 122 insertions(+), 2367 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index dd3d8fc..8381f62 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -257,7 +257,7 @@ int main(int argc, char** argv) { << std::endl; #endif - pasta::BitVector bv; + std::unique_ptr bv; std::vector text; { std::string input; @@ -268,20 +268,20 @@ int main(int argc, char** argv) { if (make_bv) { if (one_chars.empty()) { // Interpret each character as 8 bits - new (&bv) pasta::BitVector(input.size() * 8); - std::span bytes = std::as_writable_bytes(bv.data()); + bv = std::make_unique(input.size() * 8); + std::span bytes = std::as_writable_bytes(bv->data()); for (size_t i = 0; i < input.size(); ++i) { bytes[i] = std::byte{static_cast(input[i])}; } } else { // Interpret each character as a bit - new (&bv) pasta::BitVector(input.size()); + bv = std::make_unique(input.size()); std::array is_one{}; for (char c : one_chars) { is_one[static_cast(c)] = true; } for (size_t i = 0; i < input.size(); ++i) { - bv[i] = is_one[static_cast(input[i])]; + (*bv)[i] = is_one[static_cast(input[i])]; } } } else { @@ -294,7 +294,7 @@ int main(int argc, char** argv) { << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; if (make_bv) { - std::cout << " bv_size=" << bv.size(); + std::cout << " bv_size=" << bv->size(); } else { std::cout << " file_size=" << text.size(); } @@ -303,7 +303,7 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree auto bt = - std::make_unique>(bv, + std::make_unique>(*bv, arity, 1, leaf_length, @@ -313,7 +313,7 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); - bt->add_bit_rank_support(); + bt->add_bit_rank_support(threads); auto elapsed_rs = std::chrono::duration_cast( Clock::now() - now) .count(); @@ -326,24 +326,24 @@ int main(int argc, char** argv) { return 0; } - FlatRankSelect<> frs(bv); + FlatRankSelect<> frs(*bv); #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif #pragma omp parallel for - for (size_t i = 0; i < bv.size(); ++i) { + for (size_t i = 0; i < bv->size(); ++i) { const bool c = bt->access(i); - if (c != bv[i]) { + if (c != (*bv)[i]) { std::osyncstream(std::cerr) << "Access error at position " << i - << "\nExpected: " << std::boolalpha << bv[i] << "\nActual: " << c + << "\nExpected: " << std::boolalpha << (*bv)[i] << "\nActual: " << c << std::noboolalpha << std::endl; exit(1); } } #pragma omp parallel for - for (size_t i = 0; i < bv.size(); i++) { + for (size_t i = 0; i < bv->size(); i++) { const size_t bt_rank = bt->rank1(i); const size_t bv_rank = frs.rank1(i); @@ -354,7 +354,7 @@ int main(int argc, char** argv) { throw std::runtime_error("oof"); } } - const size_t num_zeros = frs.rank0(bv.size()); + const size_t num_zeros = frs.rank0(bv->size()); #pragma omp parallel for for (size_t i = 1; i <= num_zeros; i++) { @@ -368,7 +368,7 @@ int main(int argc, char** argv) { } } - const size_t num_ones = frs.rank1(bv.size()); + const size_t num_ones = frs.rank1(bv->size()); #pragma omp parallel for for (size_t i = 1; i <= num_ones; i++) { diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index b3b6444..18968d0 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -1317,16 +1317,25 @@ class RecursiveBitBlockTreeSharded } sdsl::util::bit_compress(*pointers); sdsl::util::bit_compress(*offsets); - auto* bt = - new RecursiveBitBlockTreeSharded( - *top_level.is_internal, - this->tau_, - this->s_, - this->max_leaf_length_, - threads, - queue_size); - this->block_tree_types_.push_back(bt); - this->block_tree_types_.back()->add_bit_rank_support(); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } this->block_tree_pointers_.push_back(pointers); this->block_tree_offsets_.push_back(offsets); this->block_size_lvl_.push_back(top_level.block_size); @@ -1391,6 +1400,11 @@ class RecursiveBitBlockTreeSharded } } } + if constexpr (recursion_level == 0) { + if (levels.size() == 1) { + top_level.is_internal.release(); + } + } this->amount_of_leaves = leaf_count; this->compress_leaves(); } @@ -1424,7 +1438,7 @@ class RecursiveBitBlockTreeSharded const size_type num_internal = new_num_internal[level_index]; // Allocate new vectors for the tree - BitVector is_internal(new_size); + auto* is_internal = new BitVector(new_size); auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); @@ -1453,7 +1467,7 @@ class RecursiveBitBlockTreeSharded // Add it to the is_internal bit vector const bool block_is_internal = (*level.is_internal)[i]; - is_internal[num_non_pruned] = block_is_internal; + (*is_internal)[num_non_pruned] = block_is_internal; num_non_pruned++; if (block_is_internal) { @@ -1470,15 +1484,23 @@ class RecursiveBitBlockTreeSharded sdsl::util::bit_compress(*pointers); sdsl::util::bit_compress(*offsets); - auto* bt = new RecursiveBitBlockTreeSharded( - is_internal, - this->tau_, - this->s_, - this->max_leaf_length_, - threads, - queue_size); - this->block_tree_types_.push_back(bt); - this->block_tree_types_.back()->add_bit_rank_support(); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } this->block_tree_pointers_.push_back(pointers); this->block_tree_offsets_.push_back(offsets); this->block_size_lvl_.push_back(level.block_size); @@ -1606,1574 +1628,6 @@ class RecursiveBitBlockTreeSharded omp_set_dynamic(old_dynamic); omp_set_num_threads(old); } - - ~RecursiveBitBlockTreeSharded() { - for (auto& bt : this->block_tree_types_) { - delete bt; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; - } - } -}; - -template -class RecursiveBitBlockTreeSharded - : public RecursiveBitBlockTree { - using Clock = std::chrono::high_resolution_clock; - using TimePoint = Clock::time_point; - - /// @brief For some block size (in bytes) i, return the number of trailing - /// zeros in a 64 bit integer when zeroing out characters that are not part - /// of the block. - constexpr static uint64_t MASK_TRAILING_ZEROS[9] = - {64, 56, 48, 40, 32, 24, 16, 8, 0}; - - /// @brief Masks used for the identity hash. These depend on endianness - constexpr static std::array masks() { - if constexpr (std::endian::native == std::endian::big) { - return {0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; - } else { - return {0, - static_cast(~0) >> MASK_TRAILING_ZEROS[1], - static_cast(~0) >> MASK_TRAILING_ZEROS[2], - static_cast(~0) >> MASK_TRAILING_ZEROS[3], - static_cast(~0) >> MASK_TRAILING_ZEROS[4], - static_cast(~0) >> MASK_TRAILING_ZEROS[5], - static_cast(~0) >> MASK_TRAILING_ZEROS[6], - static_cast(~0) >> MASK_TRAILING_ZEROS[7], - static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; - } - } - - /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static std::array HASH_MASKS = masks(); - - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); - - /// @brief A bit vector - using BitVector = pasta::BitVector; - /// @brief A rank data structure for a bit vector - using Rank = pasta::RankSelect; - - /// @brief A sequential hash map used as backing for the sharded hash map. - template - using SeqHashMap = - ankerl::unordered_dense::map>; - // robin_hood::unordered_flat_map>; - // std::unordered_map>; - - /// @brief A rabin karp hasher preconfigured for the current template - /// parameters - using RabinKarp = MersenneRabinKarp; - /// @brief A rabin karp hash for the preconfigured rabin karp hasher - using RabinKarpHash = MersenneHash; - - /// @brief A hash map with rabin karp hashes as keys - template update_fn_type, - template typename seq_map_type = SeqHashMap> - using RabinKarpMap = - SyncShardedMap; - -#define MIX - static uint64_t mix_select(uint64_t key) { -#ifdef MIX - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); -#endif - return key; - } - -#ifdef BT_INSTRUMENT -public: - size_t bp_hash_pairs_ns = 0; - size_t bp_scan_pairs_ns = 0; - size_t bp_markings_ns = 0; - size_t bp_bitvec_ns = 0; - - size_t b_hash_blocks_ns = 0; - size_t b_scan_blocks_ns = 0; - size_t b_update_blocks_ns = 0; -#endif - -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - int64_t block_size; - /// @brief The index of the current level. - /// First level is 0, second level is 1 etc. - int64_t level_index; - /// @brief The number of blocks on the current level - int64_t num_blocks; - - LevelData(const int64_t level_index_, - const int64_t block_size_, - const int64_t num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - PairOccurrences(PairOccurrences&&) noexcept = default; - PairOccurrences& operator=(PairOccurrences&&) = default; - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - ~BlockOccurrences() = default; - - BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { - first_occ = other.first_occ.load(); - occurrences = std::move(other.occurrences); - return *this; - } - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_offset The offset of that occurrence - void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - - /// @brief Constructs the block tree. - /// @param text The input text. - /// @param threads The number of threads to use for construction - /// @param queue_size The max number of items in each thread's queue for its - /// hash map - void construct(const std::span text, - const size_t threads, - const size_t queue_size) { -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - const size_type text_len = text.size(); - /// The number of characters a block tree with s top-level blocks and arity - /// of strictly tau would exceed over the text size - int64_t padding; - /// The height of the tree - int64_t tree_height; - /// The size of the largest blocks (i.e. the top level blocks) - int64_t top_block_size; - - this->calculate_padding(padding, text_len, tree_height, top_block_size); - - const bool is_padded = padding > 0; - - std::vector levels; - - // Prepare the top level - levels.emplace_back(0, top_block_size, text_len / top_block_size); - LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); - for (size_type i = 0; i < text_len; i += top_level.block_size) { - top_level.block_starts->push_back(i); - } - top_level.block_size = top_block_size; - top_level.num_blocks = top_level.block_starts->size(); - -#ifdef BT_INSTRUMENT - - const size_t setup_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_BENCH - std::cout << " setup=" << setup_ns; -# endif - - size_t pairs_ns = 0; - size_t blocks_ns = 0; - size_t generate_ns = 0; -#endif -#ifdef BT_DBG - std::cout << "using " << threads << " threads" << std::endl; -#endif - -#ifdef BT_BENCH - std::cout << " queue_capacity=" << queue_size; -#endif - - // Construct the pre-pruned tree level by level - for (size_t level = 0; level < static_cast(tree_height); level++) { -#ifdef BT_DBG - std::cout << "----------------- level " << level << " -----------------" - << std::endl; -#endif - -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - LevelData& current = levels.back(); - if (2 * static_cast(current.block_size) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } -#ifdef BT_INSTRUMENT - pairs_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - if (static_cast(current.block_size) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } -#ifdef BT_INSTRUMENT - blocks_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { - levels.push_back(std::move(generate_next_level(text, current))); - } -#ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); -#endif - } -#ifdef BT_INSTRUMENT -# if defined(BT_DBG) - std::cout << "pairs: " << (pairs_ns / 1'000'000) - << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) - << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) - << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) - << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) - << "ms,\nblocks: " << (blocks_ns / 1'000'000) - << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) - << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) - << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) - << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," - << std::endl; -# elif defined(BT_BENCH) - std::cout << " pairs=" << (pairs_ns / 1'000'000) - << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) - << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) - << " markings=" << (bp_markings_ns / 1'000'000) - << " bitvec=" << (bp_bitvec_ns / 1'000'000) - << " blocks=" << (blocks_ns / 1'000'000) - << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) - << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) - << " update_blocks=" << (b_update_blocks_ns / 1'000'000) - << " generate_next=" << (generate_ns / 1'000'000); - -# endif - now = Clock::now(); -#endif - prune(levels); -#ifdef BT_INSTRUMENT - size_t prune_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -# ifdef BT_DBG - std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; -# elif defined BT_BENCH - std::cout << " prune=" << (prune_ns / 1'000'000); -# endif -#endif - - make_tree(text, levels, padding); -#ifdef BT_INSTRUMENT - size_t make_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); -# ifdef BT_DBG - std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; -# elif defined BT_BENCH - std::cout << " make=" << (make_ns / 1'000'000); -# endif -#endif - } - - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - [[maybe_unused]] static void - print_aggregate(const char* name, - const tlx::Aggregate& agg, - const size_t div = 1) { - printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", - name, - static_cast(agg.min() / div), - static_cast(agg.max() / div), - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - static_cast(agg.count())); - } - - /// @brief Scan through the blocks pairwise in order to identify which blocks - /// should be replaced with back blocks. - /// - /// @param text The input string. - /// @param level The data for the current level. - /// @param is_padded `true` iff the last block on this level *does not* end at - /// the exact end of the text. - /// @param threads Number of threads to use - /// @param queue_size The size of the queue to use per thread in the sharded - /// hash map. - /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or - /// use the blocks' contents themselves as hashes. - /// For block sizes greater than 4 bytes, use Rabin-Karp. - /// - template - void scan_block_pairs(const std::span text, - LevelData& level, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - if (level.num_blocks < 4) { - level.is_internal = std::make_unique(level.num_blocks, true); - level.is_internal_rank = std::make_unique(*level.is_internal); - return; - } - - // A map containing hashed block pairs mapped to their indices of the - // pairs' first block respectively - BlockPairMap map(threads, queue_size); - - std::atomic_size_t threads_done = 0; - std::atomic_bool last_done = false; - auto& barrier = map.barrier(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, \ - map, \ - text, \ - now, \ - is_padded, \ - threads_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits, \ - threads, \ - std::cout) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) -#endif - { - const size_t thread_id = omp_get_thread_num(); - typename BlockPairMap::Shard shard = map.get_shard(thread_id); - const size_t num_threads = omp_get_num_threads(); - const size_t num_block_pairs = level.num_blocks - 1 - is_padded; - const size_t block_size = level.block_size; - const size_t pair_size = 2 * block_size; - const auto& block_starts = *level.block_starts; - - // Hash every window and determine for all block pairs whether - // they have previous occurrences. - const size_t segment_size = - std::max(1, ceil_div(num_block_pairs, num_threads)); - - // Start and end index of the current thread's segment - const auto start = thread_id * segment_size; - const auto end = - std::min(num_block_pairs, (thread_id + 1) * segment_size); - - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block - if (!level.next_is_adjacent(i)) { - continue; - } - rk.restart(block_starts[i]); - // Move the hasher to the current block pair - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } else { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start, - block_size); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } - - if (const size_t thread_order = - threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - // Now, we handle the queue asynchronously - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - { - bp_hash_pairs_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - - if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); - } - scan_windows_in_block_pair(rk, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - scan_windows_in_block_pair_identity(text, - block_starts[i], - pair_size, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } - } - -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - bp_scan_pairs_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : map.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Pair Map Loads ", map_loads); - print_aggregate("Pair Map Hits ", scan_hits); - print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); - - BT_ASSERT(map.num_inserts_.load() == map.size()); -# endif -#endif - - level.is_internal = std::make_unique(level.num_blocks); - fill_is_internal(*level.is_internal, map); - level.is_internal_rank = std::make_unique(*level.is_internal); - } - - /// @brief Fills the bit vector `is_internal` based on the values in the - /// given map. - /// @param is_internal An unfilled bit vector with a bit for each block on - /// this level. - /// @param map A map, mapping hashed block pairs to their first occurrence's - /// block index. - void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { - const size_type num_blocks = is_internal.size(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - // Set up the packed array holding the markings for each block. - // Each mark is a 2-bit number. - // The MSB is 1 iff the block and its successor have a prior - // occurrence. The LSB is 1 iff the block and its predecessor - // have a prior occurrence. - sdsl::int_vector<2> markings(num_blocks, 0); - map.for_each( - [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { - for (const size_type occ : pair_occs.occurrences) { - if (pair_occs.first_occ_block < occ) { - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; - } - } - }); -#ifdef BT_INSTRUMENT - bp_markings_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the bit vector indicating which blocks are internal - is_internal[0] = true; - is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; - for (size_type i = 0; i < num_blocks - 1; ++i) { - const bool block_is_internal = markings[i] != 0b11; - is_internal[i] = block_is_internal; - } -#ifdef BT_INSTRUMENT - bp_bitvec_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scan through the windows starting in a block and mark - /// them accordingly if they represent the earliest occurrence of some - /// block hash. - /// - /// The supplied `RabinKarp` hasher must be at the start of the block. - /// @param rk A Rabin-Karp hasher whose state is at the start of the block. - /// @param map The map containing the hashes of block pairs mapped to their - /// block indexes at which they occur. - /// @param num_iterations The number of contiguous windows to hash. - /// @param current_block_index The index of the block being currently - /// hashed. - static inline void - scan_windows_in_block_pair(RabinKarp& rk, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg -#endif - ) { - for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { - RabinKarpHash current_hash = rk.current_hash(); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { -#ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); -#else - continue; -#endif - } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - - static inline void - scan_windows_in_block_pair_identity(const std::span& text, - const size_t block_start, - const size_t pair_size, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg -#endif - ) { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_t offset = 0; offset < num_iterations; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash current_hash(text, - mix_select(hash_value), - block_start + offset, - pair_size); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { -#ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); -#else - continue; -#endif - } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - - /// @brief Determine the positions for each block's earliest occurrence if - /// there is any. - /// - /// @param text The input text - /// @param level_data The data for the current level - /// @param is_padded true, iff the last block of the level extends past the - /// end of the text - /// @param threads The number of threads to use during construction. - /// @param queue_size The max number of items in each thread's queues. - /// @tparam use_hash Determines whether to use a rabin karp hash for hashing - /// text windows or to use the block's content as a hash. For any window size - /// greater than 8 bytes, use Rabin-Karp. - template - void scan_blocks(std::span text, - LevelData& level_data, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - const size_t num_blocks = level_data.num_blocks; - - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); - level_data.offsets = - std::make_unique>(num_blocks, 0); - level_data.counters = - std::make_unique>(num_blocks, 0); - - if (num_blocks <= 2) { - return; - } - - // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size); - - // The number of threads finished with hashing blocks - std::atomic_size_t num_done = 0; - // Whether the last thread is done - std::atomic_bool last_done = false; - auto& barrier = links.barrier(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, \ - text, \ - links, \ - now, \ - is_padded, \ - num_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) -#endif - { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } else { - const uint64_t HASH_MASK = HASH_MASKS[block_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); - } - } - - if (const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - - { - b_hash_blocks_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } - } -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : links.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - - BT_ASSERT(links.num_inserts_.load() == links.size()); -# endif -#endif - - // By this point, the map should contain the first occurrences of - // every respective block's content. We then fill the pointers - // and offsets with this data and increment counters accordingly - links.for_each( - [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; - } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); - -#ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scans through block-sized windows starting inside one block and - /// tries to find blocks with matching hashes in the map. Such blocks - /// will have their earliest occurrence update. - /// @param rk A Rabin-Karp hasher whose current state is at a block start. - /// @param links A map whose keys are hashed blocks and the values - /// are all block indices of blocks matching the hash in ascending order. - /// @param level_data The data for the current level. - /// @param current_block_index The index of the block which the - /// Rabin-Karp hasher is situated in. - static void scan_windows_in_block(RabinKarp& rk, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits -#endif - ) { - for (size_type offset = 0; offset < level_data.block_size; - ++offset, rk.next()) { - RabinKarpHash hash = rk.current_hash(); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { -#ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); -#else - continue; -#endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - - static void - scan_windows_in_block_identity(const std::span& text, - const size_t block_start, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits -#endif - ) { - const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_type offset = 0; offset < level_data.block_size; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start + offset, - level_data.block_size); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { -#ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); -#else - continue; -#endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - - /// @brief Generate the block size, number of block and block start indices - /// for the next level. - /// - /// This depends on the current level's block size, number of blocks and - /// is_internal bit vector being filled. - /// - /// @param text The input text. - /// @param level The level data of the current level. - /// @return The level data of the next level. - [[nodiscard]] LevelData - generate_next_level(const std::span text, - const LevelData& level) const { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const auto& is_internal = *level.is_internal; - const size_t next_block_size = block_size / this->tau_; - - std::vector new_block_starts; - new_block_starts.reserve(num_blocks * this->tau_); - for (size_t i = 0; i < num_blocks; ++i) { - if (!is_internal[i]) { - continue; - } - - // We generate up to tau new blocks for each internal block, - // excluding blocks that start past the end of the text - const auto parent_block_start = (*level.block_starts)[i]; - for (size_t j = 0, current_block_start = parent_block_start; - j < static_cast(this->tau_) && - current_block_start < text.size(); - ++j, current_block_start += next_block_size) { - new_block_starts.push_back(current_block_start); - } - } - - LevelData next_level(level.level_index + 1, - next_block_size, - new_block_starts.size()); - next_level.block_starts = - std::make_unique>(std::move(new_block_starts)); - return next_level; - } - - /// - /// @brief Takes a vector of levels and fills the block tree fields with - /// them. - /// - /// @param[in] levels A vector containing data for each level, with the - /// first entry corresponding to the topmost level. - /// - void make_tree(const std::span text, - std::vector& levels, - const int64_t padding) { - const bool is_padded = padding > 0; - - // Count the current number of internal blocks per level - std::vector new_num_internal(levels.size(), 0); - for (size_t level = 0; level < levels.size(); level++) { - for (size_t block = 0; block < levels[level].is_internal->size(); - block++) { - if ((*levels[level].is_internal)[block]) { - ++new_num_internal[level]; - } - } - } - - // Create first level - bool found_back_block = levels.size() <= 1 || - levels[0].is_internal->size() > - static_cast(new_num_internal[0]) || - !this->CUT_FIRST_LEVELS; - LevelData& top_level = levels.front(); - if (found_back_block) { - const size_t n = top_level.num_blocks; - const size_t num_internal = new_num_internal[0]; - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_back_blocks = 0; - for (size_t i = 0; i < n; i++) { - // if a back block is found, add its pointer and offset - if (!(*top_level.is_internal)[i]) { - (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; - (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; - num_back_blocks++; - } - } - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - // We cannot "release()" is_internal yet. - // If this is the only level of the tree, we need it later when - // constructing the leaf string - this->block_tree_types_.push_back(top_level.is_internal.get()); - this->block_tree_types_rs_.push_back( - new Rank(*this->block_tree_types_.back())); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(top_level.block_size); - } - top_level.pointers.reset(); - top_level.offsets.reset(); - top_level.counters.reset(); - - // Add level data to the tree - for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData& level = levels[level_index]; - LevelData& previous_level = levels[level_index - 1]; - found_back_block |= static_cast(new_num_internal[level_index]) < - levels[level_index].is_internal->size(); - if (!found_back_block && level_index < levels.size() - 1) { - if (level_index < levels.size() - 1) { - level.is_internal.reset(); - } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - continue; - } - - make_tree_level(levels, - new_num_internal, - level_index, - is_padded, - text.size()); - - // We don't need these anymore - if (level_index < levels.size() - 1) { - level.is_internal.reset(); - } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - } - - this->leaf_size = levels.back().block_size / this->tau_; - // Construct the leaf string - int64_t leaf_count = 0; - auto& last_is_internal = *levels.back().is_internal; - std::vector& last_block_starts = *levels.back().block_starts; - for (size_t block = 0; block < last_is_internal.size(); block++) { - if (!last_is_internal[block]) { - continue; - } - const size_type block_start = last_block_starts[block]; - // For every leaf on the last level, we have tau leaf blocks - leaf_count += this->tau_; - // Iterate through all characters in this child and - // add them to the leaf string - for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); - b++) { - if (static_cast(block_start + b) < text.size()) { - this->leaves_.push_back(text[block_start + b]); - } - } - } - // Release the top-level bitvector, - // since now we definitiely don't need it anymore - levels[0].is_internal.release(); - this->amount_of_leaves = leaf_count; - this->compress_leaves(); - } - - /// @brief Generates a level and adds the relevant data to the block tree. - /// - /// @param levels The vector of levels of the tree. - /// @param level_index The index of the level to generate. This must be - /// strictly greater than 0. - /// @param is_padded Whether there is padding in the last block of the tree - void make_tree_level(std::vector& levels, - const std::vector& new_num_internal, - const size_t level_index, - const bool is_padded, - const size_t text_len) { - LevelData& previous_level = levels[level_index - 1]; - LevelData& level = levels[level_index]; - - size_type new_size = - (new_num_internal[level_index - 1] - is_padded) * this->tau_; - // Determine the number of children the last block generated - if (is_padded) { - const size_type last_block_parent_start = - previous_level.block_starts->back(); - const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); - } - previous_level.block_starts.reset(); - const size_type num_internal = new_num_internal[level_index]; - - // Allocate new vectors for the tree - auto* is_internal = new BitVector(new_size); - auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); - auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); - - // Number of non-pruned blocks before the current block - size_type num_non_pruned = 0; - // Number of back blocks before the current block - size_type num_back_blocks = 0; - // Number of pruned blocks before the current block - size_type num_pruned = 0; - - // We will reuse the allocated memory of the pointers vector to - // store the number of pruned blocks before the block. The - // invariant is that all values up to i are overwritten while all - // values starting after i will still be valid pointers - // This contains the number of pruned blocks before the block i - std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { - const size_type ptr = (*level.pointers)[i]; - prefix_pruned_blocks[i] = num_pruned; - - // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { - num_pruned++; - continue; - } - - // Add it to the is_internal bit vector - const bool block_is_internal = (*level.is_internal)[i]; - (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; - - if (block_is_internal) { - continue; - } - - // If it is a back block, add its pointer and offset - const size_type offset = (*level.offsets)[i]; - - (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; - (*offsets)[num_back_blocks] = offset; - ++num_back_blocks; - } - - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(is_internal); - this->block_tree_types_rs_.push_back(new Rank(*is_internal)); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(level.block_size); - } - - /// @brief Prunes the tree of unnecessary nodes. - /// @param levels The levels of the tre represented as a vector of levels. - void prune(std::vector& levels) { - // We need to traverse the block tree in post order, - // handling children from right to left - for (int block_index = levels[0].num_blocks - 1; block_index >= 0; - --block_index) { - prune_block(levels, 0, block_index); - } - } - - /// @brief Prunes a block and its descendants of unnecessary internal nodes. - /// @param levels The WIP levels of the tree. - /// @param level_index The level of the block to prune. - /// @param block_index The index of the block to prune. - /// @return Whether this block is/stays internal after the pruning process - bool prune_block(std::vector& levels, - const size_t level_index, - const size_t block_index) const { - LevelData& level = levels[level_index]; - auto& is_internal = *level.is_internal; - - // If the current block is a back block already, there is nothing - // to prune - if (!is_internal[block_index]) { - return false; - } - - const size_type first_child = - level.is_internal_rank->rank1(block_index) * this->tau_; - - bool has_internal_children = false; - - // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are - // not on the last level. - if (level_index < levels.size() - 1) { - const size_type last_child = - std::min(first_child + this->tau_ - 1, - levels[level_index + 1].is_internal->size() - 1); - // Iterate through children in reverse - for (size_type child = last_child; child >= first_child; --child) { - has_internal_children |= prune_block(levels, level_index + 1, child); - } - } - - // If any of the children is internal, this block stays internal - // as well - if (has_internal_children) { - return true; - } - - const size_type pointer = (*level.pointers)[block_index]; - const size_type offset = (*level.offsets)[block_index]; - const size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing - // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { - return true; - } - - // Now we know that there is an earlier occurrence, - // and nothing is pointing here. - // We will make this block here into a back block... - is_internal[block_index] = false; - (*level.counters)[pointer] += 1; - (*level.counters)[pointer + 1] += offset > 0; - - if (level_index == levels.size() - 1) { - return false; - } - - // ...and mark the children as pruned - LevelData& child_level = levels[level_index + 1]; - const size_type last_child = - std::min(first_child + this->tau_ - 1, - child_level.is_internal->size() - 1); - for (size_type child = last_child; child >= first_child; --child) { - const size_type child_pointer = (*child_level.pointers)[child]; - const size_type child_offset = (*child_level.offsets)[child]; -#ifdef BT_DBG - if (!(*child_level.is_internal)[child] && child_pointer < 0) { - std::cout << "non-internal node missing pointer" << std::endl; - std::cout << level_index << ", " << block_index << " / " - << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { - std::cout << "pruned node missing pointer" << std::endl; - } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - BT_ASSERT(child_pointer >= 0); -#endif - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; - } - - return false; - } - -public: - RecursiveBitBlockTreeSharded(const pasta::BitVector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads, - const size_t queue_size) { - const auto old = omp_get_max_threads(); - const auto old_dynamic = omp_get_dynamic(); - omp_set_dynamic(0); - omp_set_num_threads(static_cast(threads)); - this->tau_ = arity; - this->s_ = root_arity; - this->max_leaf_length_ = max_leaf_length; - this->num_bits_ = text.size(); - const std::span bytes(reinterpret_cast(text.data().data()), - ceil_div(text.size(), 8ULL)); - construct(bytes, threads, queue_size); - omp_set_dynamic(old_dynamic); - omp_set_num_threads(old); - } - - ~RecursiveBitBlockTreeSharded() { - for (auto& rank : this->block_tree_types_rs_) { - delete rank; - } - for (auto& bv : this->block_tree_types_) { - delete bv; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; - } - } }; } // namespace pasta diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index 51ea222..d0115ce 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -33,6 +33,7 @@ #include #include #include +#include #include namespace pasta { @@ -40,741 +41,16 @@ namespace pasta { template class RecursiveBitBlockTree { public: - /// If this is true, then the only levels of the tree start to be - /// included starting at the first level that contains a back block - /// - /// For example, if levels 0 to 5 do not contain any back blocks, then the - /// tree will only contain levels 6 and below. - bool CUT_FIRST_LEVELS = true; - - /// The arity of the tree - size_type tau_; - size_type max_leaf_length_; - /// The arity of the tree's root - size_type s_ = 1; - size_type leaf_size = 0; - size_type amount_of_leaves = 0; - size_type num_bits_; - bool rank_support = false; - /// Recursively compress the bit vectors of the tree - std::vector*> - block_tree_types_; - /// For each level and each back block, contains the index of the - /// block's source - std::vector*> block_tree_pointers_; - std::vector*> block_tree_offsets_; - // std::vector*> block_tree_encoded_; - std::vector block_size_lvl_; - std::vector block_per_lvl_; - std::vector leaves_; - - std::vector compress_map_; - std::vector decompress_map_; - sdsl::int_vector<> compressed_leaves_; - - /// @brief For each level and each block, contains the number of 1s up to (and - /// including) the block. - std::vector> one_ranks_; - /// @brief For each level and each back block, - /// contains the number of 1s up to (and including) the pointed-to area of - /// the back-block. - std::vector> pointer_prefix_one_counts_; - - [[nodiscard]] size_t height() const { - return block_tree_types_.size(); - } - - [[nodiscard]] size_t size() const { - return num_bits_; - } - - bool operator[](const size_type bit_index) const { - return access(bit_index); - } - - bool access(const size_type bit_index) const { - // FIXME: As of now this works on little endian systems only - const int64_t byte_index = bit_index / 8; - const int64_t bit_offset = bit_index % 8; - - int64_t block_size = block_size_lvl_[0]; - int64_t block_index = byte_index / block_size; - int64_t off = byte_index % block_size; - for (size_t i = 0; i < height(); i++) { - const auto& is_internal = *block_tree_types_[i]; - const auto& pointers = *block_tree_pointers_[i]; - const auto& offsets = *block_tree_offsets_[i]; - if (!is_internal.access(block_index)) { - // If this block is not internal, go to its pointed-to block - const size_t back_block_index = is_internal.rank0(block_index); - off = off + offsets[back_block_index]; - block_index = pointers[back_block_index]; - if (off >= block_size) { - ++block_index; - off -= block_size; - } - } - block_size /= tau_; - const int64_t child = off / block_size; - off %= block_size; - block_index = is_internal.rank1(block_index) * tau_ + child; - } - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; - return ((1 << bit_offset) & byte) != 0; - }; - -private: - template - [[nodiscard]] size_t find_initial_block(const size_t rank) const { - const auto& top_one_ranks = one_ranks_[0]; - const size_t block_size = block_size_lvl_[0]; - size_t start = (rank - 1) / (block_size * 8); - size_t end = top_one_ranks.size() - 1; - while (start != end) { - const size_t middle = start + (end - start) / 2; - size_t current_rank; - if constexpr (one) { - current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; - } else { - const size_t middle_bits = middle * block_size * 8; - current_rank = - (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; - } - if (current_rank < rank) { - if (start + 1 == end) { - size_t bits; - if constexpr (one) { - bits = top_one_ranks[middle]; - } else { - bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; - } - // If there is only one block left, it's either the current or the - // next block - if (bits < rank) { - start = middle + 1; - } - break; - } - start = middle; - } else { - end = middle - 1; - } - } - return start; - } - -public: - [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - size_t block_size = block_size_lvl_[0]; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - - size_t pos = (current_block * block_size * 8) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; - - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal.rank0(current_block); - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - size_t rank_d = - (current_block == 0) ? - top_one_ranks[current_block] : - top_one_ranks[current_block] - top_one_ranks[current_block - 1]; - rank_d -= pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& prev_is_internal = *block_tree_types_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal.rank1(current_block) * tau_; - block_size /= tau_; - const size_t start_block = current_block; - while (one_ranks[current_block] < rank) { - ++current_block; - } - rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; - pos += (current_block - start_block) * block_size * 8; - if (!is_internal.access(current_block)) { - size_t back_block_index = is_internal.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - size_t rank_d = - (current_block % tau_ == 0) ? - one_ranks[current_block] : - one_ranks[current_block] - one_ranks[current_block - 1]; - rank_d -= pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = block_tree_types_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_ones = std::popcount(byte); - if (rank > num_ones) { - rank -= num_ones; - pos += 8; - ++byte_offset; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) > 0; - } - } - } - return pos; - } - - [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - - const size_t top_block_size = block_size_lvl_[0]; - const auto top_zero_ranks = [&top_one_ranks, - top_block_size](const size_t i) -> size_t { - return (i + 1) * top_block_size * 8 - top_one_ranks[i]; - }; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - const size_t top_block_bits = top_block_size * 8; - - size_t pos = (current_block * top_block_bits) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal.rank0(current_block); - // const size_t child_block_bits = - // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block == 0) ? - top_zero_ranks(current_block) : - top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (top_block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t block_size = block_size_lvl_[0]; - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& prev_is_internal = *block_tree_types_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal.rank1(current_block) * tau_; - block_size /= tau_; - - const auto zero_ranks = - [&one_ranks, this, block_size](const size_t i) -> size_t { - const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; - return rnk; - }; - const size_t start_block = current_block; - while (zero_ranks(current_block) < rank) { - ++current_block; - } - rank -= - (current_block == start_block) ? 0 : zero_ranks(current_block - 1); - pos += (current_block - start_block) * block_size * 8; - if (!is_internal[current_block]) { - size_t back_block_index = is_internal.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block % tau_ == 0) ? - zero_ranks(current_block) : - zero_ranks(current_block) - zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = block_tree_types_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_zeros = 8 - std::popcount(byte); - if (rank > num_zeros) { - rank -= num_zeros; - pos += 8; - byte_offset++; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) == 0; - } - } - } - return pos; - } - - /// @brief Counts the number of 1-bits up to (and excluding) an index. - [[nodiscard("rank result discarded")]] size_t - rank1(const size_type bit_index) const { - const size_t byte_index = bit_index / 8; - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - size_t block_size = block_size_lvl_[0]; - size_t block_index = byte_index / block_size; - size_t block_offset = byte_index % block_size; - size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; - if (!top_is_internal[block_index]) { - // If the top block is a back block, go to it and adjust the offset - const size_t back_block_index = top_is_internal.rank0(block_index); - rank -= pointer_prefix_one_counts_[0][back_block_index]; - block_offset += top_offsets[back_block_index]; - block_index = top_pointers[back_block_index]; - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it - rank += - (block_index == 0) ? - one_ranks_[0][block_index] : - (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); - ++block_index; - block_offset -= block_size; - } - } - - // Go down to the next level - block_size /= tau_; - // How many children are we 'skipping over' - size_t child = block_offset / block_size; - block_offset %= block_size; - block_index = top_is_internal.rank1(block_index) * tau_ + child; - - size_t level = 1; - while (level < height()) { - const auto& ranks = one_ranks_[level]; - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - rank += (child == 0) ? 0 : ranks[block_index - 1]; - // If this block is internal, just go to the correct child - if (is_internal[block_index]) { - block_size /= tau_; - child = block_offset / block_size; - block_offset %= block_size; - block_index = is_internal.rank1(block_index) * tau_ + child; - level++; - continue; - } - - // If we have a back block, we need to go to the pointed-to block - const size_t back_block_index = is_internal.rank0(block_index); - rank -= pointer_ranks[back_block_index]; - block_offset += (*block_tree_offsets_[level])[back_block_index]; - block_index = (*block_tree_pointers_[level])[back_block_index]; - child = block_index % tau_; - - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it and go to the next block - rank += (child == 0) ? ranks[block_index] : - (ranks[block_index] - ranks[block_index - 1]); - ++block_index; - child = block_index % tau_; - block_offset -= block_size; - } - const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; - rank -= remove_prefix; - } - - // Number of leaves that exist before the leaves of the current block - const size_type prefix_leaves = block_index - child; - for (size_t block = 0; block < child * leaf_size; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + - block]]; - rank += std::popcount(byte); - } - for (size_t block = 0; block < block_offset; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; - rank += std::popcount(byte); - } - - // Masks to remove bits from the last byte, - // that aren't part of the ran query - static constexpr std::array MASKS = { - 0b0000'0000, - 0b0000'0001, - 0b0000'0011, - 0b0000'0111, - 0b0000'1111, - 0b0001'1111, - 0b0011'1111, - 0b0111'1111, - }; - rank += std::popcount( - decompress_map_[compressed_leaves_[block_index * leaf_size + - block_offset]] & - MASKS[bit_index % 8]); - return rank; - } - - /// @brief Counts the number of 0-bits up to (and excluding) an index. - size_t rank0(const size_type bit_index) const { - return bit_index - rank1(bit_index); - } - - size_t print_space_usage() const { - size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + - sizeof(leaf_size); - for (const auto bt : block_tree_types_) { - space_usage += bt->print_space_usage(); - } - for (const auto iv : block_tree_pointers_) { - space_usage += (int64_t)sdsl::size_in_bytes(*iv); - } - for (const auto iv : block_tree_offsets_) { - space_usage += sdsl::size_in_bytes(*iv); - } - if (rank_support) { - for (auto v : block_size_lvl_) { - space_usage += sizeof(v); - } - for (auto v : block_per_lvl_) { - space_usage += sizeof(v); - } - } - - for (auto& rs : one_ranks_) { - space_usage += sdsl::size_in_bytes(rs); - } - - for (auto& rs : pointer_prefix_one_counts_) { - space_usage += sdsl::size_in_bytes(rs); - } - - // space_usage += leaves_.size() * sizeof(uint8_t); - space_usage += sdsl::size_in_bytes(compressed_leaves_); - space_usage += compress_map_.size(); - - return space_usage; - }; - - int32_t add_bit_rank_support() { - rank_support = true; - - // Resize rank information vectors - one_ranks_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - one_ranks_[level].resize(block_tree_types_[level]->size()); - } - pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - pointer_prefix_one_counts_[level].resize( - block_tree_pointers_[level]->size()); - } - - for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { - bit_rank_block(0, block); - } + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = + std::conditional_t, + pasta::BitVector>; + using IsInternalRankType = + std::conditional_t, + pasta::RankSelect>; - for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { - one_ranks_[0][block] += one_ranks_[0][block - 1]; - } - - for (size_t level = 1; level < height(); level++) { - size_type counter = tau_; - size_t acc = 0; - for (size_t block = 0; block < one_ranks_[level].size(); block++) { - const size_type ones_in_block = one_ranks_[level][block]; - acc += ones_in_block; - one_ranks_[level][block] = acc; - --counter; - if (counter == 0) { - acc = 0; - counter = tau_; - } - } - } - for (auto& prefix_one_counts : pointer_prefix_one_counts_) { - sdsl::util::bit_compress(prefix_one_counts); - } - for (auto& ranks : one_ranks_) { - sdsl::util::bit_compress(ranks); - } - return 0; - } - -protected: - void compress_leaves() { - // Holds a 1 on every char that exists - compress_map_.resize(256, 0); - decompress_map_.resize(256, 0); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compress_map_[this->leaves_[i]] = 1; - } - for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { - const size_t tmp = compress_map_[c]; - compress_map_[c] = cur_val; - decompress_map_[cur_val] = c; - cur_val += tmp; - } - - compressed_leaves_.resize(this->leaves_.size()); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compressed_leaves_[i] = compress_map_[this->leaves_[i]]; - } - sdsl::util::bit_compress(this->compressed_leaves_); - leaves_.resize(0); - leaves_.shrink_to_fit(); - } - /// @brief Calculate the number of leading zeros for a 32-bit integer. - /// This value is capped at 31. - static size_type leading_zeros(const int32_t val) { - return __builtin_clz(static_cast(val) | 1); - } - - /// @brief Calculate the number of leading zeros for a 64-bit integer. - /// This value is capped at 64. - static size_type leading_zeros(const int64_t val) { - return __builtin_clzll(static_cast(val) | 1); - } - - /// - /// @brief Determine the padding and minimum height and the size of the blocks - /// on the top level of a block tree with s top-level blocks and an arity of - /// tau with leaves also of size tau. - /// - /// The height is the number of levels in the tree. - /// The padding is the number of characters that the top-level exceeds the - /// text length. For example, if the result was that the top level consists of - /// s = 5 blocks of size 30 and the text size being 80, then the padding would - /// be (5 * 30) - 80 = 70. - /// - /// @param[out] padding The number of characters in the last block (of the - /// first level of the tree) that are empty. - /// @param[in] text_length The number of characters in the input string. - /// @param[out] height The number of levels in the tree. - /// @param[out] blk_size The size of blocks on the first level of the tree. - /// - void calculate_padding(int64_t& padding, - int64_t text_length, - int64_t& height, - int64_t& blk_size) { - // This is the number of characters occupied by a tree with s*tau^h levels - // and leaves of size tau. At the start, we only have a tree with the first - // level with s leaf blocks which each have size tau. If we insert another - // level, the number of leaf blocks (and therefore the number of occupied - // characters) increases by a factor of tau. - int64_t tmp_padding = this->s_ * this->tau_; - int64_t h = 1; - // Size of the blocks on the current level (starting at the leaf level) - blk_size = tau_; - // While the tree does not cover the entire text, add a level - while (tmp_padding < text_length) { - tmp_padding *= this->tau_; - blk_size *= this->tau_; - h++; - } - // once the tree has enough levels to cover the entire text, we set the - // tree's values - height = h; - // The padding is the number of excess characters that the block tree covers - // over the length of the text. - padding = tmp_padding - text_length; - } - - size_type bit_rank_block(size_type level, size_type block_index) { - const auto& is_internal = *block_tree_types_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal[block_index]) { - const size_type internal_index = is_internal.rank1(block_index); - if (static_cast(level) < height() - 1) { - // If we are not on the last level recursively call - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); - } - } else { - // If we are on the last level - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - } - } - } else { - const size_type back_block_index = is_internal.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - size_type num_ones_parts = 0; - num_ones += one_ranks_[level][ptr]; - if (off > 0) { - num_ones_parts = part_bit_rank_block(level, ptr, off); - const size_type num_ones_2nd_part = - part_bit_rank_block(level, ptr + 1, off); - num_ones -= num_ones_parts; - num_ones += num_ones_2nd_part; - } - pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; - } - one_ranks_[level][block_index] = num_ones; - return num_ones; - } - - size_type part_bit_rank_block(const size_type level, - const size_type block_index, - const size_type chars_to_process) { - // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs - const auto& is_internal = *block_tree_types_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal.access(block_index)) { - const size_type internal_index = is_internal.rank1(block_index); - size_type k = 0; - size_type processed_chars = 0; - if (static_cast(level) < height() - 1) { - const size_type child_size = block_size_lvl_[level + 1]; - // We're not on the last level - // iterate over the children as long as we don't exceed the limit - for (k = 0; - k < tau_ && processed_chars + child_size <= chars_to_process; - ++k) { - num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; - processed_chars += child_size; - } - - // If we still need to process more chars and they end inside the next - // child, rank that part of the next child - if (processed_chars != chars_to_process) { - num_ones += part_bit_rank_block(level + 1, - internal_index * tau_ + k, - chars_to_process - processed_chars); - } - } else { - // We're on the last level - for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; - ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - processed_chars += leaf_size; - } - - if (processed_chars != chars_to_process) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, - chars_to_process % leaf_size); - } - } - } else { - const size_type back_block_index = is_internal.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - - // If we need to process chars beyond this block, we need to - if (chars_to_process + off >= block_size_lvl_[level]) { - // Ones in the entire block this block points to - num_ones += one_ranks_[level][ptr]; - // Ones that overflow into the next block - num_ones += part_bit_rank_block(level, - ptr + 1, - chars_to_process + off - - block_size_lvl_[level]); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } else { - // Number of ones up to the cutoff point - num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } - } - return num_ones; - } - - /// - /// @brief Count ones in leaf block. - /// - /// @param leaf_index The index of the leaf block. - /// @param max_char_index The maximum character index (exclusive) to - /// consider. This is used for when this block is at the end of the string. - /// @return The number of ones in this block. - /// - size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { - if (static_cast(leaf_index * leaf_size) >= - compressed_leaves_.size()) { - return 0; - } - - size_type result = 0; - for (size_type i = 0; i < max_char_index; ++i) { - const uint8_t byte = - decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; - result += std::popcount(byte); - } - return result; - } -}; - -template -class RecursiveBitBlockTree { -public: /// If this is true, then the only levels of the tree start to be /// included starting at the first level that contains a back block /// @@ -791,11 +67,9 @@ class RecursiveBitBlockTree { size_type amount_of_leaves = 0; size_type num_bits_; bool rank_support = false; - /// Bit vectors for each level determining whether a block is internal - /// (=1) or not (=0) - std::vector block_tree_types_; - std::vector*> - block_tree_types_rs_; + /// Recursively compress the bit vectors of the tree + std::vector block_tree_types_; + std::vector block_tree_types_rs_; /// For each level and each back block, contains the index of the /// block's source std::vector*> block_tree_pointers_; @@ -817,6 +91,26 @@ class RecursiveBitBlockTree { /// the back-block. std::vector> pointer_prefix_one_counts_; + ~RecursiveBitBlockTree() { + for (const IsInternalType* b : this->block_tree_types_) { + delete b; + } + // in any other case, block_tree_types_ and block_tree_types_rs_ point to + // the same object (a recursive block tree), so we may only free them once + if constexpr (recursion_level == 0) { + for (const RankSelect* rs : + this->block_tree_types_rs_) { + delete rs; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + [[nodiscard]] size_t height() const { return block_tree_types_.size(); } @@ -830,11 +124,10 @@ class RecursiveBitBlockTree { } bool access(const size_type bit_index) const { - // FIXME: As of now this works on little endian systems only const int64_t byte_index = bit_index / 8; const int64_t bit_offset = bit_index % 8; - int64_t block_size = height() == 0 ? leaf_size : block_size_lvl_[0]; + int64_t block_size = block_size_lvl_[0]; int64_t block_index = byte_index / block_size; int64_t off = byte_index % block_size; for (size_t i = 0; i < height(); i++) { @@ -1227,11 +520,18 @@ class RecursiveBitBlockTree { size_t print_space_usage() const { size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + sizeof(leaf_size); - for (const auto bv : block_tree_types_) { - space_usage += bv->size() / 8; + + for (const auto* bt : block_tree_types_) { + if constexpr (types_is_block_tree) { + space_usage += bt->print_space_usage(); + } else { + space_usage += bt->size() / 8; + } } - for (const auto rs : block_tree_types_rs_) { - space_usage += rs->space_usage(); + if constexpr (recursion_level == 0) { + for (const auto* rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + } } for (const auto iv : block_tree_pointers_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); @@ -1239,23 +539,20 @@ class RecursiveBitBlockTree { for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); } + space_usage += block_size_lvl_.size() * + sizeof(typename decltype(block_size_lvl_)::value_type); + space_usage += block_per_lvl_.size() * + sizeof(typename decltype(block_per_lvl_)::value_type); + if (rank_support) { - for (auto v : block_size_lvl_) { - space_usage += sizeof(v); + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); } - for (auto v : block_per_lvl_) { - space_usage += sizeof(v); + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); } } - for (auto& rs : one_ranks_) { - space_usage += sdsl::size_in_bytes(rs); - } - - for (auto& rs : pointer_prefix_one_counts_) { - space_usage += sdsl::size_in_bytes(rs); - } - // space_usage += leaves_.size() * sizeof(uint8_t); space_usage += sdsl::size_in_bytes(compressed_leaves_); space_usage += compress_map_.size(); @@ -1263,12 +560,16 @@ class RecursiveBitBlockTree { return space_usage; }; - int32_t add_bit_rank_support() { - rank_support = true; - if (height() == 0) { - return 0; + void + add_bit_rank_support(size_t threads = std::thread::hardware_concurrency()) { + // FIXME For the last level where block_tree_types_ is a bitvec, using + // multiple threads doesn't work for some reason + if constexpr (recursion_level == 0) { + threads = 1; } + rank_support = true; + // Resize rank information vectors one_ranks_.resize(height(), sdsl::int_vector<0>()); for (uint64_t level = 0; level < height(); level++) { @@ -1280,6 +581,7 @@ class RecursiveBitBlockTree { block_tree_pointers_[level]->size()); } +#pragma omp parallel for default(none) num_threads(threads) for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { bit_rank_block(0, block); } @@ -1288,6 +590,7 @@ class RecursiveBitBlockTree { one_ranks_[0][block] += one_ranks_[0][block - 1]; } +#pragma omp parallel for default(none) num_threads(threads) for (size_t level = 1; level < height(); level++) { size_type counter = tau_; size_t acc = 0; @@ -1308,7 +611,6 @@ class RecursiveBitBlockTree { for (auto& ranks : one_ranks_) { sdsl::util::bit_compress(ranks); } - return 0; } protected: @@ -1433,7 +735,6 @@ class RecursiveBitBlockTree { size_type part_bit_rank_block(const size_type level, const size_type block_index, const size_type chars_to_process) { - // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs const auto& is_internal = *block_tree_types_[level]; const auto& is_internal_rank = *block_tree_types_rs_[level]; if (static_cast(block_index) >= is_internal.size()) { From e256014c3ad441acecf41231c6929924811154ab Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 5 Dec 2023 20:41:10 +0100 Subject: [PATCH 63/92] temporary fix to add_bit_rank_support breaking --- examples/build_bt.cpp | 11 ++++++++--- include/pasta/block_tree/rec_bit_block_tree.hpp | 10 +++++++--- 2 files changed, 15 insertions(+), 6 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 8381f62..d064e57 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -257,11 +257,16 @@ int main(int argc, char** argv) { << std::endl; #endif + if (!std::filesystem::exists(file)) { + std::cerr << "File does not exist" << std::endl; + exit(1); + } + std::unique_ptr bv; std::vector text; { std::string input; - std::ifstream t(argv[1]); + std::ifstream t(file); std::stringstream buffer; buffer << t.rdbuf(); input = buffer.str(); @@ -303,7 +308,7 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree auto bt = - std::make_unique>(*bv, + std::make_unique>(*bv, arity, 1, leaf_length, @@ -313,7 +318,7 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); - bt->add_bit_rank_support(threads); + bt->add_bit_rank_support(1); auto elapsed_rs = std::chrono::duration_cast( Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index d0115ce..bfa8c65 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -562,14 +562,17 @@ class RecursiveBitBlockTree { void add_bit_rank_support(size_t threads = std::thread::hardware_concurrency()) { + if (rank_support) { + return; + } + rank_support = true; + // FIXME For the last level where block_tree_types_ is a bitvec, using // multiple threads doesn't work for some reason if constexpr (recursion_level == 0) { threads = 1; } - rank_support = true; - // Resize rank information vectors one_ranks_.resize(height(), sdsl::int_vector<0>()); for (uint64_t level = 0; level < height(); level++) { @@ -581,7 +584,8 @@ class RecursiveBitBlockTree { block_tree_pointers_[level]->size()); } -#pragma omp parallel for default(none) num_threads(threads) + // FIXME: breaks if parallelism is used + // #pragma omp parallel for default(none) num_threads(threads) for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { bit_rank_block(0, block); } From 4a8d5014eb1c2f288dd7c218a7363aa2aa87245a Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 5 Dec 2023 21:19:40 +0100 Subject: [PATCH 64/92] recursive block tree --- examples/build_bt.cpp | 39 +- .../construction/rec_block_tree_sharded.hpp | 1637 +++++++++++++++++ include/pasta/block_tree/rec_block_tree.hpp | 784 ++++++++ 3 files changed, 2452 insertions(+), 8 deletions(-) create mode 100644 include/pasta/block_tree/construction/rec_block_tree_sharded.hpp create mode 100644 include/pasta/block_tree/rec_block_tree.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index d064e57..10d1f5c 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -29,7 +29,9 @@ #include #include -#define PAR_SHARDED_SYNC_SMALL +#define RECURSION_LEVELS 0 + +#define REC_PAR_SHARDED #ifdef FP # include std::unique_ptr> @@ -127,6 +129,25 @@ make_bt(std::vector& text, queue_size); } # define ALGO_NAME "shard_sync_small" +#elif defined REC_PAR_SHARDED +# include +std::unique_ptr< + pasta::RecursiveBlockTreeSharded> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + return std::make_unique< + pasta::RecursiveBlockTreeSharded>( + text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "rec_shard" #elif defined PAR_PHMAP # include std::unique_ptr> @@ -307,13 +328,13 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree - auto bt = - std::make_unique>(*bv, - arity, - 1, - leaf_length, - threads, - queue_size); + auto bt = std::make_unique< + RecursiveBitBlockTreeSharded>(*bv, + arity, + 1, + leaf_length, + threads, + queue_size); auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); @@ -323,6 +344,7 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); const size_t rs_space = bt->print_space_usage(); + std::cout << " rec=" << RECURSION_LEVELS; std::cout << " time=" << elapsed << " space=" << no_rs_space << " time_rs=" << elapsed_rs << " space_rs=" << rs_space; std::cout << std::endl; @@ -394,6 +416,7 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); + std::cout << " rec=" << RECURSION_LEVELS; std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); std::cout << std::endl; diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp new file mode 100644 index 0000000..b6362ca --- /dev/null +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -0,0 +1,1637 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +namespace sharded { + +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + +} // namespace sharded + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveBlockTreeSharded + : public RecursiveBlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief For some block size (in bytes) i, return the number of trailing + /// zeros in a 64 bit integer when zeroing out characters that are not part + /// of the block. + constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + + /// @brief Masks used for the identity hash. These depend on endianness + constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } + } + + /// @brief Masks for identity hashes for a block size i (in bytes) + constexpr static std::array HASH_MASKS = masks(); + + /// @brief A marker for a block that has no earlier occurrence + constexpr static size_type NO_EARLIER_OCC = -1; + /// @brief A marker for a block that has been pruned + constexpr static size_type PRUNED = -2; + + /// @brief Base of the polynomial used for the Rabin-Karp hasher + constexpr static size_type SIGMA = 256; + + /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher + constexpr static uint8_t PRIME_EXPONENT = 107; + // constexpr static uint8_t PRIME_EXPONENT = 89; + // constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher + constexpr static uint128_t PRIME = pasta::primer(); + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + // robin_hood::unordered_flat_map>; + // std::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + +#define MIX + static uint64_t mix_select(uint64_t key) { +#ifdef MIX + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); +#endif + return key; + } + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } + }; + + /// @brief Contains data about the occurrences of a hashed block pair + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + + /// @brief Contains data about the occurrences of a hashed block + struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block pair + struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const RabinKarpHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } + }; + + /// @brief An update function for the sharded hash map that updates the + /// occurrences of a hashed block + struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const RabinKarpHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences init(const RabinKarpHash&, + InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } + }; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size * sizeof(input_type)) > + 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size * sizeof(input_type)) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[pair_size * sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::vector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = HASH_MASKS[pair_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = HASH_MASKS[block_size / sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { + RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::vector& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = + HASH_MASKS[level_data.block_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } + + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } + + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + RecursiveBlockTreeSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } +}; +} // namespace pasta diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp new file mode 100644 index 0000000..929ae7a --- /dev/null +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -0,0 +1,784 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { +template +class RecursiveBlockTree { +public: + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = + std::conditional_t, + pasta::BitVector>; + using IsInternalRankType = + std::conditional_t, + pasta::RankSelect>; + + /// @brief If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + size_type tau_; + size_type max_leaf_length_; + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + bool rank_support = false; + std::vector block_tree_types_; + std::vector block_tree_types_rs_; + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + ankerl::unordered_dense::map chars_index_; + std::vector chars_; + size_type u_chars_; + std::vector> top_level_c_ranks_; + std::vector>> c_ranks_; + std::vector>> pointer_c_ranks_; + + ~RecursiveBlockTree() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + if constexpr (!types_is_block_tree) { + for (auto& bv : this->block_tree_types_) { + delete bv; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + int64_t access(size_type index) { + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size_lvl_[0]; + int64_t off = index % block_size_lvl_[0]; + int64_t child; + for (size_type i = 0; static_cast(i) < block_tree_types_.size(); + i++) { + auto& lvl = *block_tree_types_[i]; + auto& lvl_rs = *block_tree_types_rs_[i]; + auto& lvl_ptr = *block_tree_pointers_[i]; + auto& lvl_off = *block_tree_offsets_[i]; + if (lvl[blk_pointer] == 0) { + size_type blk = lvl_rs.rank0(blk_pointer); + off = off + lvl_off[blk]; + blk_pointer = lvl_ptr[blk]; + if (off >= block_size) { + blk_pointer++; + off -= block_size; + } + } + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; + } + return decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; + }; + + int64_t select(input_type c, size_type j) { + auto c_index = chars_index_[c]; + auto& top_level = *block_tree_types_[0]; + + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + size_type current_block = (j - 1) / block_size_lvl_[0]; + size_type end_block = c_ranks_[c_index][0].size() - 1; + int64_t block_size = block_size_lvl_[0]; + // find first level block containing the jth occurrence of c with a bin + // search + while (current_block != end_block) { + size_type m = current_block + (end_block - current_block) / 2; + + size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; + if (f < j) { + if (end_block - current_block == 1) { + if (c_ranks_[c_index][0][m] < static_cast(j)) { + current_block = m + 1; + } + break; + } + current_block = m; + } else { + end_block = m - 1; + } + } + + // accumulator + int64_t s = current_block * block_size - 1; + // index that indicates how many c's are still unaccounted for + j -= (current_block == 0) ? 0 : c_ranks_[c_index][0][current_block - 1]; + // we translate unmarked blocks on the top level independently as it differs + // from the other levels + if (!top_level[current_block]) { + int64_t blk = top_level_rs.rank0(current_block); + current_block = top_level_ptr[blk]; + int64_t g = top_level_off[blk]; + int64_t rank_d = (current_block == 0) ? + c_ranks_[c_index][0][0] : + c_ranks_[c_index][0][current_block] - + c_ranks_[c_index][0][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][0][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][0][blk]; + s -= g; + } + } + uint64_t i = 1; + while (i < block_tree_types_.size()) { + auto& current_level = *block_tree_types_[i]; + auto& current_level_rs = *block_tree_types_rs_[i]; + auto& current_level_ptr = *block_tree_pointers_[i]; + auto& current_level_off = *block_tree_offsets_[i]; + auto& prev_level_rs = *block_tree_types_rs_[i - 1]; + current_block = prev_level_rs.rank1(current_block) * tau_; + block_size /= tau_; + int64_t k = current_block; + while ((int64_t)c_ranks_[c_index][i][current_block] < j) { + current_block++; + } + j -= (current_block == k) ? 0 : c_ranks_[c_index][i][current_block - 1]; + s += (current_block - k) * block_size; + if (!current_level[current_block]) { + int64_t blk = current_level_rs.rank0(current_block); + current_block = current_level_ptr[blk]; + int64_t g = current_level_off[blk]; + int64_t rank_d = (current_block % tau_ == 0) ? + c_ranks_[c_index][i][current_block] : + c_ranks_[c_index][i][current_block] - + c_ranks_[c_index][i][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][i][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][i][blk]; + s -= g; + } + } + i++; + } + + current_block = (*block_tree_types_rs_[i - 1]).rank1(current_block) * tau_; + int64_t l = 0; + while (j > 0) { + if (compressed_leaves_[current_block * leaf_size + l] == compress_map_[c]) + j--; + l++; + } + return s + l; + } + + int64_t rank_base(input_type c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + size_type to = off + top_level_off[blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (to >= block_size) { + int64_t adder = (child == 0) ? + c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + rank += adder; + blk_pointer++; + off = to - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + size_type to = off + ptr_off; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + + if (to >= block_size) { + auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + rank += adder; + blk_pointer++; + child = blk_pointer % tau_; + off = to - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + } + + int64_t rank(input_type c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + blk_pointer++; + off = off - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + blk_pointer++; + child = blk_pointer % tau_; + off = off - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + }; + + int64_t print_space_usage() { + int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + if constexpr (recursion_level > 0) { + for (auto bt : block_tree_types_) { + space_usage += bt->print_space_usage(); + } + } + if constexpr (recursion_level == 0) { + for (auto bv : block_tree_types_) { + space_usage += bv->size() / 8; + } + for (auto rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + } + } + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + for (const auto iv : block_tree_offsets_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + } + if (rank_support) { + for (auto c : chars_) { + int64_t sum = 0; + for (auto lvl : pointer_c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + for (auto lvl : c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + space_usage += sum; + } + } + + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + // space_usage += leaves_.size() * sizeof(input_type); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); + + return space_usage; + }; + + void compress_leaves() { + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + + int32_t add_rank_support() { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + int32_t add_rank_support_omp(int32_t threads) { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + omp_set_num_threads(threads); + +#pragma omp parallel for default(none) + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + inline size_type leading_zeros(int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + inline size_type leading_zeros(int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type rank_block(input_type c, size_type i, size_type j) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_block(c, i + 1, rank_blk * tau_ + k); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + size_type rank_g = 0; + rank_c += c_ranks_[chars_index_[c]][i][ptr]; + if (off != 0) { + rank_g = part_rank_block(c, i, ptr, off); + size_type rank_2nd = part_rank_block(c, i, ptr + 1, off); + rank_c -= rank_g; + rank_c += rank_2nd; + } + pointer_c_ranks_[chars_index_[c]][i][rank_0] = rank_g; + } + c_ranks_[chars_index_[c]][i][j] = rank_c; + return rank_c; + } + size_type + part_rank_block(input_type c, size_type i, size_type j, size_type g) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + block_size_lvl_[i + 1] <= g; k++) { + rank_c += c_ranks_[chars_index_[c]][i + 1][rank_blk * tau_ + k]; + k_sum += block_size_lvl_[i + 1]; + } + + if (k_sum != g) { + rank_c += part_rank_block(c, i + 1, rank_blk * tau_ + k, g - k_sum); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + leaf_size <= g; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + k_sum += leaf_size; + } + + if (k_sum != g) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, g % leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + if (g + off >= block_size_lvl_[i]) { + rank_c += c_ranks_[chars_index_[c]][i][ptr] - + pointer_c_ranks_[chars_index_[c]][i][rank_0] + + part_rank_block(c, i, ptr + 1, g + off - block_size_lvl_[i]); + } else { + rank_c += part_rank_block(c, i, ptr, g + off) - + pointer_c_ranks_[chars_index_[c]][i][rank_0]; + } + } + return rank_c; + } + size_type rank_leaf(input_type c, size_type leaf_index, size_type i) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + // size_type x = leaves_.size() - leaf_index * this->tau_; + // i = std::min(i, x); + size_type result = 0; + for (size_type ind = 0; ind < i; ind++) { + if (compressed_leaves_[leaf_index * leaf_size + ind] == + compress_map_[c]) { + result++; + } + } + return result; + } + + size_type map_unique_chars(const std::vector& text) { + this->u_chars_ = 0; + input_type i = 0; + for (auto a : text) { + if (chars_index_.find(a) == chars_index_.end()) { + chars_index_[a] = i; + i++; + chars_.push_back(a); + } + } + this->u_chars_ = i; + return 0; + }; + size_type + find_next_smallest_index_binary_search(size_type i, + std::vector& pVector) { + int64_t l = 0; + int64_t r = pVector.size(); + while (l < r) { + int64_t m = std::floor((l + r) / 2); + if (i < pVector[m]) { + r = m; + } else { + l = m + 1; + } + } + return r - 1; + }; + int64_t + find_next_smallest_index_linear_scan(size_type i, + std::vector& pVector) { + int64_t b = 0; + while (b < pVector.size() && i >= pVector[b]) { + b++; + } + return b - 1; + }; + size_type find_next_smallest_index_block_tree(size_type index) { + size_type block_size = this->block_size_lvl_[0]; + size_type blk_pointer = index / block_size; + size_type off = index % block_size; + size_type child = 0; + for (size_type i = 0; i < this->block_tree_types_.size(); i++) { + if ((*this->block_tree_types_[i])[blk_pointer] == 0) { + return -1; + } + if (off > 0 && (*this->block_tree_types_[i])[blk_pointer + 1] == 0) { + return -1; + } + size_type rank_blk = this->block_tree_types_rs_[i]->rank1(blk_pointer); + blk_pointer = rank_blk * this->tau_; + block_size /= this->tau_; + child = off / block_size; + off = off % block_size; + blk_pointer += child; + } + return blk_pointer; + }; +}; +} // namespace pasta +/******************************************************************************/ From e2fc0be90b8a31e0e7da516dc7c63eee37eb5562 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 6 Dec 2023 19:35:35 +0100 Subject: [PATCH 65/92] bitwise rabin karp hash --- .gitignore | 3 + include/pasta/block_tree/block_tree.hpp | 1 + .../pasta/block_tree/utils/MersenneHash.hpp | 103 ++++++++++++- .../block_tree/utils/MersenneRabinKarp.hpp | 141 ++++++++++++++++++ tests/CMakeLists.txt | 1 + tests/utils/bit_rabin_karp_test.cpp | 119 +++++++++++++++ 6 files changed, 361 insertions(+), 7 deletions(-) create mode 100644 tests/utils/bit_rabin_karp_test.cpp diff --git a/.gitignore b/.gitignore index 73d76ee..2c7cafb 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,6 @@ compile_commands.json .idea/* perf.data* .pdf +bt_* +*.txt +debug diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index 41a4510..93510f2 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -20,6 +20,7 @@ #pragma once +#include #include #include #include diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 2ff6962..63e464d 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -21,12 +21,11 @@ #pragma once -#include #include #include #include #include -#include +#include #include #include #include @@ -55,7 +54,6 @@ void print_hash_data() { } #endif - template class MersenneHash { public: @@ -118,16 +116,107 @@ class MersenneHash { }; }; +template <> +class MersenneHash { +public: + __extension__ typedef unsigned __int128 uint128_t; + /// @brief The whole string in which the substring lies + std::span text_; + uint128_t hash_; + /// @brief The bit start-position of the hashed substring + uint32_t start_; + /// @brief The number of bits in the hashed substring + uint32_t length_; + + template + constexpr MersenneHash(const std::span text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : text_(std::as_bytes(text)), + hash_(hash), + start_(start), + length_(length) {} + + MersenneHash(const pasta::BitVector& text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : MersenneHash(text.data(), hash, start, length){}; + + constexpr MersenneHash() : hash_(0), start_(0), length_(0){}; + + constexpr MersenneHash(const MersenneHash& other) = default; + constexpr MersenneHash(MersenneHash&& other) = default; + + MersenneHash& operator=(const MersenneHash& other) = default; + MersenneHash& operator=(MersenneHash&& other) = default; + + bool operator==(const MersenneHash& other) const { +#ifdef BT_INSTRUMENT + ++mersenne_hash_comparisons; +#endif + if (hash_ != other.hash_) { + return false; + } + size_t byte_index = start_ / 8; + size_t bit_index = start_ % 8; + size_t other_byte_index = other.start_ / 8; + size_t other_bit_index = other.start_ % 8; + + bool is_same = true; + for (size_t i = 0; i < length_; ++i) { + if (get_bit(text_, byte_index, bit_index) != + get_bit(other.text_, other_byte_index, other_bit_index)) { + is_same = false; + break; + } + ++bit_index; + ++other_bit_index; + if (bit_index == 8) { + bit_index = 0; + ++byte_index; + } + + if (other_bit_index == 8) { + other_bit_index = 0; + ++other_byte_index; + } + } + +#ifdef BT_INSTRUMENT + if (!is_same) { + // The hash is the same but the substring isn't => collision + ++mersenne_hash_collisions; + } else { + // The substrings are the same + ++mersenne_hash_equals; + } +#endif + return is_same; + }; + + [[nodiscard]] std::span overlapping_range() const { + return text_.subspan(start_ / 8, ((length_ - 1) / 8) + 1); + } + +private: + static bool get_bit(const std::span v, + const size_t byte_index, + const size_t bit_index) { + return (v[byte_index] & std::byte{static_cast(1 << bit_index)}) > + std::byte{0}; + } +}; + } // namespace pasta -namespace std { template -struct hash> { +struct std::hash> { typename pasta::MersenneHash::uint128_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; } -}; -} // namespace std +}; // namespace std /******************************************************************************/ diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 2e99807..bf75249 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -23,6 +23,7 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" +#include #include namespace pasta { @@ -159,6 +160,146 @@ class MersenneRabinKarp { }; }; +/// +/// @brief A Rabin-Karp rolling hasher for bitstrings. +/// +/// @tparam size_type The type to use for indexing etc. +/// @tparam mersenne_exponent If using a mersenne prime 2^p-1, then this should +/// be p. If this is 0, a normal modulus operation will be used. +/// Additionally, if this is != 0, then the prime_ attribute will be ignored +/// and '(1 << mersenne_exponent) - 1' will be used instead. +/// +template +class MersenneRabinKarp { +public: + /// The text being hashed + std::span text_; + uint64_t init_; + /// The window size of this hasher + uint64_t length_; + /// A large prime used for modulus operations + uint128_t prime_; + /// The current hash value + uint128_t hash_; + uint128_t max_sigma_; + + /// @brief Construct a new Rabin Karp hasher. + /// @param text The text to hash. (not just the window but the entire text) + /// @param init The start index of the first hashed window in the text. + /// @param length The window size. + /// @param prime A large prime used for modulus operations + /// iff not using mersenne_exponent. + template + MersenneRabinKarp(const std::span text, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : text_(std::as_bytes(text)), + init_(init), + length_(length), + prime_(prime) { + max_sigma_ = 1; + uint128_t fp = 0; + uint128_t sigma_c = 1; + for (uint64_t i = init_; i < init_ + length_; i++) { + fp = mersenneModulo(2 * fp + get_bit(i)); + } + for (uint64_t i = 0; i < length_ - 1; i++) { + sigma_c = mersenneModulo(2 * sigma_c); + } + hash_ = fp; + max_sigma_ = sigma_c; + } + + MersenneRabinKarp(const pasta::BitVector& text, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : MersenneRabinKarp(as_bytes(text.data()), init, length, prime) {} + + /// @brief Moves the hasher to the specified start index in the backing + /// vector. + void restart(const uint64_t index) { + if (index + length_ >= text_.size()) { + return; + } + init_ = index; + uint128_t fp = 0; + for (uint64_t i = init_; i < init_ + length_; i++) { + fp = fp * 2; + fp = mersenneModulo(fp + get_bit(i)); + } + hash_ = fp; + }; + + inline uint128_t mersenneModulo(uint128_t k) const { + if constexpr (mersenne_exponent == 0) { + return k % prime_; + } else { + constexpr static uint128_t MERSENNE = primer(); + uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); + i -= (i >= MERSENNE) * MERSENNE; + return i; + } + }; + + /// @brief Retrieves the hash value at the hasher's current position. + /// @return A MersenneHash object representing the current hash value. + [[nodiscard]] MersenneHash current_hash() const { + return {text_, hash_, init_, length_}; + } + + /// @brief Advances the hasher by one character. + void next() { + if (text_.size() <= init_ + length_) { + return; + } + + uint128_t fp = hash_; + const bool out_char = out_bit(); + const bool in_char = in_bit(); + const uint128_t out_char_influence = out_char * max_sigma_; + // Conditionally add the prime, of the out_char_influence is too large + if constexpr (mersenne_exponent == 0) { + fp += prime_ * (out_char_influence > hash_) - out_char_influence; + } else { + fp += primer() * (out_char_influence > hash_) - + out_char_influence; + } + fp *= 2; + fp += in_char; + fp = mersenneModulo(fp); + hash_ = fp; + init_++; + }; + +private: + [[nodiscard]] bool out_bit() const { + const size_t byte_index = init_ / 8; + const size_t bit_index = init_ % 8; + return get_bit(byte_index, bit_index); + } + + [[nodiscard]] bool in_bit() const { + const size_t idx = init_ + length_; + const size_t byte_index = idx / 8; + const size_t bit_index = idx % 8; + return get_bit(byte_index, bit_index); + } + + [[nodiscard]] bool get_bit(const size_t bit_index) const { + return (text_[bit_index / 8] & + std::byte{static_cast(1 << (bit_index % 8))}) > + std::byte{0}; + } + + [[nodiscard]] bool get_bit(const size_t byte_index, + const size_t bit_index) const { + return (text_[byte_index] & + std::byte{static_cast(1 << bit_index)}) > std::byte{0}; + } +}; + } // namespace pasta /******************************************************************************/ diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 77dba78..72fadf0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -39,5 +39,6 @@ pasta_block_tree_build_test(block_tree/block_tree_fp_test) pasta_block_tree_build_test(block_tree/block_tree_seq_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_parallel_test) +pasta_block_tree_build_test(utils/bit_rabin_karp_test) ################################################################################ diff --git a/tests/utils/bit_rabin_karp_test.cpp b/tests/utils/bit_rabin_karp_test.cpp new file mode 100644 index 0000000..56f81f3 --- /dev/null +++ b/tests/utils/bit_rabin_karp_test.cpp @@ -0,0 +1,119 @@ + +#include +#include +#include +#define asdasdkasld + +#include +#include +#include + +class BitRabinKarpTest : public ::testing::Test { +protected: + pasta::BitVector bv; + + void SetUp() override { + std::random_device rd; + std::mt19937 gen{rd()}; + std::uniform_int_distribution dist(1); + + constexpr size_t string_length = 100000; + bv.resize(string_length * 2 + 3); + for (size_t i = 0; i < string_length; ++i) { + bv[i] = dist(gen); + } + // Offset it by some amount to check that even misaligned bit sequences + // correctly match + bv[string_length] = false; + bv[string_length + 1] = false; + bv[string_length + 2] = false; + for (size_t i = 0; i < string_length; ++i) { + bv[string_length + 3 + i] = static_cast(bv[i]); + } + } + +public: + bool + compare_ranges(const size_t s1, const size_t s2, const size_t len) const { + for (size_t i = 0; i < len; ++i) { + if (get_bit(s1 + i) == get_bit(s2 + i)) { + return false; + } + } + return true; + } + + [[nodiscard]] bool get_bit(const size_t bit_index) const { + return bv[bit_index]; + } +}; + +TEST_F(BitRabinKarpTest, test_eq) { + constexpr std::array sizes = {13, 24, 59, 1220}; + + for (const size_t size : sizes) { + const size_t half_len = bv.size() / 2; + pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); + pasta::MersenneRabinKarp rk2(bv, + half_len + 3, + size, + (1ULL << 61) - 1); + for (size_t i = 0; i < half_len - size; ++i) { + const auto h1 = rk1.current_hash(); + const auto h2 = rk2.current_hash(); + if (h1 != h2) { + std::cerr << "h1: "; + for (const auto byte : h1.overlapping_range()) { + std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; + } + std::cerr << "\n"; + std::cerr << "h2: "; + for (const auto byte : h2.overlapping_range()) { + std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; + } + std::cerr << "\n"; + } + ASSERT_TRUE(h1 == h2); + rk1.next(); + rk2.next(); + } + } +} + +TEST_F(BitRabinKarpTest, test_rnd) { + constexpr std::array sizes = {13, 24, 59, 1220}; + + for (const size_t size : sizes) { + const size_t half_len = bv.size() / 2; + pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); + pasta::MersenneRabinKarp rk2(bv, + half_len, + size, + (1ULL << 61) - 1); + size_t offset = 0; + for (size_t i = 0; i < half_len - size; ++i) { + const auto h1 = rk1.current_hash(); + const auto h2 = rk2.current_hash(); + const bool success = + (h1 == h2) == compare_ranges(offset, half_len + offset, size); + if (!success) { + std::cerr << "h1: "; + for (const auto byte : h1.overlapping_range()) { + std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; + } + std::cerr << " offset: " << h1.start_ % 8 << "\n"; + std::cerr << "h2: "; + for (const auto byte : h2.overlapping_range()) { + std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; + } + std::cerr << " offset: " << h2.start_ % 8 << "\n"; + } + ASSERT_TRUE((h1 == h2) == compare_ranges(offset, half_len + offset, size)) + << "error at offset " << offset << " for half_len " << half_len + << " and window size " << size; + rk1.next(); + rk2.next(); + offset++; + } + } +} From 59b684024180a9f7cb301062cf7e7ec636cad605 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 7 Dec 2023 19:53:57 +0100 Subject: [PATCH 66/92] code cleanup --- examples/build_bt.cpp | 3 +- include/pasta/block_tree/bit_block_tree.hpp | 4 -- include/pasta/block_tree/block_tree.hpp | 5 -- .../pasta/block_tree/rec_bit_block_tree.hpp | 6 -- include/pasta/block_tree/rec_block_tree.hpp | 6 -- .../pasta/block_tree/utils/MersenneHash.hpp | 14 ++-- .../block_tree/utils/MersenneRabinKarp.hpp | 66 +++++++++---------- 7 files changed, 41 insertions(+), 63 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 10d1f5c..f11b3a8 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,8 +24,7 @@ #include #include #include -#include -#include +#include #include #include diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp index 15ad8fb..1b0267d 100644 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ b/include/pasta/block_tree/bit_block_tree.hpp @@ -23,14 +23,10 @@ #include #include -#include -#include #include -#include #include #include #include -#include #include #include diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index 93510f2..c69e202 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -21,16 +21,11 @@ #pragma once #include -#include #include #include -#include -#include -#include #include #include #include -#include #include #include #include diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index bfa8c65..8c61376 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -21,17 +21,11 @@ #pragma once -#include #include #include -#include -#include #include -#include #include -#include #include -#include #include #include #include diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index 929ae7a..7ff0daa 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -20,16 +20,10 @@ #pragma once -#include #include #include -#include -#include -#include #include -#include #include -#include #include #include #include diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 63e464d..ef6e747 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -149,10 +149,10 @@ class MersenneHash { constexpr MersenneHash(const MersenneHash& other) = default; constexpr MersenneHash(MersenneHash&& other) = default; - MersenneHash& operator=(const MersenneHash& other) = default; - MersenneHash& operator=(MersenneHash&& other) = default; + constexpr MersenneHash& operator=(const MersenneHash& other) = default; + constexpr MersenneHash& operator=(MersenneHash&& other) = default; - bool operator==(const MersenneHash& other) const { + constexpr bool operator==(const MersenneHash& other) const { #ifdef BT_INSTRUMENT ++mersenne_hash_comparisons; #endif @@ -196,14 +196,14 @@ class MersenneHash { return is_same; }; - [[nodiscard]] std::span overlapping_range() const { + [[nodiscard]] constexpr std::span overlapping_range() const { return text_.subspan(start_ / 8, ((length_ - 1) / 8) + 1); } private: - static bool get_bit(const std::span v, - const size_t byte_index, - const size_t bit_index) { + constexpr static bool get_bit(const std::span v, + const size_t byte_index, + const size_t bit_index) { return (v[byte_index] & std::byte{static_cast(1 << bit_index)}) > std::byte{0}; } diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index bf75249..f6791f7 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -31,7 +31,7 @@ namespace pasta { __extension__ typedef unsigned __int128 uint128_t; template -static constexpr uint128_t primer() { +static consteval uint128_t primer() { uint128_t res = 1; for (size_t i = 0; i < exponent; i++) { res <<= 1; @@ -72,11 +72,11 @@ class MersenneRabinKarp { /// @param length The window size. /// @param prime A large prime used for modulus operations /// iff not using mersenne_exponent. - MersenneRabinKarp(const std::span text, - const uint64_t sigma, - const uint64_t init, - const uint64_t length, - const uint128_t prime) + constexpr MersenneRabinKarp(const std::span text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) : text_(text), sigma_(sigma), init_(init), @@ -96,11 +96,11 @@ class MersenneRabinKarp { max_sigma_ = sigma_c; }; - MersenneRabinKarp(const std::vector& text, - const uint64_t sigma, - const uint64_t init, - const uint64_t length, - const uint128_t prime) + constexpr MersenneRabinKarp(const std::vector& text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) : MersenneRabinKarp(std::span(text), sigma, init, length, prime) {} /// @brief Moves the hasher to the specified start index in the backing @@ -118,11 +118,11 @@ class MersenneRabinKarp { hash_ = fp; }; - inline uint128_t mersenneModulo(uint128_t k) const { + constexpr uint128_t mersenneModulo(uint128_t k) const { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - constexpr static uint128_t MERSENNE = primer(); + constexpr uint128_t MERSENNE = primer(); uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); i -= (i >= MERSENNE) * MERSENNE; return i; @@ -131,12 +131,12 @@ class MersenneRabinKarp { /// @brief Retrieves the hash value at the hasher's current position. /// @return A MersenneHash object representing the current hash value. - inline MersenneHash current_hash() const { + constexpr MersenneHash current_hash() const { return MersenneHash(text_, hash_, init_, length_); } /// @brief Advances the hasher by one character. - void next() { + constexpr void next() { if (text_.size() <= init_ + length_) { return; } @@ -190,10 +190,10 @@ class MersenneRabinKarp { /// @param prime A large prime used for modulus operations /// iff not using mersenne_exponent. template - MersenneRabinKarp(const std::span text, - const uint64_t init, - const uint64_t length, - const uint128_t prime) + constexpr MersenneRabinKarp(const std::span text, + const uint64_t init, + const uint64_t length, + const uint128_t prime) : text_(std::as_bytes(text)), init_(init), length_(length), @@ -211,15 +211,15 @@ class MersenneRabinKarp { max_sigma_ = sigma_c; } - MersenneRabinKarp(const pasta::BitVector& text, - const uint64_t init, - const uint64_t length, - const uint128_t prime) + constexpr MersenneRabinKarp(const pasta::BitVector& text, + const uint64_t init, + const uint64_t length, + const uint128_t prime) : MersenneRabinKarp(as_bytes(text.data()), init, length, prime) {} /// @brief Moves the hasher to the specified start index in the backing /// vector. - void restart(const uint64_t index) { + constexpr void restart(const uint64_t index) { if (index + length_ >= text_.size()) { return; } @@ -232,11 +232,11 @@ class MersenneRabinKarp { hash_ = fp; }; - inline uint128_t mersenneModulo(uint128_t k) const { + [[nodiscard]] constexpr uint128_t mersenneModulo(const uint128_t k) const { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - constexpr static uint128_t MERSENNE = primer(); + constexpr uint128_t MERSENNE = primer(); uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); i -= (i >= MERSENNE) * MERSENNE; return i; @@ -245,12 +245,12 @@ class MersenneRabinKarp { /// @brief Retrieves the hash value at the hasher's current position. /// @return A MersenneHash object representing the current hash value. - [[nodiscard]] MersenneHash current_hash() const { + [[nodiscard]] constexpr MersenneHash current_hash() const { return {text_, hash_, init_, length_}; } /// @brief Advances the hasher by one character. - void next() { + constexpr void next() { if (text_.size() <= init_ + length_) { return; } @@ -274,27 +274,27 @@ class MersenneRabinKarp { }; private: - [[nodiscard]] bool out_bit() const { + [[nodiscard]] constexpr bool out_bit() const { const size_t byte_index = init_ / 8; const size_t bit_index = init_ % 8; return get_bit(byte_index, bit_index); } - [[nodiscard]] bool in_bit() const { + [[nodiscard]] constexpr bool in_bit() const { const size_t idx = init_ + length_; const size_t byte_index = idx / 8; const size_t bit_index = idx % 8; return get_bit(byte_index, bit_index); } - [[nodiscard]] bool get_bit(const size_t bit_index) const { + [[nodiscard]] constexpr bool get_bit(const size_t bit_index) const { return (text_[bit_index / 8] & std::byte{static_cast(1 << (bit_index % 8))}) > std::byte{0}; } - [[nodiscard]] bool get_bit(const size_t byte_index, - const size_t bit_index) const { + [[nodiscard]] constexpr bool get_bit(const size_t byte_index, + const size_t bit_index) const { return (text_[byte_index] & std::byte{static_cast(1 << bit_index)}) > std::byte{0}; } From cac48cef66ab39526156160710fde98a001cce7a Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 11 Dec 2023 17:23:37 +0100 Subject: [PATCH 67/92] fix algos not respecting leaf_length --- examples/build_bt.cpp | 16 +- include/pasta/block_tree/bit_block_tree.hpp | 780 ------------------ .../construction/bit_block_tree_sharded.hpp | 4 +- .../construction/block_tree_sharded.hpp | 5 +- .../rec_bit_block_tree_sharded.hpp | 42 +- .../construction/rec_block_tree_sharded.hpp | 25 +- .../pasta/block_tree/rec_bit_block_tree.hpp | 22 +- include/pasta/block_tree/rec_block_tree.hpp | 5 + .../pasta/block_tree/utils/MersenneHash.hpp | 83 +- .../block_tree/utils/MersenneRabinKarp.hpp | 43 +- 10 files changed, 131 insertions(+), 894 deletions(-) delete mode 100644 include/pasta/block_tree/bit_block_tree.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index f11b3a8..0d07740 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -18,17 +18,15 @@ * ******************************************************************************/ -#include "pasta/block_tree/utils/MersenneHash.hpp" - -#include #include #include #include #include +#include #include #include -#define RECURSION_LEVELS 0 +constexpr size_t RECURSION_LEVELS = 1; #define REC_PAR_SHARDED #ifdef FP @@ -112,7 +110,7 @@ make_bt(std::vector& text, } # define ALGO_NAME "shard_sync" #elif defined PAR_SHARDED_SYNC_SMALL -# include +# include std::unique_ptr> make_bt(std::vector& text, const size_t arity, @@ -208,6 +206,8 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; +using A = pasta::DenseBitBlockTreeSharded; + int main(int argc, char** argv) { using namespace pasta; @@ -327,6 +327,7 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree + auto bt = std::make_unique< RecursiveBitBlockTreeSharded>(*bv, arity, @@ -334,11 +335,14 @@ int main(int argc, char** argv) { leaf_length, threads, queue_size); + + // auto bt = + // std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); - bt->add_bit_rank_support(1); + bt->add_bit_rank_support(); auto elapsed_rs = std::chrono::duration_cast( Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/bit_block_tree.hpp b/include/pasta/block_tree/bit_block_tree.hpp deleted file mode 100644 index 1b0267d..0000000 --- a/include/pasta/block_tree/bit_block_tree.hpp +++ /dev/null @@ -1,780 +0,0 @@ -/******************************************************************************* - * This file is part of pasta::block_tree - * - * Copyright (C) 2022 Daniel Meyer - * Copyright (C) 2023 Etienne Palanga - * - * pasta::block_tree is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * pasta::block_tree is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with pasta::block_tree. If not, see . - * - ******************************************************************************/ - -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pasta { - -template -class BitBlockTree { -public: - /// If this is true, then the only levels of the tree start to be - /// included starting at the first level that contains a back block - /// - /// For example, if levels 0 to 5 do not contain any back blocks, then the - /// tree will only contain levels 6 and below. - bool CUT_FIRST_LEVELS = true; - - /// The arity of the tree - size_type tau_; - size_type max_leaf_length_; - /// The arity of the tree's root - size_type s_ = 1; - size_type leaf_size = 0; - size_type amount_of_leaves = 0; - size_type num_bits; - bool rank_support = false; - /// Bit vectors for each level determining whether a block is internal - /// (=1) or not (=0) - std::vector block_tree_types_; - std::vector*> - block_tree_types_rs_; - /// For each level and each back block, contains the index of the - /// block's source - std::vector*> block_tree_pointers_; - std::vector*> block_tree_offsets_; - // std::vector*> block_tree_encoded_; - std::vector block_size_lvl_; - std::vector block_per_lvl_; - std::vector leaves_; - - std::vector compress_map_; - std::vector decompress_map_; - sdsl::int_vector<> compressed_leaves_; - - /// @brief For each level and each block, contains the number of 1s up to (and - /// including) the block. - std::vector> one_ranks_; - /// @brief For each level and each back block, - /// contains the number of 1s up to (and including) the pointed-to area of - /// the back-block. - std::vector> pointer_prefix_one_counts_; - - [[nodiscard]] size_t height() const { - return block_tree_types_.size(); - } - - bool access(const size_type bit_index) const { - // FIXME: As of now this works on little endian systems only - const int64_t byte_index = bit_index / 8; - const int64_t bit_offset = bit_index % 8; - - int64_t block_size = block_size_lvl_[0]; - int64_t block_index = byte_index / block_size; - int64_t off = byte_index % block_size; - for (size_t i = 0; i < height(); i++) { - const auto& is_internal = *block_tree_types_[i]; - const auto& is_internal_rank = *block_tree_types_rs_[i]; - const auto& pointers = *block_tree_pointers_[i]; - const auto& offsets = *block_tree_offsets_[i]; - if (!is_internal[block_index]) { - // If this block is not internal, go to its pointed-to block - const size_t back_block_index = is_internal_rank.rank0(block_index); - off = off + offsets[back_block_index]; - block_index = pointers[back_block_index]; - if (off >= block_size) { - ++block_index; - off -= block_size; - } - } - block_size /= tau_; - const int64_t child = off / block_size; - off %= block_size; - block_index = is_internal_rank.rank1(block_index) * tau_ + child; - } - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; - return ((1 << bit_offset) & byte) != 0; - }; - -private: - template - [[nodiscard]] size_t find_initial_block(const size_t rank) const { - const auto& top_one_ranks = one_ranks_[0]; - const size_t block_size = block_size_lvl_[0]; - size_t start = (rank - 1) / (block_size * 8); - size_t end = top_one_ranks.size() - 1; - while (start != end) { - const size_t middle = start + (end - start) / 2; - size_t current_rank; - if constexpr (one) { - current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; - } else { - const size_t middle_bits = middle * block_size * 8; - current_rank = - (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; - } - if (current_rank < rank) { - if (start + 1 == end) { - size_t bits; - if constexpr (one) { - bits = top_one_ranks[middle]; - } else { - bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; - } - // If there is only one block left, it's either the current or the - // next block - if (bits < rank) { - start = middle + 1; - } - break; - } - start = middle; - } else { - end = middle - 1; - } - } - return start; - } - -public: - [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - size_t block_size = block_size_lvl_[0]; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - - size_t pos = (current_block * block_size * 8) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; - - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal_rank.rank0(current_block); - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - size_t rank_d = - (current_block == 0) ? - top_one_ranks[current_block] : - top_one_ranks[current_block] - top_one_ranks[current_block - 1]; - rank_d -= pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal_rank.rank1(current_block) * tau_; - block_size /= tau_; - const size_t start_block = current_block; - while (one_ranks[current_block] < rank) { - ++current_block; - } - rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; - pos += (current_block - start_block) * block_size * 8; - if (!is_internal[current_block]) { - size_t back_block_index = is_internal_rank.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - size_t rank_d = - (current_block % tau_ == 0) ? - one_ranks[current_block] : - one_ranks[current_block] - one_ranks[current_block - 1]; - rank_d -= pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = - block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_ones = std::popcount(byte); - if (rank > num_ones) { - rank -= num_ones; - pos += 8; - ++byte_offset; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) > 0; - } - } - } - return pos; - } - - [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - - const size_t top_block_size = block_size_lvl_[0]; - const auto top_zero_ranks = [&top_one_ranks, - top_block_size](const size_t i) -> size_t { - return (i + 1) * top_block_size * 8 - top_one_ranks[i]; - }; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - const size_t top_block_bits = top_block_size * 8; - - size_t pos = (current_block * top_block_bits) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal_rank.rank0(current_block); - // const size_t child_block_bits = - // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block == 0) ? - top_zero_ranks(current_block) : - top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (top_block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t block_size = block_size_lvl_[0]; - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal_rank.rank1(current_block) * tau_; - block_size /= tau_; - - const auto zero_ranks = - [&one_ranks, this, block_size](const size_t i) -> size_t { - const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; - return rnk; - }; - const size_t start_block = current_block; - while (zero_ranks(current_block) < rank) { - ++current_block; - } - rank -= - (current_block == start_block) ? 0 : zero_ranks(current_block - 1); - pos += (current_block - start_block) * block_size * 8; - if (!is_internal[current_block]) { - size_t back_block_index = is_internal_rank.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block % tau_ == 0) ? - zero_ranks(current_block) : - zero_ranks(current_block) - zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = - block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_zeros = 8 - std::popcount(byte); - if (rank > num_zeros) { - rank -= num_zeros; - pos += 8; - byte_offset++; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) == 0; - } - } - } - return pos; - } - - /// @brief Counts the number of 1-bits up to (and excluding) an index. - [[nodiscard("rank result discarded")]] size_t - rank1(const size_type bit_index) const { - const size_t byte_index = bit_index / 8; - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - size_t block_size = block_size_lvl_[0]; - size_t block_index = byte_index / block_size; - size_t block_offset = byte_index % block_size; - size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; - if (!top_is_internal[block_index]) { - // If the top block is a back block, go to it and adjust the offset - const size_t back_block_index = top_is_internal_rank.rank0(block_index); - rank -= pointer_prefix_one_counts_[0][back_block_index]; - block_offset += top_offsets[back_block_index]; - block_index = top_pointers[back_block_index]; - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it - rank += - (block_index == 0) ? - one_ranks_[0][block_index] : - (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); - ++block_index; - block_offset -= block_size; - } - } - - // Go down to the next level - block_size /= tau_; - // How many children are we 'skipping over' - size_t child = block_offset / block_size; - block_offset %= block_size; - block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; - - size_t level = 1; - while (level < height()) { - const auto& ranks = one_ranks_[level]; - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - rank += (child == 0) ? 0 : ranks[block_index - 1]; - // If this block is internal, just go to the correct child - if (is_internal[block_index]) { - block_size /= tau_; - child = block_offset / block_size; - block_offset %= block_size; - block_index = is_internal_rank.rank1(block_index) * tau_ + child; - level++; - continue; - } - - // If we have a back block, we need to go to the pointed-to block - const size_t back_block_index = is_internal_rank.rank0(block_index); - rank -= pointer_ranks[back_block_index]; - block_offset += (*block_tree_offsets_[level])[back_block_index]; - block_index = (*block_tree_pointers_[level])[back_block_index]; - child = block_index % tau_; - - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it and go to the next block - rank += (child == 0) ? ranks[block_index] : - (ranks[block_index] - ranks[block_index - 1]); - ++block_index; - child = block_index % tau_; - block_offset -= block_size; - } - const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; - rank -= remove_prefix; - } - - // Number of leaves that exist before the leaves of the current block - const size_type prefix_leaves = block_index - child; - for (size_t block = 0; block < child * leaf_size; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + - block]]; - rank += std::popcount(byte); - } - for (size_t block = 0; block < block_offset; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; - rank += std::popcount(byte); - } - - // Masks to remove bits from the last byte, - // that aren't part of the ran query - static constexpr std::array MASKS = { - 0b0000'0000, - 0b0000'0001, - 0b0000'0011, - 0b0000'0111, - 0b0000'1111, - 0b0001'1111, - 0b0011'1111, - 0b0111'1111, - }; - rank += std::popcount( - decompress_map_[compressed_leaves_[block_index * leaf_size + - block_offset]] & - MASKS[bit_index % 8]); - return rank; - } - - /// @brief Counts the number of 0-bits up to (and excluding) an index. - size_t rank0(const size_type bit_index) const { - return bit_index - rank1(bit_index); - } - - size_t print_space_usage() const { - size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + - sizeof(leaf_size); - for (const auto bv : block_tree_types_) { - space_usage += bv->size() / 8; - } - for (const auto rs : block_tree_types_rs_) { - space_usage += rs->space_usage(); - } - for (const auto iv : block_tree_pointers_) { - space_usage += (int64_t)sdsl::size_in_bytes(*iv); - } - for (const auto iv : block_tree_offsets_) { - space_usage += sdsl::size_in_bytes(*iv); - } - if (rank_support) { - for (auto v : block_size_lvl_) { - space_usage += sizeof(v); - } - for (auto v : block_per_lvl_) { - space_usage += sizeof(v); - } - } - - for (auto& rs : one_ranks_) { - space_usage += sdsl::size_in_bytes(rs); - } - - for (auto& rs : pointer_prefix_one_counts_) { - space_usage += sdsl::size_in_bytes(rs); - } - - // space_usage += leaves_.size() * sizeof(uint8_t); - space_usage += sdsl::size_in_bytes(compressed_leaves_); - space_usage += compress_map_.size(); - - return space_usage; - }; - - int32_t add_bit_rank_support() { - rank_support = true; - - // Resize rank information vectors - one_ranks_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - one_ranks_[level].resize(block_tree_types_[level]->size()); - } - pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - pointer_prefix_one_counts_[level].resize( - block_tree_pointers_[level]->size()); - } - - for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { - bit_rank_block(0, block); - } - - for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { - one_ranks_[0][block] += one_ranks_[0][block - 1]; - } - - for (size_t level = 1; level < height(); level++) { - size_type counter = tau_; - size_t acc = 0; - for (size_t block = 0; block < one_ranks_[level].size(); block++) { - const size_type ones_in_block = one_ranks_[level][block]; - acc += ones_in_block; - one_ranks_[level][block] = acc; - --counter; - if (counter == 0) { - acc = 0; - counter = tau_; - } - } - } - for (auto& prefix_one_counts : pointer_prefix_one_counts_) { - sdsl::util::bit_compress(prefix_one_counts); - } - for (auto& ranks : one_ranks_) { - sdsl::util::bit_compress(ranks); - } - return 0; - } - -protected: - void compress_leaves() { - // Holds a 1 on every char that exists - compress_map_.resize(256, 0); - decompress_map_.resize(256, 0); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compress_map_[this->leaves_[i]] = 1; - } - for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { - const size_t tmp = compress_map_[c]; - compress_map_[c] = cur_val; - decompress_map_[cur_val] = c; - cur_val += tmp; - } - - compressed_leaves_.resize(this->leaves_.size()); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compressed_leaves_[i] = compress_map_[this->leaves_[i]]; - } - sdsl::util::bit_compress(this->compressed_leaves_); - leaves_.resize(0); - leaves_.shrink_to_fit(); - } - /// @brief Calculate the number of leading zeros for a 32-bit integer. - /// This value is capped at 31. - static size_type leading_zeros(const int32_t val) { - return __builtin_clz(static_cast(val) | 1); - } - - /// @brief Calculate the number of leading zeros for a 64-bit integer. - /// This value is capped at 64. - static size_type leading_zeros(const int64_t val) { - return __builtin_clzll(static_cast(val) | 1); - } - - /// - /// @brief Determine the padding and minimum height and the size of the blocks - /// on the top level of a block tree with s top-level blocks and an arity of - /// tau with leaves also of size tau. - /// - /// The height is the number of levels in the tree. - /// The padding is the number of characters that the top-level exceeds the - /// text length. For example, if the result was that the top level consists of - /// s = 5 blocks of size 30 and the text size being 80, then the padding would - /// be (5 * 30) - 80 = 70. - /// - /// @param[out] padding The number of characters in the last block (of the - /// first level of the tree) that are empty. - /// @param[in] text_length The number of characters in the input string. - /// @param[out] height The number of levels in the tree. - /// @param[out] blk_size The size of blocks on the first level of the tree. - /// - void calculate_padding(int64_t& padding, - int64_t text_length, - int64_t& height, - int64_t& blk_size) { - // This is the number of characters occupied by a tree with s*tau^h levels - // and leaves of size tau. At the start, we only have a tree with the first - // level with s leaf blocks which each have size tau. If we insert another - // level, the number of leaf blocks (and therefore the number of occupied - // characters) increases by a factor of tau. - int64_t tmp_padding = this->s_ * this->tau_; - int64_t h = 1; - // Size of the blocks on the current level (starting at the leaf level) - blk_size = tau_; - // While the tree does not cover the entire text, add a level - while (tmp_padding < text_length) { - tmp_padding *= this->tau_; - blk_size *= this->tau_; - h++; - } - // once the tree has enough levels to cover the entire text, we set the - // tree's values - height = h; - // The padding is the number of excess characters that the block tree covers - // over the length of the text. - padding = tmp_padding - text_length; - } - - size_type bit_rank_block(size_type level, size_type block_index) { - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal[block_index]) { - const size_type internal_index = is_internal_rank.rank1(block_index); - if (static_cast(level) < height() - 1) { - // If we are not on the last level recursively call - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); - } - } else { - // If we are on the last level - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - } - } - } else { - const size_type back_block_index = is_internal_rank.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - size_type num_ones_parts = 0; - num_ones += one_ranks_[level][ptr]; - if (off > 0) { - num_ones_parts = part_bit_rank_block(level, ptr, off); - const size_type num_ones_2nd_part = - part_bit_rank_block(level, ptr + 1, off); - num_ones -= num_ones_parts; - num_ones += num_ones_2nd_part; - } - pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; - } - one_ranks_[level][block_index] = num_ones; - return num_ones; - } - - size_type part_bit_rank_block(const size_type level, - const size_type block_index, - const size_type chars_to_process) { - // FIXME: Seems to be kinda broken. Doesn't seem to report all bits it needs - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal[block_index]) { - const size_type internal_index = is_internal_rank.rank1(block_index); - size_type k = 0; - size_type processed_chars = 0; - if (static_cast(level) < height() - 1) { - const size_type child_size = block_size_lvl_[level + 1]; - // We're not on the last level - // iterate over the children as long as we don't exceed the limit - for (k = 0; - k < tau_ && processed_chars + child_size <= chars_to_process; - ++k) { - num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; - processed_chars += child_size; - } - - // If we still need to process more chars and they end inside the next - // child, rank that part of the next child - if (processed_chars != chars_to_process) { - num_ones += part_bit_rank_block(level + 1, - internal_index * tau_ + k, - chars_to_process - processed_chars); - } - } else { - // We're on the last level - for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; - ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - processed_chars += leaf_size; - } - - if (processed_chars != chars_to_process) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, - chars_to_process % leaf_size); - } - } - } else { - const size_type back_block_index = is_internal_rank.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - - // If we need to process chars beyond this block, we need to - if (chars_to_process + off >= block_size_lvl_[level]) { - // Ones in the entire block this block points to - num_ones += one_ranks_[level][ptr]; - // Ones that overflow into the next block - num_ones += part_bit_rank_block(level, - ptr + 1, - chars_to_process + off - - block_size_lvl_[level]); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } else { - // Number of ones up to the cutoff point - num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } - } - return num_ones; - } - - /// - /// @brief Count ones in leaf block. - /// - /// @param leaf_index The index of the leaf block. - /// @param max_char_index The maximum character index (exclusive) to - /// consider. This is used for when this block is at the end of the string. - /// @return The number of ones in this block. - /// - size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { - if (static_cast(leaf_index * leaf_size) >= - compressed_leaves_.size()) { - return 0; - } - - size_type result = 0; - for (size_type i = 0; i < max_char_index; ++i) { - const uint8_t byte = - decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; - result += std::popcount(byte); - } - return result; - } -}; - -} // namespace pasta - -/******************************************************************************/ diff --git a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp index 258023e..3ed7cfc 100644 --- a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp @@ -21,7 +21,7 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/bit_block_tree.hpp" +#include "pasta/block_tree/rec_bit_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/byteread.hpp" @@ -1378,6 +1378,8 @@ class BitBlockTreeSharded : public BitBlockTree { b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } diff --git a/include/pasta/block_tree/construction/block_tree_sharded.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp index 480b9b7..676c362 100644 --- a/include/pasta/block_tree/construction/block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -21,14 +21,13 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include -#include #include #include #include @@ -1384,6 +1383,8 @@ class BlockTreeSharded : public BlockTree { b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index 18968d0..be3705f 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -29,7 +29,6 @@ #include #include -#include #include #include #include @@ -42,15 +41,6 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { -/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows -/// or just use the block's content itself as a hash, stored in an integer. -enum class UseHash { - /// @brief Use a Rabin-Karp hash - RABIN_KARP, - /// @brief Use the block's content as a hash - IDENTITY -}; - /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes /// and a sharded hash map. Small blocks are not RK-hashed but rather use the /// blocks themselves. @@ -62,6 +52,15 @@ class RecursiveBitBlockTreeSharded using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; + /// @brief Determine whether to use a Rabin-Karp hash for hashing text windows + /// or just use the block's content itself as a hash, stored in an integer. + enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY + }; + /// @brief For some block size (in bytes) i, return the number of trailing /// zeros in a 64 bit integer when zeroing out characters that are not part /// of the block. @@ -107,7 +106,7 @@ class RecursiveBitBlockTreeSharded /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher constexpr static uint8_t PRIME_EXPONENT = 107; // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; + // constexpr static uint8_t PRIME_EXPONENT = 61; /// @brief A mersenne prime used for the Rabin-Karp hasher constexpr static uint128_t PRIME = pasta::primer(); @@ -120,9 +119,6 @@ class RecursiveBitBlockTreeSharded template using SeqHashMap = ankerl::unordered_dense::map>; - // robin_hood::unordered_flat_map>; - // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters @@ -137,15 +133,12 @@ class RecursiveBitBlockTreeSharded using RabinKarpMap = SyncShardedMap; -#define MIX static uint64_t mix_select(uint64_t key) { -#ifdef MIX key ^= (key >> 31); key *= 0x7fb5d329728ea185; key ^= (key >> 27); key *= 0x81dadef4bc2dd44d; key ^= (key >> 33); -#endif return key; } @@ -483,14 +476,17 @@ class RecursiveBitBlockTreeSharded #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); - } #ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif + } else { + break; + } } #ifdef BT_INSTRUMENT # if defined(BT_DBG) @@ -1397,6 +1393,8 @@ class RecursiveBitBlockTreeSharded b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index b6362ca..6f0b40e 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -24,11 +24,13 @@ #include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include #include +#include #include #include #include @@ -126,9 +128,6 @@ class RecursiveBlockTreeSharded template using SeqHashMap = ankerl::unordered_dense::map>; - // robin_hood::unordered_flat_map>; - // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters @@ -490,15 +489,19 @@ class RecursiveBlockTreeSharded #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); - } #ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif + } else { + break; + } } + #ifdef BT_INSTRUMENT # if defined(BT_DBG) std::cout << "pairs: " << (pairs_ns / 1'000'000) @@ -1361,7 +1364,9 @@ class RecursiveBlockTreeSharded found_back_block |= static_cast(new_num_internal[level_index]) < levels[level_index].is_internal->size(); if (!found_back_block) { - level.is_internal.reset(); + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); @@ -1407,6 +1412,8 @@ class RecursiveBlockTreeSharded b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index 8c61376..c18d3c5 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -22,7 +22,6 @@ #pragma once #include -#include #include #include #include @@ -515,24 +514,37 @@ class RecursiveBitBlockTree { size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + sizeof(leaf_size); + auto delta_size = 0; for (const auto* bt : block_tree_types_) { if constexpr (types_is_block_tree) { space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); } else { space_usage += bt->size() / 8; + delta_size += bt->size() / 8; } } + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; if constexpr (recursion_level == 0) { for (const auto* rs : block_tree_types_rs_) { space_usage += rs->space_usage(); + delta_size += rs->space_usage(); } + std::cout << "rs size: " << delta_size << std::endl; } + delta_size = 0; for (const auto iv : block_tree_pointers_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); + ; } + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); } + std::cout << "offs size: " << delta_size << std::endl; space_usage += block_size_lvl_.size() * sizeof(typename decltype(block_size_lvl_)::value_type); space_usage += block_per_lvl_.size() * @@ -817,14 +829,18 @@ class RecursiveBitBlockTree { size_type result = 0; for (size_type i = 0; i < max_char_index; ++i) { - const uint8_t byte = - decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; + const uint8_t compressed_byte = + compressed_leaves_[leaf_index * leaf_size + i]; + const uint8_t byte = decompress_map_[compressed_byte]; result += std::popcount(byte); } return result; } }; +template +using BitBlockTree = RecursiveBitBlockTree; + } // namespace pasta /******************************************************************************/ diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index 7ff0daa..9703d5b 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,6 +21,7 @@ #pragma once +#include #include #include #include @@ -774,5 +776,8 @@ class RecursiveBlockTree { return blk_pointer; }; }; + +template +using BlockTree = RecursiveBlockTree; } // namespace pasta /******************************************************************************/ diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index ef6e747..0167c12 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -121,30 +121,23 @@ class MersenneHash { public: __extension__ typedef unsigned __int128 uint128_t; /// @brief The whole string in which the substring lies - std::span text_; + const pasta::BitVector* text_; uint128_t hash_; /// @brief The bit start-position of the hashed substring - uint32_t start_; + uint64_t start_; /// @brief The number of bits in the hashed substring - uint32_t length_; - - template - constexpr MersenneHash(const std::span text, - const uint128_t hash, - const uint64_t start, - const uint64_t length) - : text_(std::as_bytes(text)), - hash_(hash), - start_(start), - length_(length) {} + uint64_t length_; MersenneHash(const pasta::BitVector& text, const uint128_t hash, const uint64_t start, const uint64_t length) - : MersenneHash(text.data(), hash, start, length){}; + : text_{&text}, + hash_{hash}, + start_{start}, + length_{length} {}; - constexpr MersenneHash() : hash_(0), start_(0), length_(0){}; + constexpr MersenneHash() : text_(nullptr), hash_(0), start_(0), length_(0){}; constexpr MersenneHash(const MersenneHash& other) = default; constexpr MersenneHash(MersenneHash&& other) = default; @@ -152,38 +145,34 @@ class MersenneHash { constexpr MersenneHash& operator=(const MersenneHash& other) = default; constexpr MersenneHash& operator=(MersenneHash&& other) = default; - constexpr bool operator==(const MersenneHash& other) const { + [[gnu::noinline]] bool operator==(const MersenneHash& other) const { #ifdef BT_INSTRUMENT ++mersenne_hash_comparisons; #endif if (hash_ != other.hash_) { return false; } - size_t byte_index = start_ / 8; - size_t bit_index = start_ % 8; - size_t other_byte_index = other.start_ / 8; - size_t other_bit_index = other.start_ % 8; + size_t pos = 0; bool is_same = true; - for (size_t i = 0; i < length_; ++i) { - if (get_bit(text_, byte_index, bit_index) != - get_bit(other.text_, other_byte_index, other_bit_index)) { + for (size_t remaining = length_; remaining > 64; + pos += 64, remaining -= 64) { + if (slice_at(*text_, start_ + pos) != + slice_at(*other.text_, other.start_ + pos)) { is_same = false; - break; - } - ++bit_index; - ++other_bit_index; - if (bit_index == 8) { - bit_index = 0; - ++byte_index; + goto mersenne_hash_cmp_done; } + } - if (other_bit_index == 8) { - other_bit_index = 0; - ++other_byte_index; + for (size_t i = pos; i < length_; ++i) { + if (get_bit(*text_, start_ + i) != + get_bit(*other.text_, other.start_ + i)) { + is_same = false; + goto mersenne_hash_cmp_done; } } + mersenne_hash_cmp_done: #ifdef BT_INSTRUMENT if (!is_same) { // The hash is the same but the substring isn't => collision @@ -196,16 +185,28 @@ class MersenneHash { return is_same; }; - [[nodiscard]] constexpr std::span overlapping_range() const { - return text_.subspan(start_ / 8, ((length_ - 1) / 8) + 1); +private: + static bool get_bit(const pasta::BitVector& v, const size_t bit_index) { + return v[bit_index]; } -private: - constexpr static bool get_bit(const std::span v, - const size_t byte_index, - const size_t bit_index) { - return (v[byte_index] & std::byte{static_cast(1 << bit_index)}) > - std::byte{0}; + static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + static uint64_t slice_at(const pasta::BitVector& bv, const size_t i) { + const std::span backing = bv.data(); + const uint8_t offset = i % 64; + const size_t data_index = i / 64; + // TODO Check if the right shift actually shifts in zeros + const uint64_t r = + backing[data_index] & (~static_cast(0) << offset); + if (offset == 0) { + const uint64_t l = backing[data_index + 1] & + (~static_cast(0) >> (63 - offset)); + return (l << (63 - offset)) | (r >> offset); + } + return r; } }; diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index f6791f7..1472fcc 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -173,7 +173,7 @@ template class MersenneRabinKarp { public: /// The text being hashed - std::span text_; + const pasta::BitVector& text_; uint64_t init_; /// The window size of this hasher uint64_t length_; @@ -183,18 +183,18 @@ class MersenneRabinKarp { uint128_t hash_; uint128_t max_sigma_; - /// @brief Construct a new Rabin Karp hasher. - /// @param text The text to hash. (not just the window but the entire text) - /// @param init The start index of the first hashed window in the text. - /// @param length The window size. + /// @brief Construct a new Rabin Karp hasher over a bitstring. + /// @param text The bitvector to hash. (not just the window but the entire + /// text) + /// @param init The start bit-index of the first hashed window in the text. + /// @param length The window size in bits. /// @param prime A large prime used for modulus operations /// iff not using mersenne_exponent. - template - constexpr MersenneRabinKarp(const std::span text, + constexpr MersenneRabinKarp(const pasta::BitVector& text, const uint64_t init, const uint64_t length, const uint128_t prime) - : text_(std::as_bytes(text)), + : text_(text), init_(init), length_(length), prime_(prime) { @@ -211,12 +211,6 @@ class MersenneRabinKarp { max_sigma_ = sigma_c; } - constexpr MersenneRabinKarp(const pasta::BitVector& text, - const uint64_t init, - const uint64_t length, - const uint128_t prime) - : MersenneRabinKarp(as_bytes(text.data()), init, length, prime) {} - /// @brief Moves the hasher to the specified start index in the backing /// vector. constexpr void restart(const uint64_t index) { @@ -275,28 +269,17 @@ class MersenneRabinKarp { private: [[nodiscard]] constexpr bool out_bit() const { - const size_t byte_index = init_ / 8; - const size_t bit_index = init_ % 8; - return get_bit(byte_index, bit_index); + return get_bit(init_); } [[nodiscard]] constexpr bool in_bit() const { const size_t idx = init_ + length_; - const size_t byte_index = idx / 8; - const size_t bit_index = idx % 8; - return get_bit(byte_index, bit_index); - } - - [[nodiscard]] constexpr bool get_bit(const size_t bit_index) const { - return (text_[bit_index / 8] & - std::byte{static_cast(1 << (bit_index % 8))}) > - std::byte{0}; + return get_bit(idx); } - [[nodiscard]] constexpr bool get_bit(const size_t byte_index, - const size_t bit_index) const { - return (text_[byte_index] & - std::byte{static_cast(1 << bit_index)}) > std::byte{0}; + [[nodiscard, gnu::noinline]] constexpr bool + get_bit(const size_t bit_index) const { + return text_[bit_index]; } }; From 6e62d508ca583afa5463e334861c9549c0052182 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 11 Dec 2023 19:09:49 +0100 Subject: [PATCH 68/92] reduce code duplication --- examples/build_bt.cpp | 4 +- .../construction/block_tree_sharded.hpp | 448 +++-------- .../rec_bit_block_tree_sharded.hpp | 291 +------ .../construction/rec_block_tree_sharded.hpp | 740 +++++++----------- .../pasta/block_tree/utils/sharded_util.hpp | 297 +++++++ 5 files changed, 713 insertions(+), 1067 deletions(-) create mode 100644 include/pasta/block_tree/utils/sharded_util.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 0d07740..c2d9ff6 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -28,7 +28,7 @@ constexpr size_t RECURSION_LEVELS = 1; -#define REC_PAR_SHARDED +#define PAR_SHARDED_SYNC_SMALL #ifdef FP # include std::unique_ptr> @@ -110,7 +110,7 @@ make_bt(std::vector& text, } # define ALGO_NAME "shard_sync" #elif defined PAR_SHARDED_SYNC_SMALL -# include +# include std::unique_ptr> make_bt(std::vector& text, const size_t arity, diff --git a/include/pasta/block_tree/construction/block_tree_sharded.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp index 676c362..c56b64c 100644 --- a/include/pasta/block_tree/construction/block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -24,11 +24,12 @@ #include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include -#include #include #include #include @@ -40,19 +41,6 @@ __extension__ typedef unsigned __int128 uint128_t; namespace pasta { -namespace sharded { - -/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows -/// or just use the block's content itself as a hash, stored in an integer. -enum class UseHash { - /// @brief Use a Rabin-Karp hash - RABIN_KARP, - /// @brief Use the block's content as a hash - IDENTITY -}; - -} // namespace sharded - /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes /// and a sharded hash map. Small blocks are not RK-hashed but rather use the /// blocks themselves. @@ -61,74 +49,37 @@ enum class UseHash { /// in the sharded hash map. template class BlockTreeSharded : public BlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; - /// @brief For some block size (in bytes) i, return the number of trailing - /// zeros in a 64 bit integer when zeroing out characters that are not part - /// of the block. - constexpr static uint64_t MASK_TRAILING_ZEROS[9] = - {64, 56, 48, 40, 32, 24, 16, 8, 0}; - - /// @brief Masks used for the identity hash. These depend on endianness - constexpr static std::array masks() { - if constexpr (std::endian::native == std::endian::big) { - return {0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; - } else { - return {0, - static_cast(~0) >> MASK_TRAILING_ZEROS[1], - static_cast(~0) >> MASK_TRAILING_ZEROS[2], - static_cast(~0) >> MASK_TRAILING_ZEROS[3], - static_cast(~0) >> MASK_TRAILING_ZEROS[4], - static_cast(~0) >> MASK_TRAILING_ZEROS[5], - static_cast(~0) >> MASK_TRAILING_ZEROS[6], - static_cast(~0) >> MASK_TRAILING_ZEROS[7], - static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; - } - } - - /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static std::array HASH_MASKS = masks(); - - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); - /// @brief A bit vector using BitVector = pasta::BitVector; /// @brief A rank data structure for a bit vector using Rank = pasta::RankSelect; + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = ankerl::unordered_dense::map>; - // robin_hood::unordered_flat_map>; - // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; @@ -139,17 +90,14 @@ class BlockTreeSharded : public BlockTree { using RabinKarpMap = SyncShardedMap; -#define MIX - static uint64_t mix_select(uint64_t key) { -#ifdef MIX - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); -#endif - return key; - } + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on #ifdef BT_INSTRUMENT public: @@ -163,219 +111,6 @@ class BlockTreeSharded : public BlockTree { size_t b_update_blocks_ns = 0; #endif -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - int64_t block_size; - /// @brief The index of the current level. - /// First level is 0, second level is 1 etc. - int64_t level_index; - /// @brief The number of blocks on the current level - int64_t num_blocks; - - LevelData(const int64_t level_index_, - const int64_t block_size_, - const int64_t num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - PairOccurrences(PairOccurrences&&) noexcept = default; - PairOccurrences& operator=(PairOccurrences&&) = default; - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - ~BlockOccurrences() = default; - - BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { - first_occ = other.first_occ.load(); - occurrences = std::move(other.occurrences); - return *this; - } - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_offset The offset of that occurrence - void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - /// @brief Constructs the block tree. /// @param text The input text. /// @param threads The number of threads to use for construction @@ -447,17 +182,17 @@ class BlockTreeSharded : public BlockTree { LevelData& current = levels.back(); if (2 * static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( @@ -466,17 +201,17 @@ class BlockTreeSharded : public BlockTree { now = Clock::now(); #endif if (static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( @@ -583,7 +318,7 @@ class BlockTreeSharded : public BlockTree { /// use the blocks' contents themselves as hashes. /// For block sizes greater than 4 bytes, use Rabin-Karp. /// - template + template void scan_block_pairs(const std::vector& text, LevelData& level, const bool is_padded, @@ -625,10 +360,18 @@ class BlockTreeSharded : public BlockTree { handle_queue_ns, \ scan_hits, \ threads, \ - std::cout) + std::cout, \ + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { const size_t thread_id = omp_get_thread_num(); @@ -649,8 +392,12 @@ class BlockTreeSharded : public BlockTree { const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair // starting at the current block @@ -665,14 +412,15 @@ class BlockTreeSharded : public BlockTree { shard.insert(hash, i); } } else { - const uint64_t HASH_MASK = HASH_MASKS[pair_size * sizeof(input_type)]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start, block_size); // Try to find the hash in the map, insert a new entry if it @@ -711,8 +459,12 @@ class BlockTreeSharded : public BlockTree { #endif if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -891,13 +643,14 @@ class BlockTreeSharded : public BlockTree { tlx::Aggregate& agg #endif ) { - const uint64_t HASH_MASK = HASH_MASKS[pair_size / sizeof(input_type)]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash current_hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, pair_size); // Find the hash of the current window among the hashed block @@ -930,7 +683,7 @@ class BlockTreeSharded : public BlockTree { /// @tparam use_hash Determines whether to use a rabin karp hash for hashing /// text windows or to use the block's content as a hash. For any window size /// greater than 8 bytes, use Rabin-Karp. - template + template void scan_blocks(const std::vector& text, LevelData& level_data, const bool is_padded, @@ -938,8 +691,9 @@ class BlockTreeSharded : public BlockTree { const size_t queue_size) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -978,10 +732,18 @@ class BlockTreeSharded : public BlockTree { finish_idle_ns, \ total_idle_ns, \ handle_queue_ns, \ - scan_hits) + scan_hits, \ + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { const size_t num_threads = omp_get_num_threads(); @@ -1000,22 +762,27 @@ class BlockTreeSharded : public BlockTree { (thread_id + 1) * segment_size); // Hash each block and store their hashes in the map - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { rk.restart(block_starts[i]); RabinKarpHash hash = rk.current_hash(); shard.insert(hash, {i, 0}); } } else { - const uint64_t HASH_MASK = HASH_MASKS[block_size / sizeof(input_type)]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start, block_size); @@ -1054,8 +821,12 @@ class BlockTreeSharded : public BlockTree { // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { continue; @@ -1206,13 +977,14 @@ class BlockTreeSharded : public BlockTree { #endif ) { const uint64_t HASH_MASK = - HASH_MASKS[level_data.block_size / sizeof(input_type)]; + internal::sharded::HASH_MASKS[level_data.block_size / + sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, level_data.block_size); // Find all blocks in the multimap that match our hash @@ -1441,7 +1213,7 @@ class BlockTreeSharded : public BlockTree { prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -1529,7 +1301,7 @@ class BlockTreeSharded : public BlockTree { const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -1567,7 +1339,7 @@ class BlockTreeSharded : public BlockTree { (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index be3705f..d87b2b3 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -25,10 +25,12 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include +#include #include #include #include @@ -49,17 +51,8 @@ namespace pasta { template class RecursiveBitBlockTreeSharded : public RecursiveBitBlockTree { - using Clock = std::chrono::high_resolution_clock; - using TimePoint = Clock::time_point; - /// @brief Determine whether to use a Rabin-Karp hash for hashing text windows - /// or just use the block's content itself as a hash, stored in an integer. - enum class UseHash { - /// @brief Use a Rabin-Karp hash - RABIN_KARP, - /// @brief Use the block's content as a hash - IDENTITY - }; + // ---------------------------------- Constants ---------------------------------- /// @brief For some block size (in bytes) i, return the number of trailing /// zeros in a 64 bit integer when zeroing out characters that are not part @@ -95,26 +88,36 @@ class RecursiveBitBlockTreeSharded /// @brief Masks for identity hashes for a block size i (in bytes) constexpr static std::array HASH_MASKS = masks(); - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - /// @brief Base of the polynomial used for the Rabin-Karp hasher constexpr static size_type SIGMA = 256; /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; + /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); + constexpr static uint128_t PRIME = pasta::mersenne_prime(); + + // ---------------------------------- End Constants ---------------------------------- + + // ---------------------------------- Type Defs ---------------------------------- + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; /// @brief A bit vector using BitVector = pasta::BitVector; /// @brief A rank data structure for a bit vector using Rank = pasta::RankSelect; + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = @@ -133,14 +136,12 @@ class RecursiveBitBlockTreeSharded using RabinKarpMap = SyncShardedMap; - static uint64_t mix_select(uint64_t key) { - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); - return key; - } + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // ---------------------------------- End Type Defs ---------------------------------- #ifdef BT_INSTRUMENT public: @@ -154,219 +155,6 @@ class RecursiveBitBlockTreeSharded size_t b_update_blocks_ns = 0; #endif -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - int64_t block_size; - /// @brief The index of the current level. - /// First level is 0, second level is 1 etc. - int64_t level_index; - /// @brief The number of blocks on the current level - int64_t num_blocks; - - LevelData(const int64_t level_index_, - const int64_t block_size_, - const int64_t num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - PairOccurrences(PairOccurrences&&) noexcept = default; - PairOccurrences& operator=(PairOccurrences&&) = default; - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - ~BlockOccurrences() = default; - - BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { - first_occ = other.first_occ.load(); - occurrences = std::move(other.occurrences); - return *this; - } - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_offset The offset of that occurrence - void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - /// @brief Constructs the block tree. /// @param text The input text. /// @param threads The number of threads to use for construction @@ -665,7 +453,7 @@ class RecursiveBitBlockTreeSharded const uint64_t hash_value = pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start, block_size); // Try to find the hash in the map, insert a new entry if it @@ -889,7 +677,7 @@ class RecursiveBitBlockTreeSharded const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash current_hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, pair_size); // Find the hash of the current window among the hashed block @@ -930,8 +718,9 @@ class RecursiveBitBlockTreeSharded const size_t queue_size) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -1007,7 +796,7 @@ class RecursiveBitBlockTreeSharded const uint64_t hash_value = pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start, block_size); @@ -1203,7 +992,7 @@ class RecursiveBitBlockTreeSharded const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, level_data.block_size); // Find all blocks in the multimap that match our hash @@ -1458,7 +1247,7 @@ class RecursiveBitBlockTreeSharded prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -1561,7 +1350,7 @@ class RecursiveBitBlockTreeSharded const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -1589,17 +1378,19 @@ class RecursiveBitBlockTreeSharded std::cout << "non-internal node missing pointer" << std::endl; std::cout << level_index << ", " << block_index << " / " << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); BT_ASSERT(child_pointer >= 0); #endif // Decrement the counter of where the child points (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index 6f0b40e..c55b1bd 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -21,17 +21,16 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp" #include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include #include -#include -#include -#include #include #include #include @@ -39,23 +38,8 @@ #include #include -__extension__ typedef unsigned __int128 uint128_t; - namespace pasta { -namespace sharded { - -/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows -/// or just use the block's content itself as a hash, stored in an integer. -enum class UseHash { - /// @brief Use a Rabin-Karp hash - RABIN_KARP, - /// @brief Use the block's content as a hash - IDENTITY -}; - -} // namespace sharded - /// @brief A parallel block tree construction algorithm using Rabin-Karp hashes /// and a sharded hash map. Small blocks are not RK-hashed but rather use the /// blocks themselves. @@ -67,63 +51,27 @@ template class RecursiveBlockTreeSharded : public RecursiveBlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; - /// @brief For some block size (in bytes) i, return the number of trailing - /// zeros in a 64 bit integer when zeroing out characters that are not part - /// of the block. - constexpr static uint64_t MASK_TRAILING_ZEROS[9] = - {64, 56, 48, 40, 32, 24, 16, 8, 0}; - - /// @brief Masks used for the identity hash. These depend on endianness - constexpr static std::array masks() { - if constexpr (std::endian::native == std::endian::big) { - return {0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; - } else { - return {0, - static_cast(~0) >> MASK_TRAILING_ZEROS[1], - static_cast(~0) >> MASK_TRAILING_ZEROS[2], - static_cast(~0) >> MASK_TRAILING_ZEROS[3], - static_cast(~0) >> MASK_TRAILING_ZEROS[4], - static_cast(~0) >> MASK_TRAILING_ZEROS[5], - static_cast(~0) >> MASK_TRAILING_ZEROS[6], - static_cast(~0) >> MASK_TRAILING_ZEROS[7], - static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; - } - } - - /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static std::array HASH_MASKS = masks(); - - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); - /// @brief A bit vector using BitVector = pasta::BitVector; /// @brief A rank data structure for a bit vector using Rank = pasta::RankSelect; + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = @@ -131,7 +79,9 @@ class RecursiveBlockTreeSharded /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; @@ -142,18 +92,14 @@ class RecursiveBlockTreeSharded using RabinKarpMap = SyncShardedMap; -#define MIX - static uint64_t mix_select(uint64_t key) { -#ifdef MIX - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); -#endif - return key; - } + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on #ifdef BT_INSTRUMENT public: size_t bp_hash_pairs_ns = 0; @@ -166,219 +112,6 @@ class RecursiveBlockTreeSharded size_t b_update_blocks_ns = 0; #endif -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - int64_t block_size; - /// @brief The index of the current level. - /// First level is 0, second level is 1 etc. - int64_t level_index; - /// @brief The number of blocks on the current level - int64_t num_blocks; - - LevelData(const int64_t level_index_, - const int64_t block_size_, - const int64_t num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - PairOccurrences(PairOccurrences&&) noexcept = default; - PairOccurrences& operator=(PairOccurrences&&) = default; - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - ~BlockOccurrences() = default; - - BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { - first_occ = other.first_occ.load(); - occurrences = std::move(other.occurrences); - return *this; - } - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_offset The offset of that occurrence - void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - /// @brief Constructs the block tree. /// @param text The input text. /// @param threads The number of threads to use for construction @@ -450,17 +183,17 @@ class RecursiveBlockTreeSharded LevelData& current = levels.back(); if (2 * static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( @@ -469,17 +202,17 @@ class RecursiveBlockTreeSharded now = Clock::now(); #endif if (static_cast(current.block_size * sizeof(input_type)) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); + scan_blocks(text, + current, + is_padded, + threads, + queue_size); } #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( @@ -590,7 +323,7 @@ class RecursiveBlockTreeSharded /// use the blocks' contents themselves as hashes. /// For block sizes greater than 4 bytes, use Rabin-Karp. /// - template + template void scan_block_pairs(const std::vector& text, LevelData& level, const bool is_padded, @@ -632,10 +365,18 @@ class RecursiveBlockTreeSharded handle_queue_ns, \ scan_hits, \ threads, \ - std::cout) + std::cout, \ + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { const size_t thread_id = omp_get_thread_num(); @@ -656,8 +397,12 @@ class RecursiveBlockTreeSharded const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair // starting at the current block @@ -672,14 +417,15 @@ class RecursiveBlockTreeSharded shard.insert(hash, i); } } else { - const uint64_t HASH_MASK = HASH_MASKS[pair_size * sizeof(input_type)]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const input_type* block_start_ptr = text.data() + block_start; const uint64_t hash_value = pasta::copy_le(block_start_ptr) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start, block_size); // Try to find the hash in the map, insert a new entry if it @@ -718,8 +464,12 @@ class RecursiveBlockTreeSharded #endif if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -898,13 +648,14 @@ class RecursiveBlockTreeSharded tlx::Aggregate& agg #endif ) { - const uint64_t HASH_MASK = HASH_MASKS[pair_size / sizeof(input_type)]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash current_hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, pair_size); // Find the hash of the current window among the hashed block @@ -937,7 +688,7 @@ class RecursiveBlockTreeSharded /// @tparam use_hash Determines whether to use a rabin karp hash for hashing /// text windows or to use the block's content as a hash. For any window size /// greater than 8 bytes, use Rabin-Karp. - template + template void scan_blocks(const std::vector& text, LevelData& level_data, const bool is_padded, @@ -945,8 +696,9 @@ class RecursiveBlockTreeSharded const size_t queue_size) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -985,181 +737,214 @@ class RecursiveBlockTreeSharded finish_idle_ns, \ total_idle_ns, \ handle_queue_ns, \ - scan_hits) + scan_hits, + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - // Hash each block and store their hashes in the map - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } else { - const uint64_t HASH_MASK = HASH_MASKS[block_size / sizeof(input_type)]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const input_type* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); - } - } - - if (const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = + *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = + level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = + ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = + std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / + sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = + text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & + HASH_MASK; + RabinKarpHash hash( + text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); #pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT - { - b_hash_blocks_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } + { + b_hash_blocks_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + } - tlx::Aggregate thread_scan_hits; + tlx::Aggregate thread_scan_hits; #else { } #endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == sharded::UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != + block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } - } - } + ); + } + } + } #ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); # pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; #endif - } + } #ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); + b_scan_blocks_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); # ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : links.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - - BT_ASSERT(links.num_inserts_.load() == links.size()); + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", + total_idle_ns, + 1'000'000); + print_aggregate("Block Handle Queue (ms)", + finish_idle_ns, + 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); # endif #endif - // By this point, the map should contain the first occurrences of - // every respective block's content. We then fill the pointers - // and offsets with this data and increment counters accordingly - links.for_each( - [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; - } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); + // By this point, the map should contain the first occurrences + // of every respective block's content. We then fill the + // pointers and offsets with this data and increment counters + // accordingly + links.for_each([&level_data](const RabinKarpHash&, + const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); #ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); + b_update_blocks_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif } @@ -1213,13 +998,14 @@ class RecursiveBlockTreeSharded #endif ) { const uint64_t HASH_MASK = - HASH_MASKS[level_data.block_size / sizeof(input_type)]; + internal::sharded::HASH_MASKS[level_data.block_size / + sizeof(input_type)]; const input_type* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = pasta::copy_le(block_start_ptr + offset) & HASH_MASK; RabinKarpHash hash(text, - mix_select(hash_value), + internal::sharded::mix_select(hash_value), block_start + offset, level_data.block_size); // Find all blocks in the multimap that match our hash @@ -1472,7 +1258,7 @@ class RecursiveBlockTreeSharded prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -1577,7 +1363,7 @@ class RecursiveBlockTreeSharded const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -1615,7 +1401,7 @@ class RecursiveBlockTreeSharded (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp new file mode 100644 index 0000000..1ba4b6b --- /dev/null +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -0,0 +1,297 @@ +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include +#include +#include +#include + +/// @brief Utilities for the construction algorithms using the sharded hash map +namespace pasta::internal::sharded { + +__extension__ typedef unsigned __int128 uint128_t; + +/// @brief A marker for a block that has no earlier occurrence +inline constexpr int64_t NO_EARLIER_OCC = -1; + +/// @brief A marker for a block that has been pruned +inline constexpr int64_t PRUNED = -2; + +/// @brief For some block size (in bytes) i, return the number of trailing +/// zeros in a 64 bit integer when zeroing out characters that are not part +/// of the block. +constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + +/// @brief Masks used for the identity hash. These depend on endianness +constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } +} + +/// @brief Masks for identity hashes for a block size i (in bytes) +inline constexpr std::array HASH_MASKS = masks(); + +/// @brief Base of the polynomial used for the Rabin-Karp hasher +inline constexpr size_t SIGMA = 256; + +/// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher +inline constexpr uint8_t PRIME_EXPONENT = 107; + +/// @brief A mersenne prime used for the Rabin-Karp hasher +inline constexpr uint128_t PRIME = pasta::mersenne_prime(); + +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + +/// @brief Contains data about a block tree level under construction +template +struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } +}; + +/// @brief Contains data about the occurrences of a hashed block pair +template +struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } +}; + +/// @brief Contains data about the occurrences of a hashed block +template +struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } +}; + +/// @brief An update function for the sharded hash map that updates the +/// occurrences of a hashed block pair +template +struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const MersenneHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const MersenneHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } +}; + +/// @brief An update function for the sharded hash map that updates the +/// occurrences of a hashed block +template +struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const MersenneHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences + init(const MersenneHash&, InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } +}; + +constexpr uint64_t mix_select(uint64_t key) { + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); + return key; +} + +} // namespace pasta::internal::sharded \ No newline at end of file From f697701d028cf41df280ed04889bce2a8393ae32 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Mon, 11 Dec 2023 20:01:51 +0100 Subject: [PATCH 69/92] rename mersenne prime function --- include/pasta/block_tree/utils/MersenneRabinKarp.hpp | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 1472fcc..deea686 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -31,7 +31,7 @@ namespace pasta { __extension__ typedef unsigned __int128 uint128_t; template -static consteval uint128_t primer() { +static consteval uint128_t mersenne_prime() { uint128_t res = 1; for (size_t i = 0; i < exponent; i++) { res <<= 1; @@ -122,7 +122,7 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - constexpr uint128_t MERSENNE = primer(); + constexpr uint128_t MERSENNE = mersenne_prime(); uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); i -= (i >= MERSENNE) * MERSENNE; return i; @@ -149,7 +149,7 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { fp += prime_ * (out_char_influence > hash_) - out_char_influence; } else { - fp += primer() * (out_char_influence > hash_) - + fp += mersenne_prime() * (out_char_influence > hash_) - out_char_influence; } fp *= sigma_; @@ -230,7 +230,7 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { return k % prime_; } else { - constexpr uint128_t MERSENNE = primer(); + constexpr uint128_t MERSENNE = mersenne_prime(); uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); i -= (i >= MERSENNE) * MERSENNE; return i; @@ -257,7 +257,7 @@ class MersenneRabinKarp { if constexpr (mersenne_exponent == 0) { fp += prime_ * (out_char_influence > hash_) - out_char_influence; } else { - fp += primer() * (out_char_influence > hash_) - + fp += mersenne_prime() * (out_char_influence > hash_) - out_char_influence; } fp *= 2; From 4c5f4f3df8e82107eb9e3f77e3773b2bd8beae0a Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 12 Dec 2023 00:53:24 +0100 Subject: [PATCH 70/92] WIP dense bit block tree with access and select --- examples/build_bt.cpp | 25 +- .../rec_bit_block_tree_sharded.hpp | 89 +- .../rec_dense_bit_block_tree_sharded.hpp | 1394 +++++++++++++++++ .../pasta/block_tree/dense_bit_block_tree.hpp | 791 ++++++++++ include/pasta/block_tree/rec_block_tree.hpp | 1 + .../block_tree/rec_dense_bit_block_tree.hpp | 785 ++++++++++ .../block_tree/utils/MersenneRabinKarp.hpp | 3 - .../pasta/block_tree/utils/sharded_util.hpp | 7 + 8 files changed, 3022 insertions(+), 73 deletions(-) create mode 100644 include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp create mode 100644 include/pasta/block_tree/dense_bit_block_tree.hpp create mode 100644 include/pasta/block_tree/rec_dense_bit_block_tree.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index c2d9ff6..678957f 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,7 +22,7 @@ #include #include #include -#include +#include #include #include @@ -206,7 +206,7 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; -using A = pasta::DenseBitBlockTreeSharded; +using A = pasta::RecursiveDenseBitBlockTreeSharded; int main(int argc, char** argv) { using namespace pasta; @@ -328,16 +328,17 @@ int main(int argc, char** argv) { if (make_bv) { // Make bit vector block tree - auto bt = std::make_unique< - RecursiveBitBlockTreeSharded>(*bv, - arity, - 1, - leaf_length, - threads, - queue_size); - - // auto bt = - // std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); + /* + auto bt = std::make_unique< + RecursiveBitBlockTreeSharded>(*bv, + arity, + 1, + leaf_length, + threads, + queue_size); + */ + auto bt = + std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index d87b2b3..b2df147 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -31,7 +31,6 @@ #include #include #include -#include #include #include #include @@ -51,55 +50,9 @@ namespace pasta { template class RecursiveBitBlockTreeSharded : public RecursiveBitBlockTree { - - // ---------------------------------- Constants ---------------------------------- - - /// @brief For some block size (in bytes) i, return the number of trailing - /// zeros in a 64 bit integer when zeroing out characters that are not part - /// of the block. - constexpr static uint64_t MASK_TRAILING_ZEROS[9] = - {64, 56, 48, 40, 32, 24, 16, 8, 0}; - - /// @brief Masks used for the identity hash. These depend on endianness - constexpr static std::array masks() { - if constexpr (std::endian::native == std::endian::big) { - return {0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; - } else { - return {0, - static_cast(~0) >> MASK_TRAILING_ZEROS[1], - static_cast(~0) >> MASK_TRAILING_ZEROS[2], - static_cast(~0) >> MASK_TRAILING_ZEROS[3], - static_cast(~0) >> MASK_TRAILING_ZEROS[4], - static_cast(~0) >> MASK_TRAILING_ZEROS[5], - static_cast(~0) >> MASK_TRAILING_ZEROS[6], - static_cast(~0) >> MASK_TRAILING_ZEROS[7], - static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; - } - } - - /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static std::array HASH_MASKS = masks(); - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; - - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::mersenne_prime(); - - // ---------------------------------- End Constants ---------------------------------- - + // clang-format off // ---------------------------------- Type Defs ---------------------------------- + // clang-format on using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; @@ -125,7 +78,8 @@ class RecursiveBitBlockTreeSharded /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = + MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; @@ -141,7 +95,9 @@ class RecursiveBitBlockTreeSharded /// @brief A map containing hashed blocks mapped to their occurrences using BlockMap = RabinKarpMap; + // clang-format off // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on #ifdef BT_INSTRUMENT public: @@ -431,7 +387,11 @@ class RecursiveBitBlockTreeSharded std::min(num_block_pairs, (thread_id + 1) * segment_size); if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair // starting at the current block @@ -446,7 +406,7 @@ class RecursiveBitBlockTreeSharded shard.insert(hash, i); } } else { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const uint8_t* block_start_ptr = text.data() + block_start; @@ -493,7 +453,11 @@ class RecursiveBitBlockTreeSharded if (start < static_cast(num_block_pairs)) { if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -671,7 +635,7 @@ class RecursiveBitBlockTreeSharded tlx::Aggregate& agg #endif ) { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; const uint8_t* block_start_ptr = text.data() + block_start; for (size_t offset = 0; offset < num_iterations; ++offset) { const uint64_t hash_value = @@ -782,14 +746,18 @@ class RecursiveBitBlockTreeSharded // Hash each block and store their hashes in the map if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { rk.restart(block_starts[i]); RabinKarpHash hash = rk.current_hash(); shard.insert(hash, {i, 0}); } } else { - const uint64_t HASH_MASK = HASH_MASKS[block_size]; + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[block_size]; for (size_t i = start; i < end; ++i) { const size_t block_start = block_starts[i]; const uint8_t* block_start_ptr = text.data() + block_start; @@ -836,7 +804,11 @@ class RecursiveBitBlockTreeSharded // block. if (start < block_starts.size() - is_padded) { if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { continue; @@ -986,7 +958,8 @@ class RecursiveBitBlockTreeSharded tlx::Aggregate& hits #endif ) { - const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size]; const uint8_t* block_start_ptr = text.data() + block_start; for (size_type offset = 0; offset < level_data.block_size; ++offset) { const uint64_t hash_value = diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp new file mode 100644 index 0000000..8c36536 --- /dev/null +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -0,0 +1,1394 @@ + +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_dense_bit_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveDenseBitBlockTreeSharded + : public RecursiveDenseBitBlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = + MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const pasta::BitVector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size) > 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const pasta::BitVector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } /*else { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + }*/ + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + /* + static inline void + scan_windows_in_block_pair_identity(const pasta::BitVector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index + #ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg + #endif + ) { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + #ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); + #else + continue; + #endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + */ + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const pasta::BitVector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = + internal::sharded::ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } /*else { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[block_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + }*/ + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + /* + static void + scan_windows_in_block_identity(const std::span& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index + #ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits + #endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + #ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); + #else + continue; + #endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + */ + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData generate_next_level(const pasta::BitVector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const pasta::BitVector& text, + std::vector& levels, + const int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels.size() <= 1 || + levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = new RecursiveDenseBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + size_t bit_index = 0; + const size_t final_num_internals = + levels.back().is_internal_rank->rank1(last_is_internal.size()); + this->leaf_bits_ = std::make_unique( + final_num_internals * this->leaf_size * this->tau_, + false); + std::cout << "leaf bit size: " << this->leaf_bits_->size() << std::endl; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + (*this->leaf_bits_)[bit_index++] = + static_cast(text[block_start + b]); + } else { + (*this->leaf_bits_)[bit_index++] = false; + } + } + } + std::cout << "leaf count: " << leaf_count << std::endl; + if constexpr (recursion_level == 0) { + if (levels.size() == 1) { + top_level.is_internal.release(); + } + } + this->amount_of_leaves = leaf_count; + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveDenseBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + auto& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + RecursiveDenseBitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->num_bits_ = text.size(); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/dense_bit_block_tree.hpp b/include/pasta/block_tree/dense_bit_block_tree.hpp new file mode 100644 index 0000000..4222e1f --- /dev/null +++ b/include/pasta/block_tree/dense_bit_block_tree.hpp @@ -0,0 +1,791 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class DenseBitBlockTree { +public: + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits; + bool rank_support = false; + /// Bit vectors for each level determining whether a block is internal + /// (=1) or not (=0) + std::vector block_tree_types_; + std::vector*> + block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + bool access(const size_type bit_index) const { + // FIXME: As of now this works on little endian systems only + const int64_t byte_index = bit_index / 8; + const int64_t bit_offset = bit_index % 8; + + int64_t block_size = block_size_lvl_[0]; + int64_t block_index = byte_index / block_size; + int64_t off = byte_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + } + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; + return ((1 << bit_offset) & byte) != 0; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + pos += 8; + ++byte_offset; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + return pos; + } + + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_zeros = 8 - std::popcount(byte); + if (rank > num_zeros) { + rank -= num_zeros; + pos += 8; + byte_offset++; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) == 0; + } + } + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const size_t byte_index = bit_index / 8; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = byte_index / block_size; + size_t block_offset = byte_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + + block]]; + rank += std::popcount(byte); + } + for (size_t block = 0; block < block_offset; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; + rank += std::popcount(byte); + } + + // Masks to remove bits from the last byte, + // that aren't part of the ran query + static constexpr std::array MASKS = { + 0b0000'0000, + 0b0000'0001, + 0b0000'0011, + 0b0000'0111, + 0b0000'1111, + 0b0001'1111, + 0b0011'1111, + 0b0111'1111, + }; + rank += std::popcount( + decompress_map_[compressed_leaves_[block_index * leaf_size + + block_offset]] & + MASKS[bit_index % 8]); + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + auto delta_size = 0; + for (const auto bv : block_tree_types_) { + space_usage += bv->size() / 8; + delta_size += bv->size() / 8; + } + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; + for (const auto rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + delta_size += rs->space_usage(); + } + std::cout << "rs size: " << delta_size << std::endl; + delta_size = 0; + for (const auto iv : block_tree_pointers_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); + } + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); + } + std::cout << "offs size: " << delta_size << std::endl; + if (rank_support) { + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + } + + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + + // space_usage += leaves_.size() * sizeof(uint8_t); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); + + return space_usage; + }; + + int32_t add_bit_rank_support() { + rank_support = true; + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + return 0; + } + +protected: + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] bv_length The number of bits in the input bit vector. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t bv_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Bit size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < bv_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - bv_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + const uint8_t byte = + decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; + result += std::popcount(byte); + } + return result; + } +}; + + + +} // namespace pasta + +/******************************************************************************/ diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index 9703d5b..a22c904 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -27,6 +27,7 @@ #include #include #include +#include #include #include diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp new file mode 100644 index 0000000..322722c --- /dev/null +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -0,0 +1,785 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class RecursiveDenseBitBlockTree { +public: + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = std::conditional_t< + types_is_block_tree, + RecursiveDenseBitBlockTree, + pasta::BitVector>; + using IsInternalRankType = std::conditional_t< + types_is_block_tree, + RecursiveDenseBitBlockTree, + pasta::RankSelect>; + + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits_; + bool rank_support = false; + /// Recursively compress the bit vectors of the tree + std::vector block_tree_types_; + std::vector block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + std::unique_ptr leaf_bits_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + ~RecursiveDenseBitBlockTree() { + for (const IsInternalType* b : this->block_tree_types_) { + delete b; + } + // in any other case, block_tree_types_ and block_tree_types_rs_ point to + // the same object (a recursive block tree), so we may only free them once + if constexpr (recursion_level == 0) { + for (const RankSelect* rs : + this->block_tree_types_rs_) { + delete rs; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + [[nodiscard]] size_t size() const { + return num_bits_; + } + + bool operator[](const size_type bit_index) const { + return access(bit_index); + } + + bool access(const size_type bit_index) const { + int64_t block_size = block_size_lvl_[0]; + int64_t block_index = bit_index / block_size; + int64_t off = bit_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + } + + return (*leaf_bits_)[block_index * leaf_size + off]; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + /// FIXME DOES NOT WORK YET + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + pos += 8; + ++byte_offset; + } else { + for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + return pos; + } + + /// FIXME DOES NOT WORK YET + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + if (omp_get_thread_num() == 1) { + std::osyncstream(std::cout) << *leaf_bits_ << std::endl; + } + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + for (uint8_t bit = 0; rank > 0; ++bit, ++pos) { + rank -= (*leaf_bits_)[current_block * leaf_size + bit]; + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = bit_index / block_size; + size_t block_offset = bit_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + rank += (*leaf_bits_)[prefix_leaves * leaf_size + block]; + } + for (size_t block = 0; block < block_offset; block++) { + rank += (*leaf_bits_)[block_index * leaf_size + block]; + } + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + + auto delta_size = 0; + for (const auto* bt : block_tree_types_) { + if constexpr (types_is_block_tree) { + space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); + } else { + space_usage += bt->size() / 8; + delta_size += bt->size() / 8; + } + } + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; + if constexpr (recursion_level == 0) { + for (const auto* rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + delta_size += rs->space_usage(); + } + std::cout << "rs size: " << delta_size << std::endl; + } + delta_size = 0; + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); + ; + } + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + } + std::cout << "offs size: " << delta_size << std::endl; + space_usage += block_size_lvl_.size() * + sizeof(typename decltype(block_size_lvl_)::value_type); + space_usage += block_per_lvl_.size() * + sizeof(typename decltype(block_per_lvl_)::value_type); + + if (rank_support) { + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + } + + space_usage += leaf_bits_->size() / 8; + + return space_usage; + }; + + void + add_bit_rank_support(size_t threads = std::thread::hardware_concurrency()) { + if (rank_support) { + return; + } + rank_support = true; + + // FIXME For the last level where block_tree_types_ is a bitvec, using + // multiple threads doesn't work for some reason + if constexpr (recursion_level == 0) { + threads = 1; + } + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + // FIXME: breaks if parallelism is used + // #pragma omp parallel for default(none) num_threads(threads) + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + +#pragma omp parallel for default(none) num_threads(threads) + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + } + +protected: + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= leaf_bits_->size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + result += (*leaf_bits_)[leaf_index * leaf_size + i]; + } + return result; + } +}; + +template +using DenseBitBlockTree = RecursiveDenseBitBlockTree; + +} // namespace pasta + +/******************************************************************************/ diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index deea686..8b48583 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -23,9 +23,6 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" -#include -#include - namespace pasta { __extension__ typedef unsigned __int128 uint128_t; diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp index 1ba4b6b..287df57 100644 --- a/include/pasta/block_tree/utils/sharded_util.hpp +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -294,4 +294,11 @@ constexpr uint64_t mix_select(uint64_t key) { return key; } +/// @brief Returns the ceiling of x / y for x > 0; +/// +/// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c +inline size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + (x - 1) / y; +} + } // namespace pasta::internal::sharded \ No newline at end of file From f66bf88899f1fc5ee4761e9e1d196a0b299a847d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 12 Dec 2023 16:09:52 +0100 Subject: [PATCH 71/92] recursive dense bit block trees --- examples/build_bt.cpp | 68 +++++++++-------- .../rec_bit_block_tree_sharded.hpp | 24 +++++- .../construction/rec_block_tree_sharded.hpp | 3 +- .../rec_dense_bit_block_tree_sharded.hpp | 4 +- .../pasta/block_tree/rec_bit_block_tree.hpp | 9 +++ include/pasta/block_tree/rec_block_tree.hpp | 33 +++++++++ .../block_tree/rec_dense_bit_block_tree.hpp | 74 +++++++++---------- 7 files changed, 135 insertions(+), 80 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 678957f..d122bfb 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,13 +22,18 @@ #include #include #include -#include #include +#include #include +#define REC_PAR_SHARDED + +#if defined REC_BIT || defined REC_PAR_SHARDED constexpr size_t RECURSION_LEVELS = 1; +#else +constexpr size_t RECURSION_LEVELS = 0; +#endif -#define PAR_SHARDED_SYNC_SMALL #ifdef FP # include std::unique_ptr> @@ -110,20 +115,20 @@ make_bt(std::vector& text, } # define ALGO_NAME "shard_sync" #elif defined PAR_SHARDED_SYNC_SMALL -# include -std::unique_ptr> +# include +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { - return std::make_unique>( - text, - arity, - 1, - leaf_length, - threads, - queue_size); + return std::make_unique< + pasta::RecursiveBlockTreeSharded>(text, + arity, + 1, + leaf_length, + threads, + queue_size); } # define ALGO_NAME "shard_sync_small" #elif defined REC_PAR_SHARDED @@ -164,13 +169,14 @@ make_bt(std::vector& text, # define ALGO_NAME "par_map" #elif defined REC_BIT # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { - return std::make_unique>( + return std::make_unique< + pasta::RecursiveBitBlockTreeSharded>( text, arity, 1, @@ -206,7 +212,7 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; -using A = pasta::RecursiveDenseBitBlockTreeSharded; +using BBT = pasta::RecursiveDenseBitBlockTreeSharded; int main(int argc, char** argv) { using namespace pasta; @@ -273,8 +279,7 @@ int main(int argc, char** argv) { #ifdef BT_DBG std::cout << "building block tree with parameters:" << "\narity: " << arity << "\nmax leaf length: " << leaf_length - << "\nsaving to " << out_path << "\nusing " << threads << " threads" - << std::endl; + << "\nusing " << threads << " threads" << std::endl; #endif if (!std::filesystem::exists(file)) { @@ -319,9 +324,9 @@ int main(int argc, char** argv) { << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; if (make_bv) { - std::cout << " bv_size=" << bv->size(); + std::cout << " input_size=" << bv->size() / 8; } else { - std::cout << " file_size=" << text.size(); + std::cout << " input_size=" << text.size(); } TimePoint now = Clock::now(); @@ -338,7 +343,7 @@ int main(int argc, char** argv) { queue_size); */ auto bt = - std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); + std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); @@ -386,19 +391,6 @@ int main(int argc, char** argv) { } } const size_t num_zeros = frs.rank0(bv->size()); - -#pragma omp parallel for - for (size_t i = 1; i <= num_zeros; i++) { - const size_t bv_rank = frs.select0(i); - const size_t bt_rank = bt->select0(i); - if (bv_rank != bt_rank) { - std::osyncstream(std::cerr) << "Select zero error at position " << i - << "\nExpected: " << bv_rank - << "\nActual: " << bt_rank << std::endl; - throw std::runtime_error("oof"); - } - } - const size_t num_ones = frs.rank1(bv->size()); #pragma omp parallel for @@ -413,6 +405,18 @@ int main(int argc, char** argv) { } } +#pragma omp parallel for + for (size_t i = 1; i <= num_zeros; i++) { + const size_t bv_rank = frs.select0(i); + const size_t bt_rank = bt->select0(i); + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "Select zero error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + } else { // Make text block tree auto bt = make_bt(text, arity, leaf_length, threads, queue_size); diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index b2df147..3535e71 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -362,10 +362,18 @@ class RecursiveBitBlockTreeSharded handle_queue_ns, \ scan_hits, \ threads, \ - std::cout) + std::cout, \ + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { const size_t thread_id = omp_get_thread_num(); @@ -723,10 +731,18 @@ class RecursiveBitBlockTreeSharded finish_idle_ns, \ total_idle_ns, \ handle_queue_ns, \ - scan_hits) + scan_hits, \ + internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) #endif { const size_t num_threads = omp_get_num_threads(); diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index c55b1bd..f6d6091 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -1391,7 +1391,8 @@ class RecursiveBlockTreeSharded std::cout << "non-internal node missing pointer" << std::endl; std::cout << level_index << ", " << block_index << " / " << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index 8c36536..fabe619 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -1139,13 +1139,12 @@ class RecursiveDenseBitBlockTreeSharded this->leaf_bits_ = std::make_unique( final_num_internals * this->leaf_size * this->tau_, false); - std::cout << "leaf bit size: " << this->leaf_bits_->size() << std::endl; for (size_t block = 0; block < last_is_internal.size(); block++) { if (!last_is_internal[block]) { continue; } const size_type block_start = last_block_starts[block]; - // For every leaf on the last level, we have tau leaf blocks + // For every leaf on the las#elift level, we have tau leaf blocks leaf_count += this->tau_; // Iterate through all characters in this child and // add them to the leaf string @@ -1159,7 +1158,6 @@ class RecursiveDenseBitBlockTreeSharded } } } - std::cout << "leaf count: " << leaf_count << std::endl; if constexpr (recursion_level == 0) { if (levels.size() == 1) { top_level.is_internal.release(); diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index c18d3c5..621c70a 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -524,14 +524,18 @@ class RecursiveBitBlockTree { delta_size += bt->size() / 8; } } +#ifdef BT_DBG std::cout << "bv size: " << delta_size << std::endl; delta_size = 0; +#endif if constexpr (recursion_level == 0) { for (const auto* rs : block_tree_types_rs_) { space_usage += rs->space_usage(); delta_size += rs->space_usage(); } +#ifdef BT_DBG std::cout << "rs size: " << delta_size << std::endl; +#endif } delta_size = 0; for (const auto iv : block_tree_pointers_) { @@ -539,12 +543,17 @@ class RecursiveBitBlockTree { delta_size += (int64_t)sdsl::size_in_bytes(*iv); ; } +#ifdef BT_DBG std::cout << "ptrs size: " << delta_size << std::endl; delta_size = 0; +#endif for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); } +#ifdef BT_DBG std::cout << "offs size: " << delta_size << std::endl; +#endif space_usage += block_size_lvl_.size() * sizeof(typename decltype(block_size_lvl_)::value_type); space_usage += block_per_lvl_.size() * diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index a22c904..b00eb90 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -390,25 +390,53 @@ class RecursiveBlockTree { int64_t print_space_usage() { int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + sizeof(leaf_size); + auto delta_size = 0; if constexpr (recursion_level > 0) { for (auto bt : block_tree_types_) { space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif } if constexpr (recursion_level == 0) { for (auto bv : block_tree_types_) { space_usage += bv->size() / 8; + delta_size += bv->size() / 8; } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif for (auto rs : block_tree_types_rs_) { space_usage += rs->space_usage(); + delta_size += rs->space_usage(); } +#ifdef BT_DBG + std::cout << "rs size: " << delta_size << std::endl; + delta_size = 0; +#endif } for (const auto iv : block_tree_pointers_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); } +#ifdef BT_DBG + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; +#endif + for (const auto iv : block_tree_offsets_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); } +#ifdef BT_DBG + std::cout << "offs size: " << delta_size << std::endl; + delta_size = 0; +#endif + if (rank_support) { for (auto c : chars_) { int64_t sum = 0; @@ -431,6 +459,11 @@ class RecursiveBlockTree { // space_usage += leaves_.size() * sizeof(input_type); space_usage += sdsl::size_in_bytes(compressed_leaves_); space_usage += compress_map_.size(); +#ifdef BT_DBG + std::cout << "leaves size: " << sdsl::size_in_bytes(compressed_leaves_) + << std::endl; + delta_size = 0; +#endif return space_usage; }; diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp index 322722c..bac918a 100644 --- a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -27,7 +27,6 @@ #include #include #include -#include #include #include @@ -152,7 +151,7 @@ class RecursiveDenseBitBlockTree { [[nodiscard]] size_t find_initial_block(const size_t rank) const { const auto& top_one_ranks = one_ranks_[0]; const size_t block_size = block_size_lvl_[0]; - size_t start = (rank - 1) / (block_size * 8); + size_t start = (rank - 1) / block_size; size_t end = top_one_ranks.size() - 1; while (start != end) { const size_t middle = start + (end - start) / 2; @@ -160,7 +159,7 @@ class RecursiveDenseBitBlockTree { if constexpr (one) { current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; } else { - const size_t middle_bits = middle * block_size * 8; + const size_t middle_bits = middle * block_size; current_rank = (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; } @@ -170,7 +169,7 @@ class RecursiveDenseBitBlockTree { if constexpr (one) { bits = top_one_ranks[middle]; } else { - bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + bits = (middle + 1) * block_size - top_one_ranks[middle]; } // If there is only one block left, it's either the current or the // next block @@ -200,7 +199,7 @@ class RecursiveDenseBitBlockTree { // Binary Search for the correct top level block containing the correct 1 size_t current_block = find_initial_block(rank); - size_t pos = (current_block * block_size * 8) - 1; + size_t pos = current_block * block_size - 1; // ReSharper disable once CppDFAUnreachableCode rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; @@ -216,11 +215,11 @@ class RecursiveDenseBitBlockTree { rank_d -= pointer_prefix_one_counts_[0][back_block_index]; if (rank > rank_d) { rank -= rank_d; - pos += (block_size - offset) * 8; + pos += block_size - offset; ++current_block; } else { rank += pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; + pos -= offset; } } @@ -241,7 +240,7 @@ class RecursiveDenseBitBlockTree { ++current_block; } rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; - pos += (current_block - start_block) * block_size * 8; + pos += (current_block - start_block) * block_size; if (!is_internal[current_block]) { size_t back_block_index = is_internal_rank.rank0(current_block); current_block = pointers[back_block_index]; @@ -253,11 +252,11 @@ class RecursiveDenseBitBlockTree { rank_d -= pointer_ranks[back_block_index]; if (rank > rank_d) { rank -= rank_d; - pos += (block_size - offset) * 8; + pos += (block_size - offset); ++current_block; } else { rank += pointer_ranks[back_block_index]; - pos -= offset * 8; + pos -= offset; } } ++level; @@ -265,22 +264,8 @@ class RecursiveDenseBitBlockTree { current_block = block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_ones = std::popcount(byte); - if (rank > num_ones) { - rank -= num_ones; - pos += 8; - ++byte_offset; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) > 0; - } - } + for (uint8_t bit = 0; rank > 0; ++bit, ++pos) { + rank -= (*leaf_bits_)[current_block * leaf_size + bit]; } return pos; } @@ -296,12 +281,12 @@ class RecursiveDenseBitBlockTree { const size_t top_block_size = block_size_lvl_[0]; const auto top_zero_ranks = [&top_one_ranks, top_block_size](const size_t i) -> size_t { - return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + return (i + 1) * top_block_size - top_one_ranks[i]; }; // Binary Search for the correct top level block containing the correct 1 size_t current_block = find_initial_block(rank); - const size_t top_block_bits = top_block_size * 8; + const size_t top_block_bits = top_block_size; size_t pos = (current_block * top_block_bits) - 1; // ReSharper disable once CppDFAUnreachableCode @@ -313,7 +298,7 @@ class RecursiveDenseBitBlockTree { // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; current_block = top_pointers[back_block_index]; const size_t offset = top_offsets[back_block_index]; - const size_t prefix_bits = offset * 8; + const size_t prefix_bits = offset; size_t rank_d = (current_block == 0) ? top_zero_ranks(current_block) : @@ -321,11 +306,11 @@ class RecursiveDenseBitBlockTree { rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; if (rank > rank_d) { rank -= rank_d; - pos += (top_block_size - offset) * 8; + pos += top_block_size - offset; ++current_block; } else { rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; + pos -= offset; } } @@ -345,7 +330,7 @@ class RecursiveDenseBitBlockTree { const auto zero_ranks = [&one_ranks, this, block_size](const size_t i) -> size_t { - const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + const size_t rnk = (i % this->tau_ + 1) * block_size - one_ranks[i]; return rnk; }; const size_t start_block = current_block; @@ -354,12 +339,12 @@ class RecursiveDenseBitBlockTree { } rank -= (current_block == start_block) ? 0 : zero_ranks(current_block - 1); - pos += (current_block - start_block) * block_size * 8; + pos += (current_block - start_block) * block_size; if (!is_internal[current_block]) { size_t back_block_index = is_internal_rank.rank0(current_block); current_block = pointers[back_block_index]; const size_t offset = offsets[back_block_index]; - const size_t prefix_bits = offset * 8; + const size_t prefix_bits = offset; size_t rank_d = (current_block % tau_ == 0) ? zero_ranks(current_block) : @@ -367,23 +352,20 @@ class RecursiveDenseBitBlockTree { rank_d -= prefix_bits - pointer_ranks[back_block_index]; if (rank > rank_d) { rank -= rank_d; - pos += (block_size - offset) * 8; + pos += block_size - offset; ++current_block; } else { rank += prefix_bits - pointer_ranks[back_block_index]; - pos -= offset * 8; + pos -= offset; } } ++level; } - if (omp_get_thread_num() == 1) { - std::osyncstream(std::cout) << *leaf_bits_ << std::endl; - } current_block = block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; for (uint8_t bit = 0; rank > 0; ++bit, ++pos) { - rank -= (*leaf_bits_)[current_block * leaf_size + bit]; + rank -= !(*leaf_bits_)[current_block * leaf_size + bit]; } return pos; } @@ -491,14 +473,18 @@ class RecursiveDenseBitBlockTree { delta_size += bt->size() / 8; } } +#ifdef BT_DBG std::cout << "bv size: " << delta_size << std::endl; delta_size = 0; +#endif if constexpr (recursion_level == 0) { for (const auto* rs : block_tree_types_rs_) { space_usage += rs->space_usage(); delta_size += rs->space_usage(); } +#ifdef BT_DBG std::cout << "rs size: " << delta_size << std::endl; +#endif } delta_size = 0; for (const auto iv : block_tree_pointers_) { @@ -506,12 +492,17 @@ class RecursiveDenseBitBlockTree { delta_size += (int64_t)sdsl::size_in_bytes(*iv); ; } +#ifdef BT_DBG std::cout << "ptrs size: " << delta_size << std::endl; delta_size = 0; +#endif for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); } +#ifdef BT_DBG std::cout << "offs size: " << delta_size << std::endl; +#endif space_usage += block_size_lvl_.size() * sizeof(typename decltype(block_size_lvl_)::value_type); space_usage += block_per_lvl_.size() * @@ -527,6 +518,9 @@ class RecursiveDenseBitBlockTree { } space_usage += leaf_bits_->size() / 8; +#ifdef BT_DBG + std::cout << "leaf string: " << leaf_bits_->size() / 8 << std::endl; +#endif return space_usage; }; From f1ac9e4067601dae59287a9516654a53efadbe8b Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 14 Dec 2023 18:49:58 +0100 Subject: [PATCH 72/92] fix seq fp2 and phmap --- examples/build_bt.cpp | 43 +- .../construction/block_tree_fp2_seq.hpp | 145 +--- .../construction/block_tree_fp_par_phmap.hpp | 270 ++---- .../rec_bit_block_tree_sharded.hpp | 17 +- .../pasta/block_tree/dense_bit_block_tree.hpp | 791 ------------------ .../pasta/block_tree/utils/sharded_util.hpp | 4 +- 6 files changed, 142 insertions(+), 1128 deletions(-) delete mode 100644 include/pasta/block_tree/dense_bit_block_tree.hpp diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index d122bfb..33b18fa 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -22,18 +22,27 @@ #include #include #include -#include -#include #include -#define REC_PAR_SHARDED +#define PAR_PHMAP +#define REC_BIT -#if defined REC_BIT || defined REC_PAR_SHARDED -constexpr size_t RECURSION_LEVELS = 1; +#if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED +constexpr size_t RECURSION_LEVELS = 0; #else constexpr size_t RECURSION_LEVELS = 0; #endif +#if defined REC_DENSE_BIT +# include +using BBT = pasta::RecursiveDenseBitBlockTreeSharded; +# define BIT_ALGO_NAME "rec_dense_bit" +#elif defined REC_BIT +# include +using BBT = pasta::RecursiveBitBlockTreeSharded; +# define BIT_ALGO_NAME "rec_bit" +#endif + #ifdef FP # include std::unique_ptr> @@ -167,24 +176,6 @@ make_bt(std::vector& text, threads); } # define ALGO_NAME "par_map" -#elif defined REC_BIT -# include -std::unique_ptr> -make_bt(std::vector& text, - const size_t arity, - const size_t leaf_length, - const size_t threads, - const size_t queue_size) { - return std::make_unique< - pasta::RecursiveBitBlockTreeSharded>( - text, - arity, - 1, - leaf_length, - threads, - queue_size); -} -# define ALGO_NAME "rec_bit_shard" #elif defined PAR_PARLAY # include std::unique_ptr> @@ -212,8 +203,6 @@ using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; using Duration = Clock::duration; -using BBT = pasta::RecursiveDenseBitBlockTreeSharded; - int main(int argc, char** argv) { using namespace pasta; @@ -319,7 +308,7 @@ int main(int argc, char** argv) { } } - std::cout << "RESULT algo=" << ALGO_NAME + std::cout << "RESULT" << " file=" << std::filesystem::path(file).filename().string() << " threads=" << threads << " arity=" << arity << " leaf_length=" << leaf_length; @@ -331,6 +320,7 @@ int main(int argc, char** argv) { TimePoint now = Clock::now(); if (make_bv) { + std::cout << " algo=" << BIT_ALGO_NAME; // Make bit vector block tree /* @@ -418,6 +408,7 @@ int main(int argc, char** argv) { } } else { + std::cout << " algo=" << ALGO_NAME; // Make text block tree auto bt = make_bt(text, arity, leaf_length, threads, queue_size); auto elapsed = std::chrono::duration_cast( diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 4cedb82..6d41d22 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -21,15 +21,16 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include #include #include #include #include -#include __extension__ typedef unsigned __int128 uint128_t; @@ -37,78 +38,23 @@ namespace pasta { template class BlockTreeFP2 : public BlockTree { - constexpr static size_type NO_EARLIER_OCC = -1; - constexpr static size_type PRUNED = -2; - - static constexpr size_type SIGMA = 256; - static constexpr uint128_t K_PRIME = 2305843009213693951ULL; - static constexpr uint8_t MERSENNE_EXPONENT = 61; - using BitVector = pasta::BitVector; - // using Rank = pasta::FlatRank; using Rank = pasta::RankSelect; template > - using HashMap = robin_hood::unordered_map; + using HashMap = ankerl::unordered_dense::map; - using RabinKarp = MersenneRabinKarp; - // using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; using RabinKarpHash = MersenneHash; - // using RabinKarpHash = MersenneHash; template using RabinKarpMap = HashMap; - struct LevelData { - /// Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// The block from which a back block is copying - std::unique_ptr> pointers; - /// The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// The number of back blocks pointing to the block - std::unique_ptr> counters; - /// Block start indices - std::unique_ptr> block_starts; - /// The block size on this level - size_type block_size; - /// The index of the current level. First level is 0, second level is 1 etc. - size_type level_index; - /// The number of blocks on the current level - size_type num_blocks; - - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; - } - - /// @brief Checks whether a block is adjacent in the text - /// to its predecessor on this level - [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { - return (*block_starts)[i - 1] + static_cast(block_size) == - (*block_starts)[i]; - } - }; + using LevelData = internal::sharded::LevelData; void construct(const std::vector& text) { const size_type text_len = text.size(); @@ -129,7 +75,8 @@ class BlockTreeFP2 : public BlockTree { // Prepare the top level levels.emplace_back(0, top_block_size, text_len / top_block_size); LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); } @@ -143,7 +90,8 @@ class BlockTreeFP2 : public BlockTree { scan_blocks(text, current, is_padded); // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); } } @@ -152,14 +100,8 @@ class BlockTreeFP2 : public BlockTree { make_tree(text, levels, padding); } - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - /// @briefScan through the blocks pairwise in order to identify which blocks should + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should /// be replaced with back blocks. /// /// @param text The input string. @@ -190,7 +132,11 @@ class BlockTreeFP2 : public BlockTree { sdsl::int_vector<2> markings(num_blocks, 0); { - RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + 0, + pair_size, + internal::sharded::PRIME); for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { // If the next block is not adjacent, we cannot hash the pair starting // at the current block @@ -206,7 +152,11 @@ class BlockTreeFP2 : public BlockTree { // Hash every window and determine for all block pairs whether they have // previous occurrences. - RabinKarp rk(text, SIGMA, 0, pair_size, K_PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + 0, + pair_size, + internal::sharded::PRIME); for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -283,8 +233,9 @@ class BlockTreeFP2 : public BlockTree { const size_t num_blocks = level_data.num_blocks; const std::vector& block_starts = *level_data.block_starts; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -300,14 +251,21 @@ class BlockTreeFP2 : public BlockTree { // hash has already been processed RabinKarpMap> links(num_blocks); for (size_t i = 0; i < num_blocks - is_padded; ++i) { - const RabinKarpHash hash = - RabinKarp(s, SIGMA, block_starts[i], block_size, K_PRIME) - .current_hash(); + const RabinKarpHash hash = RabinKarp(s, + internal::sharded::SIGMA, + block_starts[i], + block_size, + internal::sharded::PRIME) + .current_hash(); links[hash].push_back(i); } // Hash every window and find the first occurrences for every block. - RabinKarp rk(s, SIGMA, block_starts[0], block_size, K_PRIME); + RabinKarp rk(s, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); for (size_t current_block_index = 0; current_block_index < num_blocks - is_padded - 1; ++current_block_index) { @@ -518,6 +476,8 @@ class BlockTreeFP2 : public BlockTree { b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } @@ -546,7 +506,9 @@ class BlockTreeFP2 : public BlockTree { const size_type last_block_parent_start = previous_level.block_starts->back(); const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; @@ -574,7 +536,7 @@ class BlockTreeFP2 : public BlockTree { prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -660,7 +622,7 @@ class BlockTreeFP2 : public BlockTree { const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing to this, // then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -688,7 +650,7 @@ class BlockTreeFP2 : public BlockTree { (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; @@ -706,21 +668,6 @@ class BlockTreeFP2 : public BlockTree { construct(text); } - ~BlockTreeFP2() { - for (auto& rank : this->block_tree_types_rs_) { - delete rank; - } - for (auto& bv : this->block_tree_types_) { - delete bv; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; - } - } - /// @brief Validates that a back-pointer actually points to the same text /// content. /// @param text The input text. diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp index 8067333..41ddf94 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -20,19 +20,15 @@ #pragma once -#include "data-structures/hash_table_mods.hpp" #include "pasta/bit_vector/bit_vector.hpp" #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" -#include -#include -#include +#include #include #include -#include -#include #include #include @@ -46,13 +42,6 @@ namespace pasta { template class BlockTreeFPParPH : public BlockTree { - constexpr static size_type NO_EARLIER_OCC = -1; - constexpr static size_type PRUNED = -2; - - static constexpr size_type SIGMA = 256; - static constexpr uint128_t K_PRIME = 2305843009213693951ULL; - static constexpr uint8_t MERSENNE_EXPONENT = 61; - using BitVector = pasta::BitVector; using Rank = pasta::RankSelect; @@ -75,21 +64,24 @@ class BlockTreeFPParPH : public BlockTree { template > - using HashMap = - robin_hood::unordered_node_map; + using HashMap = ankerl::unordered_dense::map; /// A rabin karp hasher preconfigured for the current template parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; /// A hash map with rabin karp hashes as keys - template + template using RabinKarpMap = HashMap>; + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + #ifdef BT_DBG public: size_t bp_hash_pairs_ns = 0; @@ -103,55 +95,6 @@ class BlockTreeFPParPH : public BlockTree { private: #endif - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// The block from which a back block is copying - std::unique_ptr> pointers; - /// The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// The number of back blocks pointing to the block - std::unique_ptr> counters; - /// Block start indices - std::unique_ptr> block_starts; - /// The block size on this level - size_type block_size; - /// The index of the current level. First level is 0, second level is 1 etc. - size_type level_index; - /// The number of blocks on the current level - size_type num_blocks; - - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; - } - - /// @brief Checks whether a block is adjacent in the text - /// to its predecessor on this level - [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { - return (*block_starts)[i - 1] + static_cast(block_size) == - (*block_starts)[i]; - } - }; void construct(const std::vector& text, const size_t threads) { const size_type text_len = text.size(); @@ -172,7 +115,8 @@ class BlockTreeFPParPH : public BlockTree { // Prepare the top level levels.emplace_back(0, top_block_size, text_len / top_block_size); LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); } @@ -187,14 +131,15 @@ class BlockTreeFPParPH : public BlockTree { // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { - // std::cout << "level " << level << std::endl; - #ifdef BT_DBG + std::cout << "level " << level << std::endl; + TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); scan_block_pairs(text, current, is_padded, threads); #ifdef BT_DBG + std::cout << "scanned block pairs " << std::endl; pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); @@ -202,6 +147,7 @@ class BlockTreeFPParPH : public BlockTree { #endif scan_blocks(text, current, is_padded, threads); #ifdef BT_DBG + std::cout << "scanned blocks " << std::endl; blocks_ns += std::chrono::duration_cast( Clock::now() - now) .count(); @@ -209,14 +155,17 @@ class BlockTreeFPParPH : public BlockTree { #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); - } #ifdef BT_DBG - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif + } else { + break; + } } #ifdef BT_DBG TimePoint now = Clock::now(); @@ -251,40 +200,6 @@ class BlockTreeFPParPH : public BlockTree { #endif } - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - inline void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - inline void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - /// @brief Scan through the blocks pairwise in order to identify which blocks /// should /// be replaced with back blocks. @@ -322,24 +237,31 @@ class BlockTreeFPParPH : public BlockTree { const size_t num_block_pairs = num_blocks - 1 - is_padded; const auto& block_starts = *level.block_starts; #pragma omp single - for (size_t i = 0; i < num_block_pairs; ++i) { - // If the next block is not adjacent, we cannot hash the pair starting - // at the current block - if (!level.next_is_adjacent(i)) { - continue; - } - // Move the hasher to the current block pair - RabinKarp rk(text, SIGMA, block_starts[i], pair_size, K_PRIME); - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it doesn't - // exist, and add the current block to the entry - auto ptr = map.find(hash); - if (ptr == map.end()) { - auto [insert_ptr, _] = map.insert({hash, PairOccurrences(i)}); - ptr = insert_ptr; + { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = 0; i < num_block_pairs; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + auto ptr = map.find(hash); + if (ptr == map.end()) { + auto [insert_ptr, _] = map.insert({hash, PairOccurrences(i)}); + ptr = insert_ptr; + } + ptr->second.add_block_pair(i); + ptr->second.update(i); } - ptr->second.add_block(i); - ptr->second.update(i); } #pragma omp barrier #ifdef BT_DBG @@ -355,8 +277,9 @@ class BlockTreeFPParPH : public BlockTree { // Hash every window and determine for all block pairs whether they have // previous occurrences. - size_t segment_size = - std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); + size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, omp_get_num_threads())); const size_t thread_id = omp_get_thread_num(); // Start and end index of the current thread's segment @@ -365,7 +288,11 @@ class BlockTreeFPParPH : public BlockTree { std::min(num_block_pairs, (thread_id + 1) * segment_size); if (start < static_cast(num_block_pairs)) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, K_PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -395,7 +322,6 @@ class BlockTreeFPParPH : public BlockTree { markings[occ + 1] = markings[occ + 1] | 0b01; } } - map.erase(it); } #ifdef BT_DBG bp_markings_ns += @@ -449,62 +375,6 @@ class BlockTreeFPParPH : public BlockTree { } } - struct BlockOccurrences { - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - std::mutex list_mutex; - - BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences(), - list_mutex() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences), - list_mutex() {} - - BlockOccurrences(BlockOccurrences&& other) - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)), - list_mutex() {} - - inline void add_block(size_type block_index) { - const std::lock_guard lock(list_mutex); - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - inline void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - //||(first_occ_block == block_index && block_offset < first_occ_offset)) { - } - }; - /// @brief Determine the positions for each block's earliest occurrence if /// there is any. /// @@ -518,8 +388,9 @@ class BlockTreeFPParPH : public BlockTree { const size_t threads) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -548,10 +419,10 @@ class BlockTreeFPParPH : public BlockTree { #pragma omp single for (size_type i = 0; i < level_data.num_blocks - is_padded; ++i) { const RabinKarp rk(text, - SIGMA, + internal::sharded::SIGMA, block_starts[i], level_data.block_size, - K_PRIME); + internal::sharded::PRIME); const RabinKarpHash hash = rk.current_hash(); auto ptr = links.find(hash); if (ptr == links.end()) { @@ -579,7 +450,8 @@ class BlockTreeFPParPH : public BlockTree { const size_t thread_id = omp_get_thread_num(); const size_t segment_size = - ceil_div(num_total_iterations, omp_get_num_threads()); + internal::sharded::ceil_div(num_total_iterations, + omp_get_num_threads()); const size_t start = thread_id * segment_size; const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); @@ -587,10 +459,10 @@ class BlockTreeFPParPH : public BlockTree { // Hash every window and find the first occurrences for every block. if (start < block_starts.size() - is_padded) { RabinKarp rk(text, - SIGMA, + internal::sharded::SIGMA, block_starts[start], level_data.block_size, - K_PRIME); + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { continue; @@ -837,7 +709,9 @@ class BlockTreeFPParPH : public BlockTree { const size_type last_block_parent_start = previous_level.block_starts->back(); const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; @@ -865,7 +739,7 @@ class BlockTreeFPParPH : public BlockTree { prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -951,7 +825,7 @@ class BlockTreeFPParPH : public BlockTree { const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing to this, // then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -979,7 +853,7 @@ class BlockTreeFPParPH : public BlockTree { (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index 3535e71..e006fcc 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -140,7 +140,7 @@ class RecursiveBitBlockTreeSharded // Prepare the top level levels.emplace_back(0, top_block_size, text_len / top_block_size); LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + top_level.block_starts->reserve(internal::sharded::ceil_div(text_len, top_level.block_size)); for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); } @@ -286,13 +286,6 @@ class RecursiveBitBlockTreeSharded #endif } - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - [[maybe_unused]] static void print_aggregate(const char* name, const tlx::Aggregate& agg, @@ -387,7 +380,7 @@ class RecursiveBitBlockTreeSharded // Hash every window and determine for all block pairs whether // they have previous occurrences. const size_t segment_size = - std::max(1, ceil_div(num_block_pairs, num_threads)); + std::max(1, internal::sharded::ceil_div(num_block_pairs, num_threads)); // Start and end index of the current thread's segment const auto start = thread_id * segment_size; @@ -754,7 +747,7 @@ class RecursiveBitBlockTreeSharded // Number of total iterations the for loop should do const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); + const size_t segment_size = internal::sharded::ceil_div(num_total_iterations, num_threads); // The start and end index of the current thread's segment const size_t start = thread_id * segment_size; const size_t end = std::min(num_total_iterations, @@ -1208,7 +1201,7 @@ class RecursiveBitBlockTreeSharded const size_type last_block_parent_start = previous_level.block_starts->back(); const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); + new_size += internal::sharded::ceil_div(text_len - last_block_parent_start, block_size); } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; @@ -1401,7 +1394,7 @@ class RecursiveBitBlockTreeSharded this->max_leaf_length_ = max_leaf_length; this->num_bits_ = text.size(); const std::span bytes(reinterpret_cast(text.data().data()), - ceil_div(text.size(), 8ULL)); + internal::sharded::ceil_div(text.size(), 8ULL)); construct(bytes, threads, queue_size); omp_set_dynamic(old_dynamic); omp_set_num_threads(old); diff --git a/include/pasta/block_tree/dense_bit_block_tree.hpp b/include/pasta/block_tree/dense_bit_block_tree.hpp deleted file mode 100644 index 4222e1f..0000000 --- a/include/pasta/block_tree/dense_bit_block_tree.hpp +++ /dev/null @@ -1,791 +0,0 @@ -/******************************************************************************* - * This file is part of pasta::block_tree - * - * Copyright (C) 2023 Etienne Palanga - * - * pasta::block_tree is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * pasta::block_tree is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with pasta::block_tree. If not, see . - * - ******************************************************************************/ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace pasta { - -template -class DenseBitBlockTree { -public: - /// If this is true, then the only levels of the tree start to be - /// included starting at the first level that contains a back block - /// - /// For example, if levels 0 to 5 do not contain any back blocks, then the - /// tree will only contain levels 6 and below. - bool CUT_FIRST_LEVELS = true; - - /// The arity of the tree - size_type tau_; - size_type max_leaf_length_; - /// The arity of the tree's root - size_type s_ = 1; - size_type leaf_size = 0; - size_type amount_of_leaves = 0; - size_type num_bits; - bool rank_support = false; - /// Bit vectors for each level determining whether a block is internal - /// (=1) or not (=0) - std::vector block_tree_types_; - std::vector*> - block_tree_types_rs_; - /// For each level and each back block, contains the index of the - /// block's source - std::vector*> block_tree_pointers_; - std::vector*> block_tree_offsets_; - - std::vector block_size_lvl_; - std::vector block_per_lvl_; - std::vector leaves_; - - std::vector compress_map_; - std::vector decompress_map_; - sdsl::int_vector<> compressed_leaves_; - - /// @brief For each level and each block, contains the number of 1s up to (and - /// including) the block. - std::vector> one_ranks_; - /// @brief For each level and each back block, - /// contains the number of 1s up to (and including) the pointed-to area of - /// the back-block. - std::vector> pointer_prefix_one_counts_; - - [[nodiscard]] size_t height() const { - return block_tree_types_.size(); - } - - bool access(const size_type bit_index) const { - // FIXME: As of now this works on little endian systems only - const int64_t byte_index = bit_index / 8; - const int64_t bit_offset = bit_index % 8; - - int64_t block_size = block_size_lvl_[0]; - int64_t block_index = byte_index / block_size; - int64_t off = byte_index % block_size; - for (size_t i = 0; i < height(); i++) { - const auto& is_internal = *block_tree_types_[i]; - const auto& is_internal_rank = *block_tree_types_rs_[i]; - const auto& pointers = *block_tree_pointers_[i]; - const auto& offsets = *block_tree_offsets_[i]; - if (!is_internal[block_index]) { - // If this block is not internal, go to its pointed-to block - const size_t back_block_index = is_internal_rank.rank0(block_index); - off = off + offsets[back_block_index]; - block_index = pointers[back_block_index]; - if (off >= block_size) { - ++block_index; - off -= block_size; - } - } - block_size /= tau_; - const int64_t child = off / block_size; - off %= block_size; - block_index = is_internal_rank.rank1(block_index) * tau_ + child; - } - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; - return ((1 << bit_offset) & byte) != 0; - }; - -private: - template - [[nodiscard]] size_t find_initial_block(const size_t rank) const { - const auto& top_one_ranks = one_ranks_[0]; - const size_t block_size = block_size_lvl_[0]; - size_t start = (rank - 1) / (block_size * 8); - size_t end = top_one_ranks.size() - 1; - while (start != end) { - const size_t middle = start + (end - start) / 2; - size_t current_rank; - if constexpr (one) { - current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; - } else { - const size_t middle_bits = middle * block_size * 8; - current_rank = - (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; - } - if (current_rank < rank) { - if (start + 1 == end) { - size_t bits; - if constexpr (one) { - bits = top_one_ranks[middle]; - } else { - bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; - } - // If there is only one block left, it's either the current or the - // next block - if (bits < rank) { - start = middle + 1; - } - break; - } - start = middle; - } else { - end = middle - 1; - } - } - return start; - } - -public: - [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - size_t block_size = block_size_lvl_[0]; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - - size_t pos = (current_block * block_size * 8) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; - - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal_rank.rank0(current_block); - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - size_t rank_d = - (current_block == 0) ? - top_one_ranks[current_block] : - top_one_ranks[current_block] - top_one_ranks[current_block - 1]; - rank_d -= pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal_rank.rank1(current_block) * tau_; - block_size /= tau_; - const size_t start_block = current_block; - while (one_ranks[current_block] < rank) { - ++current_block; - } - rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; - pos += (current_block - start_block) * block_size * 8; - if (!is_internal[current_block]) { - size_t back_block_index = is_internal_rank.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - size_t rank_d = - (current_block % tau_ == 0) ? - one_ranks[current_block] : - one_ranks[current_block] - one_ranks[current_block - 1]; - rank_d -= pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = - block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_ones = std::popcount(byte); - if (rank > num_ones) { - rank -= num_ones; - pos += 8; - ++byte_offset; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) > 0; - } - } - } - return pos; - } - - [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - const auto& top_one_ranks = one_ranks_[0]; - - const size_t top_block_size = block_size_lvl_[0]; - const auto top_zero_ranks = [&top_one_ranks, - top_block_size](const size_t i) -> size_t { - return (i + 1) * top_block_size * 8 - top_one_ranks[i]; - }; - - // Binary Search for the correct top level block containing the correct 1 - size_t current_block = find_initial_block(rank); - const size_t top_block_bits = top_block_size * 8; - - size_t pos = (current_block * top_block_bits) - 1; - // ReSharper disable once CppDFAUnreachableCode - rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); - // If that block is a back block, we need to move to the back-pointed block - if (!top_is_internal[current_block]) { - const size_t back_block_index = top_is_internal_rank.rank0(current_block); - // const size_t child_block_bits = - // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; - current_block = top_pointers[back_block_index]; - const size_t offset = top_offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block == 0) ? - top_zero_ranks(current_block) : - top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (top_block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; - pos -= offset * 8; - } - } - - size_t block_size = block_size_lvl_[0]; - size_t level = 1; - while (level < height()) { - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; - const auto& offsets = *block_tree_offsets_[level]; - const auto& pointers = *block_tree_pointers_[level]; - const auto& one_ranks = one_ranks_[level]; - - current_block = prev_is_internal_rank.rank1(current_block) * tau_; - block_size /= tau_; - - const auto zero_ranks = - [&one_ranks, this, block_size](const size_t i) -> size_t { - const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; - return rnk; - }; - const size_t start_block = current_block; - while (zero_ranks(current_block) < rank) { - ++current_block; - } - rank -= - (current_block == start_block) ? 0 : zero_ranks(current_block - 1); - pos += (current_block - start_block) * block_size * 8; - if (!is_internal[current_block]) { - size_t back_block_index = is_internal_rank.rank0(current_block); - current_block = pointers[back_block_index]; - const size_t offset = offsets[back_block_index]; - const size_t prefix_bits = offset * 8; - size_t rank_d = - (current_block % tau_ == 0) ? - zero_ranks(current_block) : - zero_ranks(current_block) - zero_ranks(current_block - 1); - rank_d -= prefix_bits - pointer_ranks[back_block_index]; - if (rank > rank_d) { - rank -= rank_d; - pos += (block_size - offset) * 8; - ++current_block; - } else { - rank += prefix_bits - pointer_ranks[back_block_index]; - pos -= offset * 8; - } - } - ++level; - } - - current_block = - block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - size_t byte_offset = 0; - while (rank > 0) { - const uint8_t byte = - decompress_map_[compressed_leaves_[current_block * leaf_size + - byte_offset]]; - const uint8_t num_zeros = 8 - std::popcount(byte); - if (rank > num_zeros) { - rank -= num_zeros; - pos += 8; - byte_offset++; - } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { - pos++; - rank -= ((1 << bit) & byte) == 0; - } - } - } - return pos; - } - - /// @brief Counts the number of 1-bits up to (and excluding) an index. - [[nodiscard("rank result discarded")]] size_t - rank1(const size_type bit_index) const { - const size_t byte_index = bit_index / 8; - const auto& top_is_internal = *block_tree_types_[0]; - const auto& top_is_internal_rank = *block_tree_types_rs_[0]; - const auto& top_pointers = *block_tree_pointers_[0]; - const auto& top_offsets = *block_tree_offsets_[0]; - size_t block_size = block_size_lvl_[0]; - size_t block_index = byte_index / block_size; - size_t block_offset = byte_index % block_size; - size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; - if (!top_is_internal[block_index]) { - // If the top block is a back block, go to it and adjust the offset - const size_t back_block_index = top_is_internal_rank.rank0(block_index); - rank -= pointer_prefix_one_counts_[0][back_block_index]; - block_offset += top_offsets[back_block_index]; - block_index = top_pointers[back_block_index]; - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it - rank += - (block_index == 0) ? - one_ranks_[0][block_index] : - (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); - ++block_index; - block_offset -= block_size; - } - } - - // Go down to the next level - block_size /= tau_; - // How many children are we 'skipping over' - size_t child = block_offset / block_size; - block_offset %= block_size; - block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; - - size_t level = 1; - while (level < height()) { - const auto& ranks = one_ranks_[level]; - const auto& pointer_ranks = pointer_prefix_one_counts_[level]; - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - rank += (child == 0) ? 0 : ranks[block_index - 1]; - // If this block is internal, just go to the correct child - if (is_internal[block_index]) { - block_size /= tau_; - child = block_offset / block_size; - block_offset %= block_size; - block_index = is_internal_rank.rank1(block_index) * tau_ + child; - level++; - continue; - } - - // If we have a back block, we need to go to the pointed-to block - const size_t back_block_index = is_internal_rank.rank0(block_index); - rank -= pointer_ranks[back_block_index]; - block_offset += (*block_tree_offsets_[level])[back_block_index]; - block_index = (*block_tree_pointers_[level])[back_block_index]; - child = block_index % tau_; - - if (block_offset >= block_size) { - // If we're exceeding the pointed-to block's offset, - // add the ones inside of it and go to the next block - rank += (child == 0) ? ranks[block_index] : - (ranks[block_index] - ranks[block_index - 1]); - ++block_index; - child = block_index % tau_; - block_offset -= block_size; - } - const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; - rank -= remove_prefix; - } - - // Number of leaves that exist before the leaves of the current block - const size_type prefix_leaves = block_index - child; - for (size_t block = 0; block < child * leaf_size; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + - block]]; - rank += std::popcount(byte); - } - for (size_t block = 0; block < block_offset; block++) { - const uint8_t byte = - decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; - rank += std::popcount(byte); - } - - // Masks to remove bits from the last byte, - // that aren't part of the ran query - static constexpr std::array MASKS = { - 0b0000'0000, - 0b0000'0001, - 0b0000'0011, - 0b0000'0111, - 0b0000'1111, - 0b0001'1111, - 0b0011'1111, - 0b0111'1111, - }; - rank += std::popcount( - decompress_map_[compressed_leaves_[block_index * leaf_size + - block_offset]] & - MASKS[bit_index % 8]); - return rank; - } - - /// @brief Counts the number of 0-bits up to (and excluding) an index. - size_t rank0(const size_type bit_index) const { - return bit_index - rank1(bit_index); - } - - size_t print_space_usage() const { - size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + - sizeof(leaf_size); - auto delta_size = 0; - for (const auto bv : block_tree_types_) { - space_usage += bv->size() / 8; - delta_size += bv->size() / 8; - } - std::cout << "bv size: " << delta_size << std::endl; - delta_size = 0; - for (const auto rs : block_tree_types_rs_) { - space_usage += rs->space_usage(); - delta_size += rs->space_usage(); - } - std::cout << "rs size: " << delta_size << std::endl; - delta_size = 0; - for (const auto iv : block_tree_pointers_) { - space_usage += sdsl::size_in_bytes(*iv); - delta_size += sdsl::size_in_bytes(*iv); - } - std::cout << "ptrs size: " << delta_size << std::endl; - delta_size = 0; - for (const auto iv : block_tree_offsets_) { - space_usage += sdsl::size_in_bytes(*iv); - delta_size += sdsl::size_in_bytes(*iv); - } - std::cout << "offs size: " << delta_size << std::endl; - if (rank_support) { - for (auto v : block_size_lvl_) { - space_usage += sizeof(v); - } - for (auto v : block_per_lvl_) { - space_usage += sizeof(v); - } - } - - for (auto& rs : one_ranks_) { - space_usage += sdsl::size_in_bytes(rs); - } - - for (auto& rs : pointer_prefix_one_counts_) { - space_usage += sdsl::size_in_bytes(rs); - } - - // space_usage += leaves_.size() * sizeof(uint8_t); - space_usage += sdsl::size_in_bytes(compressed_leaves_); - space_usage += compress_map_.size(); - - return space_usage; - }; - - int32_t add_bit_rank_support() { - rank_support = true; - - // Resize rank information vectors - one_ranks_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - one_ranks_[level].resize(block_tree_types_[level]->size()); - } - pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); - for (uint64_t level = 0; level < height(); level++) { - pointer_prefix_one_counts_[level].resize( - block_tree_pointers_[level]->size()); - } - - for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { - bit_rank_block(0, block); - } - - for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { - one_ranks_[0][block] += one_ranks_[0][block - 1]; - } - - for (size_t level = 1; level < height(); level++) { - size_type counter = tau_; - size_t acc = 0; - for (size_t block = 0; block < one_ranks_[level].size(); block++) { - const size_type ones_in_block = one_ranks_[level][block]; - acc += ones_in_block; - one_ranks_[level][block] = acc; - --counter; - if (counter == 0) { - acc = 0; - counter = tau_; - } - } - } - for (auto& prefix_one_counts : pointer_prefix_one_counts_) { - sdsl::util::bit_compress(prefix_one_counts); - } - for (auto& ranks : one_ranks_) { - sdsl::util::bit_compress(ranks); - } - return 0; - } - -protected: - void compress_leaves() { - // Holds a 1 on every char that exists - compress_map_.resize(256, 0); - decompress_map_.resize(256, 0); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compress_map_[this->leaves_[i]] = 1; - } - for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { - const size_t tmp = compress_map_[c]; - compress_map_[c] = cur_val; - decompress_map_[cur_val] = c; - cur_val += tmp; - } - - compressed_leaves_.resize(this->leaves_.size()); - for (size_t i = 0; i < this->leaves_.size(); ++i) { - compressed_leaves_[i] = compress_map_[this->leaves_[i]]; - } - sdsl::util::bit_compress(this->compressed_leaves_); - leaves_.resize(0); - leaves_.shrink_to_fit(); - } - /// @brief Calculate the number of leading zeros for a 32-bit integer. - /// This value is capped at 31. - static size_type leading_zeros(const int32_t val) { - return __builtin_clz(static_cast(val) | 1); - } - - /// @brief Calculate the number of leading zeros for a 64-bit integer. - /// This value is capped at 64. - static size_type leading_zeros(const int64_t val) { - return __builtin_clzll(static_cast(val) | 1); - } - - /// - /// @brief Determine the padding and minimum height and the size of the blocks - /// on the top level of a block tree with s top-level blocks and an arity of - /// tau with leaves also of size tau. - /// - /// The height is the number of levels in the tree. - /// The padding is the number of characters that the top-level exceeds the - /// text length. For example, if the result was that the top level consists of - /// s = 5 blocks of size 30 and the text size being 80, then the padding would - /// be (5 * 30) - 80 = 70. - /// - /// @param[out] padding The number of characters in the last block (of the - /// first level of the tree) that are empty. - /// @param[in] bv_length The number of bits in the input bit vector. - /// @param[out] height The number of levels in the tree. - /// @param[out] blk_size The size of blocks on the first level of the tree. - /// - void calculate_padding(int64_t& padding, - int64_t bv_length, - int64_t& height, - int64_t& blk_size) { - // This is the number of characters occupied by a tree with s*tau^h levels - // and leaves of size tau. At the start, we only have a tree with the first - // level with s leaf blocks which each have size tau. If we insert another - // level, the number of leaf blocks (and therefore the number of occupied - // characters) increases by a factor of tau. - int64_t tmp_padding = this->s_ * this->tau_; - int64_t h = 1; - // Bit size of the blocks on the current level (starting at the leaf level) - blk_size = tau_; - // While the tree does not cover the entire text, add a level - while (tmp_padding < bv_length) { - tmp_padding *= this->tau_; - blk_size *= this->tau_; - h++; - } - // once the tree has enough levels to cover the entire text, we set the - // tree's values - height = h; - // The padding is the number of excess characters that the block tree covers - // over the length of the text. - padding = tmp_padding - bv_length; - } - - size_type bit_rank_block(size_type level, size_type block_index) { - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal[block_index]) { - const size_type internal_index = is_internal_rank.rank1(block_index); - if (static_cast(level) < height() - 1) { - // If we are not on the last level recursively call - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); - } - } else { - // If we are on the last level - for (size_type k = 0; k < tau_; ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - } - } - } else { - const size_type back_block_index = is_internal_rank.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - size_type num_ones_parts = 0; - num_ones += one_ranks_[level][ptr]; - if (off > 0) { - num_ones_parts = part_bit_rank_block(level, ptr, off); - const size_type num_ones_2nd_part = - part_bit_rank_block(level, ptr + 1, off); - num_ones -= num_ones_parts; - num_ones += num_ones_2nd_part; - } - pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; - } - one_ranks_[level][block_index] = num_ones; - return num_ones; - } - - size_type part_bit_rank_block(const size_type level, - const size_type block_index, - const size_type chars_to_process) { - const auto& is_internal = *block_tree_types_[level]; - const auto& is_internal_rank = *block_tree_types_rs_[level]; - if (static_cast(block_index) >= is_internal.size()) { - return 0; - } - - size_type num_ones = 0; - if (is_internal[block_index]) { - const size_type internal_index = is_internal_rank.rank1(block_index); - size_type k = 0; - size_type processed_chars = 0; - if (static_cast(level) < height() - 1) { - const size_type child_size = block_size_lvl_[level + 1]; - // We're not on the last level - // iterate over the children as long as we don't exceed the limit - for (k = 0; - k < tau_ && processed_chars + child_size <= chars_to_process; - ++k) { - num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; - processed_chars += child_size; - } - - // If we still need to process more chars and they end inside the next - // child, rank that part of the next child - if (processed_chars != chars_to_process) { - num_ones += part_bit_rank_block(level + 1, - internal_index * tau_ + k, - chars_to_process - processed_chars); - } - } else { - // We're on the last level - for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; - ++k) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); - processed_chars += leaf_size; - } - - if (processed_chars != chars_to_process) { - num_ones += bit_rank_leaf(internal_index * tau_ + k, - chars_to_process % leaf_size); - } - } - } else { - const size_type back_block_index = is_internal_rank.rank0(block_index); - const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; - const size_type off = (*block_tree_offsets_[level])[back_block_index]; - - // If we need to process chars beyond this block, we need to - if (chars_to_process + off >= block_size_lvl_[level]) { - // Ones in the entire block this block points to - num_ones += one_ranks_[level][ptr]; - // Ones that overflow into the next block - num_ones += part_bit_rank_block(level, - ptr + 1, - chars_to_process + off - - block_size_lvl_[level]); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } else { - // Number of ones up to the cutoff point - num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); - // Num ones in the pointed-to block *before* the pointed-to area - num_ones -= pointer_prefix_one_counts_[level][back_block_index]; - } - } - return num_ones; - } - - /// - /// @brief Count ones in leaf block. - /// - /// @param leaf_index The index of the leaf block. - /// @param max_char_index The maximum character index (exclusive) to - /// consider. This is used for when this block is at the end of the string. - /// @return The number of ones in this block. - /// - size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { - if (static_cast(leaf_index * leaf_size) >= - compressed_leaves_.size()) { - return 0; - } - - size_type result = 0; - for (size_type i = 0; i < max_char_index; ++i) { - const uint8_t byte = - decompress_map_[compressed_leaves_[leaf_index * leaf_size + i]]; - result += std::popcount(byte); - } - return result; - } -}; - - - -} // namespace pasta - -/******************************************************************************/ diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp index 287df57..9436819 100644 --- a/include/pasta/block_tree/utils/sharded_util.hpp +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -297,8 +297,8 @@ constexpr uint64_t mix_select(uint64_t key) { /// @brief Returns the ceiling of x / y for x > 0; /// /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c -inline size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + (x - 1) / y; +size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + (static_cast(x) - 1) / static_cast(y); } } // namespace pasta::internal::sharded \ No newline at end of file From 7d7040c30dac960bd3f0b5183710e8a69fd98907 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 15 Dec 2023 00:52:58 +0100 Subject: [PATCH 73/92] refactor and update block_tree_sharded --- CMakeLists.txt | 3 +- CMakePresets.json | 2 +- examples/build_bt.cpp | 2 +- .../construction/block_tree_sharded.hpp | 1860 +++++++++-------- .../pasta/block_tree/utils/sharded_map.hpp | 38 +- .../pasta/block_tree/utils/sharded_util.hpp | 49 +- .../block_tree/utils/sync_sharded_map.hpp | 60 +- 7 files changed, 996 insertions(+), 1018 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index c7dad0c..9bcbd18 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -133,7 +133,8 @@ target_link_libraries(pasta_block_tree INTERFACE sdsl #jiffy jiffy1 - libzstd_static) + # libzstd_static +) target_include_directories(pasta_block_tree INTERFACE ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) diff --git a/CMakePresets.json b/CMakePresets.json index 15aa4a4..df26352 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -17,7 +17,7 @@ "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -fsanitize=address -fsanitize=leak -fsanitize=undefined" + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -static-libasan -fsanitize=address -fsanitize=leak -fsanitize=undefined" } }, { diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 33b18fa..293f9ee 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,7 @@ #include #include -#define PAR_PHMAP +#define PAR_SHARDED #define REC_BIT #if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED diff --git a/include/pasta/block_tree/construction/block_tree_sharded.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp index c56b64c..acde1ed 100644 --- a/include/pasta/block_tree/construction/block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -221,129 +221,133 @@ class BlockTreeSharded : public BlockTree { #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); - } #ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif - } + } else { + break; + } #ifdef BT_INSTRUMENT # if defined(BT_DBG) - std::cout << "pairs: " << (pairs_ns / 1'000'000) - << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) - << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) - << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) - << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) - << "ms,\nblocks: " << (blocks_ns / 1'000'000) - << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) - << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) - << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) - << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," - << std::endl; + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; # elif defined(BT_BENCH) - std::cout << " pairs=" << (pairs_ns / 1'000'000) - << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) - << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) - << " markings=" << (bp_markings_ns / 1'000'000) - << " bitvec=" << (bp_bitvec_ns / 1'000'000) - << " blocks=" << (blocks_ns / 1'000'000) - << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) - << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) - << " update_blocks=" << (b_update_blocks_ns / 1'000'000) - << " generate_next=" << (generate_ns / 1'000'000); + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); # endif - now = Clock::now(); + now = Clock::now(); #endif - prune(levels); + prune(levels); #ifdef BT_INSTRUMENT - size_t prune_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); + size_t prune_ns = std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); # ifdef BT_DBG - std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; # elif defined BT_BENCH - std::cout << " prune=" << (prune_ns / 1'000'000); + std::cout << " prune=" << (prune_ns / 1'000'000); # endif #endif - make_tree(text, levels, padding); + make_tree(text, levels, padding); #ifdef BT_INSTRUMENT - size_t make_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); + size_t make_ns = std::chrono::duration_cast( + Clock::now() - now) + .count(); # ifdef BT_DBG - std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; # elif defined BT_BENCH - std::cout << " make=" << (make_ns / 1'000'000); + std::cout << " make=" << (make_ns / 1'000'000); # endif #endif - } - - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - [[maybe_unused]] static void - print_aggregate(const char* name, - const tlx::Aggregate& agg, - const size_t div = 1) { - printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", - name, - static_cast(agg.min() / div), - static_cast(agg.max() / div), - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - static_cast(agg.count())); - } - - /// @brief Scan through the blocks pairwise in order to identify which blocks - /// should be replaced with back blocks. - /// - /// @param text The input string. - /// @param level The data for the current level. - /// @param is_padded `true` iff the last block on this level *does not* end at - /// the exact end of the text. - /// @param threads Number of threads to use - /// @param queue_size The size of the queue to use per thread in the sharded - /// hash map. - /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or - /// use the blocks' contents themselves as hashes. - /// For block sizes greater than 4 bytes, use Rabin-Karp. - /// - template - void scan_block_pairs(const std::vector& text, - LevelData& level, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - if (level.num_blocks < 4) { - level.is_internal = std::make_unique(level.num_blocks, true); - level.is_internal_rank = std::make_unique(*level.is_internal); - return; } - // A map containing hashed block pairs mapped to their indices of the - // pairs' first block respectively - BlockPairMap map(threads, queue_size); + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void print_aggregate( + const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } - std::atomic_size_t threads_done = 0; - std::atomic_bool last_done = false; - auto& barrier = map.barrier(); + /// @brief Scan through the blocks pairwise in order to identify which + /// blocks should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end + /// at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings + /// or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); #ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; # pragma omp parallel default(none) num_threads(threads) \ shared(level, \ @@ -373,351 +377,351 @@ class BlockTreeSharded : public BlockTree { barrier, \ internal::sharded::HASH_MASKS) #endif - { - const size_t thread_id = omp_get_thread_num(); - typename BlockPairMap::Shard shard = map.get_shard(thread_id); - const size_t num_threads = omp_get_num_threads(); - const size_t num_block_pairs = level.num_blocks - 1 - is_padded; - const size_t block_size = level.block_size; - const size_t pair_size = 2 * block_size; - const auto& block_starts = *level.block_starts; - - // Hash every window and determine for all block pairs whether - // they have previous occurrences. - const size_t segment_size = - std::max(1, ceil_div(num_block_pairs, num_threads)); - - // Start and end index of the current thread's segment - const auto start = thread_id * segment_size; - const auto end = - std::min(num_block_pairs, (thread_id + 1) * segment_size); - - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - internal::sharded::SIGMA, - block_starts[0], - pair_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block - if (!level.next_is_adjacent(i)) { - continue; - } - rk.restart(block_starts[i]); - // Move the hasher to the current block pair - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } else { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const input_type* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start, - block_size); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } - - if (const size_t thread_order = - threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - // Now, we handle the queue asynchronously - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - { - bp_hash_pairs_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - tlx::Aggregate thread_scan_hits; -#else { - } -#endif + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); - if (start < static_cast(num_block_pairs)) { if constexpr (use_hash == UseHash::RABIN_KARP) { RabinKarp rk(text, internal::sharded::SIGMA, - block_starts[start], + block_starts[0], pair_size, internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { continue; } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); - } - scan_windows_in_block_pair(rk, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); } } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); } - scan_windows_in_block_pair_identity(text, - block_starts[i], - pair_size, - map, - block_size, - i + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); + ); + } } } - } #ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); # pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; #endif - } + } #ifdef BT_INSTRUMENT - bp_scan_pairs_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); + bp_scan_pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); # ifdef BT_DBG - tlx::Aggregate map_loads; + tlx::Aggregate map_loads; - for (size_t load : map.map_loads()) { - map_loads.add(load); - } + for (size_t load : map.map_loads()) { + map_loads.add(load); + } - print_aggregate("Pair Map Loads ", map_loads); - print_aggregate("Pair Map Hits ", scan_hits); - print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); - BT_ASSERT(map.num_inserts_.load() == map.size()); + BT_ASSERT(map.num_inserts_.load() == map.size()); # endif #endif - level.is_internal = std::make_unique(level.num_blocks); - fill_is_internal(*level.is_internal, map); - level.is_internal_rank = std::make_unique(*level.is_internal); - } - - /// @brief Fills the bit vector `is_internal` based on the values in the - /// given map. - /// @param is_internal An unfilled bit vector with a bit for each block on - /// this level. - /// @param map A map, mapping hashed block pairs to their first occurrence's - /// block index. - void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { - const size_type num_blocks = is_internal.size(); + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector & is_internal, BlockPairMap & map) { + const size_type num_blocks = is_internal.size(); #ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); + TimePoint now = Clock::now(); #endif - // Set up the packed array holding the markings for each block. - // Each mark is a 2-bit number. - // The MSB is 1 iff the block and its successor have a prior - // occurrence. The LSB is 1 iff the block and its predecessor - // have a prior occurrence. - sdsl::int_vector<2> markings(num_blocks, 0); - map.for_each( - [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { - for (const size_type occ : pair_occs.occurrences) { - if (pair_occs.first_occ_block < occ) { - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } } - } - }); + }); #ifdef BT_INSTRUMENT - bp_markings_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); + bp_markings_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); #endif - // Generate the bit vector indicating which blocks are internal - is_internal[0] = true; - is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; - for (size_type i = 0; i < num_blocks - 1; ++i) { - const bool block_is_internal = markings[i] != 0b11; - is_internal[i] = block_is_internal; - } + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } #ifdef BT_INSTRUMENT - bp_bitvec_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); + bp_bitvec_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif - } - - /// @brief Scan through the windows starting in a block and mark - /// them accordingly if they represent the earliest occurrence of some - /// block hash. - /// - /// The supplied `RabinKarp` hasher must be at the start of the block. - /// @param rk A Rabin-Karp hasher whose state is at the start of the block. - /// @param map The map containing the hashes of block pairs mapped to their - /// block indexes at which they occur. - /// @param num_iterations The number of contiguous windows to hash. - /// @param current_block_index The index of the block being currently - /// hashed. - static inline void - scan_windows_in_block_pair(RabinKarp& rk, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void scan_windows_in_block_pair( + RabinKarp & rk, + BlockPairMap & map, + const size_t num_iterations, + const size_type current_block_index #ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg + , + tlx::Aggregate& agg #endif - ) { - for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { - RabinKarpHash current_hash = rk.current_hash(); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { #ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); + agg.add(0); + continue; + } else { + agg.add(100); #else - continue; + continue; #endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); } - } - - static inline void - scan_windows_in_block_pair_identity(const std::vector& text, - const size_t block_start, - const size_t pair_size, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index + + static inline void scan_windows_in_block_pair_identity( + const std::vector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index #ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg + , + tlx::Aggregate& agg #endif - ) { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; - const input_type* block_start_ptr = text.data() + block_start; - for (size_t offset = 0; offset < num_iterations; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash current_hash(text, - internal::sharded::mix_select(hash_value), - block_start + offset, - pair_size); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { #ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); + agg.add(0); + continue; + } else { + agg.add(100); #else - continue; + continue; #endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - - /// @brief Determine the positions for each block's earliest occurrence if - /// there is any. - /// - /// @param text The input text - /// @param level_data The data for the current level - /// @param is_padded true, iff the last block of the level extends past the - /// end of the text - /// @param threads The number of threads to use during construction. - /// @param queue_size The max number of items in each thread's queues. - /// @tparam use_hash Determines whether to use a rabin karp hash for hashing - /// text windows or to use the block's content as a hash. For any window size - /// greater than 8 bytes, use Rabin-Karp. - template - void scan_blocks(const std::vector& text, - LevelData& level_data, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - const size_t num_blocks = level_data.num_blocks; - - level_data.pointers = std::make_unique>( - num_blocks, - internal::sharded::NO_EARLIER_OCC); - level_data.offsets = - std::make_unique>(num_blocks, 0); - level_data.counters = - std::make_unique>(num_blocks, 0); - - if (num_blocks <= 2) { - return; } - // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size); + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window + /// size greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); - // The number of threads finished with hashing blocks - std::atomic_size_t num_done = 0; - // Whether the last thread is done - std::atomic_bool last_done = false; - auto& barrier = links.barrier(); + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); #ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; # pragma omp parallel default(none) num_threads(threads) \ shared(level_data, \ @@ -745,370 +749,390 @@ class BlockTreeSharded : public BlockTree { barrier, \ internal::sharded::HASH_MASKS) #endif - { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - internal::sharded::SIGMA, - block_starts[0], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } else { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const input_type* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = + level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } } - } - if (const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); #pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT - { - b_hash_blocks_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } + { + b_hash_blocks_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + } - tlx::Aggregate thread_scan_hits; + tlx::Aggregate thread_scan_hits; #else - { - } + { + } #endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - internal::sharded::SIGMA, - block_starts[start], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; + ); } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); + ); + } } } - } #ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); # pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; #endif - } + } #ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); + b_scan_blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); # ifdef BT_DBG - tlx::Aggregate map_loads; + tlx::Aggregate map_loads; - for (size_t load : links.map_loads()) { - map_loads.add(load); - } + for (size_t load : links.map_loads()) { + map_loads.add(load); + } - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - BT_ASSERT(links.num_inserts_.load() == links.size()); + BT_ASSERT(links.num_inserts_.load() == links.size()); # endif #endif - // By this point, the map should contain the first occurrences of - // every respective block's content. We then fill the pointers - // and offsets with this data and increment counters accordingly - links.for_each( - [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); + }); #ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); #endif - } - - /// @brief Scans through block-sized windows starting inside one block and - /// tries to find blocks with matching hashes in the map. Such blocks - /// will have their earliest occurrence update. - /// @param rk A Rabin-Karp hasher whose current state is at a block start. - /// @param links A map whose keys are hashed blocks and the values - /// are all block indices of blocks matching the hash in ascending order. - /// @param level_data The data for the current level. - /// @param current_block_index The index of the block which the - /// Rabin-Karp hasher is situated in. - static void scan_windows_in_block(RabinKarp& rk, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp & rk, + BlockMap & links, + LevelData & level_data, + const size_type current_block_index #ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits + , + tlx::Aggregate& hits #endif - ) { - for (size_type offset = 0; offset < level_data.block_size; - ++offset, rk.next()) { - RabinKarpHash hash = rk.current_hash(); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { #ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); + hits.add(0.0); + continue; + } else { + hits.add(100.0); #else - continue; + continue; #endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); } - } - - static void - scan_windows_in_block_identity(const std::vector& text, - const size_t block_start, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index + + static void scan_windows_in_block_identity( + const std::vector& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index #ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits + , + tlx::Aggregate& hits #endif - ) { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[level_data.block_size / - sizeof(input_type)]; - const input_type* block_start_ptr = text.data() + block_start; - for (size_type offset = 0; offset < level_data.block_size; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start + offset, - level_data.block_size); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size / + sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { #ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); + hits.add(0.0); + continue; + } else { + hits.add(100.0); #else - continue; + continue; #endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); } - } - - /// @brief Generate the block size, number of block and block start indices - /// for the next level. - /// - /// This depends on the current level's block size, number of blocks and - /// is_internal bit vector being filled. - /// - /// @param text The input text. - /// @param level The level data of the current level. - /// @return The level data of the next level. - [[nodiscard]] LevelData - generate_next_level(const std::vector& text, - const LevelData& level) const { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const auto& is_internal = *level.is_internal; - const size_t next_block_size = block_size / this->tau_; - - std::vector new_block_starts; - new_block_starts.reserve(num_blocks * this->tau_); - for (size_t i = 0; i < num_blocks; ++i) { - if (!is_internal[i]) { - continue; - } - // We generate up to tau new blocks for each internal block, - // excluding blocks that start past the end of the text - const auto parent_block_start = (*level.block_starts)[i]; - for (size_t j = 0, current_block_start = parent_block_start; - j < static_cast(this->tau_) && - current_block_start < text.size(); - ++j, current_block_start += next_block_size) { - new_block_starts.push_back(current_block_start); + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData generate_next_level( + const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } } - } - LevelData next_level(level.level_index + 1, - next_block_size, - new_block_starts.size()); - next_level.block_starts = - std::make_unique>(std::move(new_block_starts)); - return next_level; - } - - /// - /// @brief Takes a vector of levels and fills the block tree fields with - /// them. - /// - /// @param[in] levels A vector containing data for each level, with the - /// first entry corresponding to the topmost level. - /// - void make_tree(const std::vector& text, - std::vector& levels, - int64_t padding) { - const bool is_padded = padding > 0; + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } - // Count the current number of internal blocks per level - std::vector new_num_internal(levels.size(), 0); - for (size_t level = 0; level < levels.size(); level++) { - for (size_t block = 0; block < levels[level].is_internal->size(); - block++) { - if ((*levels[level].is_internal)[block]) { - ++new_num_internal[level]; + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } } } - } - // Create first level - bool found_back_block = levels[0].is_internal->size() > - static_cast(new_num_internal[0]) || - !this->CUT_FIRST_LEVELS; - LevelData& top_level = levels.front(); - if (found_back_block) { - const size_t n = top_level.num_blocks; - const size_t num_internal = new_num_internal[0]; - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_back_blocks = 0; - for (size_t i = 0; i < n; i++) { - // if a back block is found, add its pointer and offset - if (!(*top_level.is_internal)[i]) { - (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; - (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; - num_back_blocks++; + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); } - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(top_level.is_internal.release()); - this->block_tree_types_rs_.push_back( - new Rank(*this->block_tree_types_.back())); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(top_level.block_size); - } - top_level.pointers.reset(); - top_level.offsets.reset(); - top_level.counters.reset(); + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } - // Add level data to the tree - for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData& level = levels[level_index]; - LevelData& previous_level = levels[level_index - 1]; - found_back_block |= static_cast(new_num_internal[level_index]) < - levels[level_index].is_internal->size(); - if (!found_back_block && level_index < levels.size() - 1) { + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore if (level_index < levels.size() - 1) { level.is_internal.reset(); } @@ -1117,267 +1141,251 @@ class BlockTreeSharded : public BlockTree { level.offsets.reset(); level.counters.reset(); previous_level.block_starts.reset(); - continue; } - make_tree_level(levels, - new_num_internal, - level_index, - is_padded, - text.size()); - - // We don't need these anymore - if (level_index < levels.size() - 1) { - level.is_internal.reset(); + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; + b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); + this->amount_of_leaves = leaf_count; + this->compress_leaves(); } - this->leaf_size = levels.back().block_size / this->tau_; - // Construct the leaf string - int64_t leaf_count = 0; - auto& last_is_internal = *levels.back().is_internal; - std::vector& last_block_starts = *levels.back().block_starts; - for (size_t block = 0; block < last_is_internal.size(); block++) { - if (!last_is_internal[block]) { - continue; + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector & levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); } - const size_type block_start = last_block_starts[block]; - // For every leaf on the last level, we have tau leaf blocks - leaf_count += this->tau_; - // Iterate through all characters in this child and - // add them to the leaf string - for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); - b++) { - if (static_cast(block_start + b) < text.size()) { - this->leaves_.push_back(text[block_start + b]); - } else { - this->leaves_.push_back(0); + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); } - this->amount_of_leaves = leaf_count; - this->compress_leaves(); - } - - /// @brief Generates a level and adds the relevant data to the block tree. - /// - /// @param levels The vector of levels of the tree. - /// @param level_index The index of the level to generate. This must be - /// strictly greater than 0. - /// @param is_padded Whether there is padding in the last block of the tree - void make_tree_level(std::vector& levels, - const std::vector& new_num_internal, - const size_t level_index, - const bool is_padded, - const size_t text_len) { - LevelData& previous_level = levels[level_index - 1]; - LevelData& level = levels[level_index]; - - size_type new_size = - (new_num_internal[level_index - 1] - is_padded) * this->tau_; - // Determine the number of children the last block generated - if (is_padded) { - const size_type last_block_parent_start = - previous_level.block_starts->back(); - const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); - } - previous_level.block_starts.reset(); - const size_type num_internal = new_num_internal[level_index]; - - // Allocate new vectors for the tree - auto* is_internal = new BitVector(new_size); - auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); - auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); - - // Number of non-pruned blocks before the current block - size_type num_non_pruned = 0; - // Number of back blocks before the current block - size_type num_back_blocks = 0; - // Number of pruned blocks before the current block - size_type num_pruned = 0; - - // We will reuse the allocated memory of the pointers vector to - // store the number of pruned blocks before the block. The - // invariant is that all values up to i are overwritten while all - // values starting after i will still be valid pointers - // This contains the number of pruned blocks before the block i - std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { - const size_type ptr = (*level.pointers)[i]; - prefix_pruned_blocks[i] = num_pruned; - - // If the current block is not pruned, add it to the new tree - if (ptr == internal::sharded::PRUNED) { - num_pruned++; - continue; + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector & levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); } + } - // Add it to the is_internal bit vector - const bool block_is_internal = (*level.is_internal)[i]; - (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector & levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; - if (block_is_internal) { - continue; + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; } - // If it is a back block, add its pointer and offset - const size_type offset = (*level.offsets)[i]; + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; - (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; - (*offsets)[num_back_blocks] = offset; - ++num_back_blocks; - } + bool has_internal_children = false; - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(is_internal); - this->block_tree_types_rs_.push_back(new Rank(*is_internal)); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(level.block_size); - } - - /// @brief Prunes the tree of unnecessary nodes. - /// @param levels The levels of the tre represented as a vector of levels. - void prune(std::vector& levels) { - // We need to traverse the block tree in post order, - // handling children from right to left - for (int block_index = levels[0].num_blocks - 1; block_index >= 0; - --block_index) { - prune_block(levels, 0, block_index); - } - } - - /// @brief Prunes a block and its descendants of unnecessary internal nodes. - /// @param levels The WIP levels of the tree. - /// @param level_index The level of the block to prune. - /// @param block_index The index of the block to prune. - /// @return Whether this block is/stays internal after the pruning process - bool prune_block(std::vector& levels, - const size_t level_index, - const size_t block_index) const { - LevelData& level = levels[level_index]; - BitVector& is_internal = *level.is_internal; - - // If the current block is a back block already, there is nothing - // to prune - if (!is_internal[block_index]) { - return false; - } + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = std::min( + first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } - const size_type first_child = - level.is_internal_rank->rank1(block_index) * this->tau_; + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } - bool has_internal_children = false; + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; - // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are - // not on the last level. - if (level_index < levels.size() - 1) { + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; const size_type last_child = std::min(first_child + this->tau_ - 1, - levels[level_index + 1].is_internal->size() - 1); - // Iterate through children in reverse + child_level.is_internal->size() - 1); for (size_type child = last_child; child >= first_child; --child) { - has_internal_children |= prune_block(levels, level_index + 1, child); + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; } - } - - // If any of the children is internal, this block stays internal - // as well - if (has_internal_children) { - return true; - } - const size_type pointer = (*level.pointers)[block_index]; - const size_type offset = (*level.offsets)[block_index]; - const size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing - // to this, then this must stay internal - if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { - return true; - } - - // Now we know that there is an earlier occurrence, - // and nothing is pointing here. - // We will make this block here into a back block... - is_internal[block_index] = false; - (*level.counters)[pointer] += 1; - (*level.counters)[pointer + 1] += offset > 0; - - if (level_index == levels.size() - 1) { return false; } - // ...and mark the children as pruned - LevelData& child_level = levels[level_index + 1]; - const size_type last_child = - std::min(first_child + this->tau_ - 1, - child_level.is_internal->size() - 1); - for (size_type child = last_child; child >= first_child; --child) { - const size_type child_pointer = (*child_level.pointers)[child]; - const size_type child_offset = (*child_level.offsets)[child]; -#ifdef BT_DBG - if (!(*child_level.is_internal)[child] && child_pointer < 0) { - std::cout << "non-internal node missing pointer" << std::endl; - std::cout << level_index << ", " << block_index << " / " - << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { - std::cout << "pruned node missing pointer" << std::endl; - } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - BT_ASSERT(child_pointer >= 0); -#endif - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child] = internal::sharded::PRUNED; + public: + BlockTreeSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); } - return false; - } - -public: - BlockTreeSharded(const std::vector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads, - const size_t queue_size) { - const auto old = omp_get_max_threads(); - const auto old_dynamic = omp_get_dynamic(); - omp_set_dynamic(0); - omp_set_num_threads(static_cast(threads)); - this->tau_ = arity; - this->s_ = root_arity; - this->max_leaf_length_ = max_leaf_length; - this->map_unique_chars(text); - construct(text, threads, queue_size); - omp_set_dynamic(old_dynamic); - omp_set_num_threads(old); - } - - ~BlockTreeSharded() { - for (auto& rank : this->block_tree_types_rs_) { - delete rank; - } - for (auto& bv : this->block_tree_types_) { - delete bv; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; + ~BlockTreeSharded() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } } - } -}; + }; } // namespace pasta diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp index 8e6ff73..d377eae 100644 --- a/include/pasta/block_tree/utils/sharded_map.hpp +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -30,42 +30,6 @@ namespace pasta { -/// -/// @brief An update function which on update just overwrites the value. -/// -/// @tparam K The key type saved in the hash map. -/// @tparam V The value type saved in the hash map. -/// -template -struct Overwrite { - using InputValue = V; - - inline static void update(K&, V& value, V&& input_value) { - value = input_value; - } - - inline static V init(K&, V&& input_value) { - return input_value; - } -}; - -/// -/// @brief An update function which upon update does nothing besides -/// inserting the value if it doesn't exist. -/// -/// @tparam K The key type saved in the hash map. -/// @tparam V The value type saved in the hash map. -/// -template -struct Keep { - using InputValue = V; - inline static void update(K&, V&, V&&) {} - - inline static V init(K&, V&& input_value) { - return input_value; - } -}; - /// @brief A hash map that must be used by multiple threads, each thread having /// only having write access to a certain segment of the input space. /// @tparam K The type of the keys in the hash map. @@ -169,8 +133,8 @@ class ShardedMap { auto res = map_.find(k); if (res == map_.end()) { // If the value does not exist, insert it - V initial = UpdateFn::init(k, std::move(in_value)); K key = k; + V initial = UpdateFn::init(key, std::move(in_value)); map_.emplace(key, std::move(initial)); } else { // Otherwise, update it. diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp index 9436819..e1b9521 100644 --- a/include/pasta/block_tree/utils/sharded_util.hpp +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -11,7 +11,9 @@ #include /// @brief Utilities for the construction algorithms using the sharded hash map -namespace pasta::internal::sharded { +namespace pasta { + +namespace internal::sharded { __extension__ typedef unsigned __int128 uint128_t; @@ -285,6 +287,10 @@ struct UpdateBlockOccurrences { } }; +/// @brief A mixing functions to provide better avalanching to intermediate hash +/// values. +/// +/// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html constexpr uint64_t mix_select(uint64_t key) { key ^= (key >> 31); key *= 0x7fb5d329728ea185; @@ -301,4 +307,43 @@ size_t ceil_div(std::integral auto x, std::integral auto y) { return 1 + (static_cast(x) - 1) / static_cast(y); } -} // namespace pasta::internal::sharded \ No newline at end of file +} // namespace internal::sharded + +/// +/// @brief An update function for sharded maps which on update just overwrites +/// the value. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Overwrite { + using InputValue = V; + + inline static void update(K&, V& value, V&& input_value) { + value = input_value; + } + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// +/// @brief An update function for sharded maps which upon update does nothing +/// besides inserting the value if it doesn't exist. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Keep { + using InputValue = V; + inline static void update(K&, V&, V&&) {} + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 9b225d3..735b876 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -19,8 +19,10 @@ ******************************************************************************/ #pragma once +#include "sharded_util.hpp" + +#include #include -#include #include #include #include @@ -34,41 +36,6 @@ namespace pasta { enum Whereabouts { NOWHERE, IN_MAP, IN_QUEUE }; -/// -/// @brief An update function which on update just overwrites the value. -/// -/// @tparam K The key type saved in the hash map. -/// @tparam V The value type saved in the hash map. -template -struct [[maybe_unused]] Overwrite { - using InputValue [[maybe_unused]] = V; - - inline static void update(const K&, V& value, V&& input_value) { - value = input_value; - } - - inline static V init(const K&, V&& input_value) { - return input_value; - } -}; - -/// -/// @brief An update function which upon update does nothing besides -/// inserting the value if it doesn't exist. -/// -/// @tparam K The key type saved in the hash map. -/// @tparam V The value type saved in the hash map. -/// -template -struct [[maybe_unused]] Keep { - using InputValue [[maybe_unused]] = V; - inline static void update(const K&, V&, V&&) {} - - inline static V init(const K&, V&& input_value) { - return input_value; - } -}; - /// @brief A hash map that must be used by multiple threads, each thread having /// only having write access to a certain segment of the input space. /// @tparam K The type of the keys in the hash map. @@ -81,7 +48,7 @@ template typename SeqHashMapType = std::unordered_map, - UpdateFunction UpdateFn = Overwrite> + UpdateFunction UpdateFn = pasta::Overwrite> requires std::movable class SyncShardedMap { /// The sequential backing hash map type @@ -128,16 +95,6 @@ class SyncShardedMap { std::barrier barrier_; - /// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html - [[nodiscard]] uint64_t mix_select(uint64_t key) const { - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); - return key % thread_count_; - } - public: std::atomic_size_t num_updates_; std::atomic_size_t num_inserts_; @@ -299,7 +256,8 @@ class SyncShardedMap { handle_queue_sync(); } const size_t hash = Hasher{}(pair.first); - const size_t target_thread_id = sharded_map_.mix_select(hash); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % sharded_map_.thread_count_; // Otherwise enqueue the new value in the target thread std::atomic_size_t& target_task_count = @@ -375,7 +333,8 @@ class SyncShardedMap { [[maybe_unused]] Whereabouts where(const K& k) { const size_t hash = Hasher{}(k); - const size_t target_thread_id = mix_select(hash); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % thread_count_; SeqHashMap& map = map_[target_thread_id]; typename SeqHashMap::iterator it = map.find(k); if (it != map.end()) { @@ -409,7 +368,8 @@ class SyncShardedMap { typename SeqHashMap::iterator find(const K& key) { const size_t hash = Hasher{}(key); - const size_t target_thread_id = mix_select(hash); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % thread_count_; SeqHashMap& map = map_[target_thread_id]; typename SeqHashMap::iterator it = map.find(key); if (it == map.end()) { From f81695d05e072fe7025996782ba7ea3fa87a51c7 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 15 Dec 2023 01:08:15 +0100 Subject: [PATCH 74/92] make sync shard respect max leaf length --- examples/build_bt.cpp | 2 +- .../block_tree_fp_par_sync_sharded.hpp | 288 ++++-------------- 2 files changed, 53 insertions(+), 237 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 293f9ee..24a496b 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,7 @@ #include #include -#define PAR_SHARDED +#define PAR_SHARDED_SYNC #define REC_BIT #if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp index 21a62c8..c75731d 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -24,14 +24,14 @@ #include "pasta/block_tree/block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" +#include #include -#include #include #include #include -#include #include #include #include @@ -51,23 +51,6 @@ class BlockTreeFPParShardedSync : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - // constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); - // constexpr static uint128_t PRIME = (static_cast(0x97009E545BB) - // << (14 * 4)) | static_cast(0x2DA8B4A8C9A82B); - /// @brief A bit vector using BitVector = pasta::BitVector; /// @brief A rank data structure for a bit vector @@ -76,12 +59,14 @@ class BlockTreeFPParShardedSync : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_node_map>; + ankerl::unordered_dense::map>; // std::unordered_map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; @@ -91,6 +76,18 @@ class BlockTreeFPParShardedSync : public BlockTree { using RabinKarpMap = SyncShardedMap; + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + #ifdef BT_INSTRUMENT public: size_t bp_hash_pairs_ns = 0; @@ -104,209 +101,6 @@ class BlockTreeFPParShardedSync : public BlockTree { #endif private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - size_type block_size; - /// @brief The index of the current level. First level is 0, second level is - /// 1 etc. - size_type level_index; - /// @brief The number of blocks on the current level - size_type num_blocks; - - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - [[gnu::noinline]] void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - [[gnu::noinline]] inline void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - [[gnu::noinline]] inline void update(size_type block_index, - size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - /// @brief Constructs the block tree. /// @param text The input text. void construct(const std::vector& text, @@ -389,7 +183,8 @@ class BlockTreeFPParShardedSync : public BlockTree { #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); } #ifdef BT_INSTRUMENT @@ -545,7 +340,11 @@ class BlockTreeFPParShardedSync : public BlockTree { const auto end = std::min(num_block_pairs, (thread_id + 1) * segment_size); - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { // If the next block is not adjacent, we cannot hash the pair // starting at the current block @@ -592,7 +391,11 @@ class BlockTreeFPParShardedSync : public BlockTree { #endif if (start < static_cast(num_block_pairs)) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -755,8 +558,9 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t queue_size) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -816,7 +620,11 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_t end = std::min(num_total_iterations, (thread_id + 1) * segment_size); - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); // Hash each block and store their hashes in the map for (size_t i = start; i < end; ++i) { rk.restart(block_starts[i]); @@ -857,7 +665,11 @@ class BlockTreeFPParShardedSync : public BlockTree { // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { continue; @@ -1128,6 +940,8 @@ class BlockTreeFPParShardedSync : public BlockTree { b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } @@ -1184,7 +998,7 @@ class BlockTreeFPParShardedSync : public BlockTree { prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -1272,7 +1086,7 @@ class BlockTreeFPParShardedSync : public BlockTree { const size_type counter = (*level.counters)[block_index]; // If there is no earlier occurrence or there are blocks pointing // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -1300,17 +1114,19 @@ class BlockTreeFPParShardedSync : public BlockTree { std::cout << "non-internal node missing pointer" << std::endl; std::cout << level_index << ", " << block_index << " / " << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); BT_ASSERT(child_pointer >= 0); #endif // Decrement the counter of where the child points (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; From e64d10c1cd32f7376180e804b69b7a0de3930290 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 20 Dec 2023 02:17:27 +0100 Subject: [PATCH 75/92] fix a bug in dense bbt generating way too large leaf bit string --- examples/build_bt.cpp | 12 +- .../block_tree_fp_par_sharded.hpp | 331 +++++------------- .../rec_dense_bit_block_tree_sharded.hpp | 34 +- .../block_tree/rec_dense_bit_block_tree.hpp | 8 +- 4 files changed, 124 insertions(+), 261 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 24a496b..9513aee 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -25,7 +25,7 @@ #include #define PAR_SHARDED_SYNC -#define REC_BIT +#define REC_DENSE_BIT #if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED constexpr size_t RECURSION_LEVELS = 0; @@ -334,6 +334,16 @@ int main(int argc, char** argv) { */ auto bt = std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); + +#ifdef BT_DBG + size_t cnt = 0; + for (const auto &b : *bt->leaf_bits_) { + if (b) { + cnt++; + } + } + std::cout << "num ones: " << cnt << "/" << bt->leaf_bits_->size() << " (" << static_cast(cnt) * 100 / bt->leaf_bits_->size() << "%)" << std::endl; +#endif auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp index 81c0509..c3539ab 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -20,23 +20,24 @@ #pragma once -#include "data-structures/hash_table_mods.hpp" #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" #include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" #include "pasta/block_tree/utils/sharded_map.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include #include #include +#include #include #include #include #include #include -#include #include #include @@ -60,18 +61,6 @@ class BlockTreeFPParSharded : public BlockTree { using Clock = std::chrono::high_resolution_clock; using TimePoint = Clock::time_point; - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = 2305843009213693951ULL; - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A bit vector using BitVector = pasta::BitVector; /// @brief A rank data structure for a bit vector @@ -86,11 +75,13 @@ class BlockTreeFPParSharded : public BlockTree { /// @brief A sequential hash map used as backing for the sharded hash map. template using SeqHashMap = - robin_hood::unordered_node_map>; + ankerl::unordered_dense::map>; /// @brief A rabin karp hasher preconfigured for the current template /// parameters - using RabinKarp = MersenneRabinKarp; + using RabinKarp = MersenneRabinKarp; /// @brief A rabin karp hash for the preconfigured rabin karp hasher using RabinKarpHash = MersenneHash; @@ -100,6 +91,19 @@ class BlockTreeFPParSharded : public BlockTree { using RabinKarpMap = ShardedMap; + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + #ifdef BT_DBG public: size_t bp_hash_pairs_ns = 0; @@ -113,207 +117,6 @@ class BlockTreeFPParSharded : public BlockTree { private: #endif - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - size_type block_size; - /// @brief The index of the current level. First level is 0, second level is - /// 1 etc. - size_type level_index; - /// @brief The number of blocks on the current level - size_type num_blocks; - - inline LevelData(size_type level_index_, - size_type block_size_, - size_type num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] inline bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + static_cast(block_size) == - (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - inline void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - inline void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - inline void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_index The offset of that occurrence - inline void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; /// @brief Constructs the block tree. /// @param text The input text. @@ -364,9 +167,7 @@ class BlockTreeFPParSharded : public BlockTree { now = Clock::now(); #endif -#ifdef BT_DBG scan_blocks(text, current, is_padded, threads); -#endif #ifdef BT_DBG blocks_ns += std::chrono::duration_cast( @@ -376,14 +177,17 @@ class BlockTreeFPParSharded : public BlockTree { #endif // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { levels.push_back(std::move(generate_next_level(text, current))); - } #ifdef BT_DBG - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); #endif + } else { + break; + } } #ifdef BT_DBG TimePoint now = Clock::now(); @@ -484,7 +288,11 @@ class BlockTreeFPParSharded : public BlockTree { continue; } // Move the hasher to the current block pair - RabinKarp rk(text, SIGMA, block_starts[i], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[i], + pair_size, + internal::sharded::PRIME); RabinKarpHash hash = rk.current_hash(); // Try to find the hash in the map, insert a new entry if it doesn't // exist, and add the current block to the entry @@ -513,7 +321,11 @@ class BlockTreeFPParSharded : public BlockTree { #endif if (start < static_cast(num_block_pairs)) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { continue; @@ -623,8 +435,12 @@ class BlockTreeFPParSharded : public BlockTree { const size_t threads) { const size_t num_blocks = level_data.num_blocks; - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + std::cout << "\n pointers have " << level_data.pointers->size() + << std::endl; + ; level_data.offsets = std::make_unique>(num_blocks, 0); level_data.counters = @@ -664,7 +480,11 @@ class BlockTreeFPParSharded : public BlockTree { // Hash each block and store their hashes in the map for (size_t i = start; i < end; ++i) { - const RabinKarp rk(text, SIGMA, block_starts[i], block_size, PRIME); + const RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[i], + block_size, + internal::sharded::PRIME); RabinKarpHash hash = rk.current_hash(); shard.insert(hash, {i, 0}); // The thread checks whether it should handle the inserts in its queue @@ -692,7 +512,11 @@ class BlockTreeFPParSharded : public BlockTree { // Hash every window and find the first occurrences for every block. if (start < block_starts.size() - is_padded) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { if (!level_data.next_is_adjacent(i)) { continue; @@ -868,7 +692,9 @@ class BlockTreeFPParSharded : public BlockTree { found_back_block |= static_cast(new_num_internal[level_index]) < levels[level_index].is_internal->size(); if (!found_back_block) { - level.is_internal.reset(); + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); @@ -912,6 +738,8 @@ class BlockTreeFPParSharded : public BlockTree { b++) { if (static_cast(block_start + b) < text.size()) { this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); } } } @@ -968,7 +796,7 @@ class BlockTreeFPParSharded : public BlockTree { prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { + if (ptr == internal::sharded::PRUNED) { num_pruned++; continue; } @@ -1017,12 +845,12 @@ class BlockTreeFPParSharded : public BlockTree { /// @return Whether this block is/stays internal after the pruning process bool prune_block(std::vector& levels, const size_t level_index, - const size_t block_index_) { - volatile size_t block_index = block_index_; + const size_t block_index) const { LevelData& level = levels[level_index]; BitVector& is_internal = *level.is_internal; - // If the current block is a back block already, there is nothing to prune + // If the current block is a back block already, there is nothing + // to prune if (!is_internal[block_index]) { return false; } @@ -1033,10 +861,10 @@ class BlockTreeFPParSharded : public BlockTree { bool has_internal_children = false; // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are not on the - // last level. + // none of which can be pointed to. So only recurse, if we are + // not on the last level. if (level_index < levels.size() - 1) { - volatile size_type last_child = + const size_type last_child = std::min(first_child + this->tau_ - 1, levels[level_index + 1].is_internal->size() - 1); // Iterate through children in reverse @@ -1045,17 +873,19 @@ class BlockTreeFPParSharded : public BlockTree { } } - // If any of the children is internal, this block stays internal as well + // If any of the children is internal, this block stays internal + // as well if (has_internal_children) { return true; } - volatile size_type pointer = (*level.pointers)[block_index]; - volatile size_type offset = (*level.offsets)[block_index]; - volatile size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing to this, - // then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { + std::cout << "\n" << level.pointers->size() << std::endl; + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { return true; } @@ -1081,18 +911,19 @@ class BlockTreeFPParSharded : public BlockTree { #ifdef BT_DBG if (!(*child_level.is_internal)[child] && child_pointer < 0) { std::cout << "non-internal node missing pointer" << std::endl; - std::cout << level_index << ", " << block_index << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; } else if (child_pointer == PRUNED && child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); #endif - assert(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - assert(child_pointer >= 0); // Decrement the counter of where the child points (*child_level.counters)[child_pointer] -= 1; (*child_level.counters)[child_pointer + 1] -= child_offset > 0; // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; + (*child_level.pointers)[child] = internal::sharded::PRUNED; } return false; diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index fabe619..3bf5335 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -31,6 +31,7 @@ #include #include #include +#include #include #include #include @@ -1134,17 +1135,34 @@ class RecursiveDenseBitBlockTreeSharded auto& last_is_internal = *levels.back().is_internal; std::vector& last_block_starts = *levels.back().block_starts; size_t bit_index = 0; - const size_t final_num_internals = - levels.back().is_internal_rank->rank1(last_is_internal.size()); + size_t final_num_internals = 0; + // the rank DS is broken so we count manually + // levels.back().is_internal_rank->rank1(last_is_internal.size()); + for (const bool b : last_is_internal) { + if (b) { + final_num_internals++; + } + } + this->leaf_bits_ = std::make_unique( final_num_internals * this->leaf_size * this->tau_, false); +#ifdef BT_DBG + std::cout << "last size: " << last_is_internal.size() << std::endl; + std::cout << "tau: " << this->tau_ << "\nleaf_size: " << this->leaf_size + << "\nfinal_num_internals: " << final_num_internals + << "\nleaf creation size: " + << final_num_internals * this->leaf_size * this->tau_ + << std::endl; + size_t padded = 0; + size_t non_padded = 0; +#endif for (size_t block = 0; block < last_is_internal.size(); block++) { if (!last_is_internal[block]) { continue; } const size_type block_start = last_block_starts[block]; - // For every leaf on the las#elift level, we have tau leaf blocks + // For every leaf on the last level, we have tau leaf blocks leaf_count += this->tau_; // Iterate through all characters in this child and // add them to the leaf string @@ -1153,11 +1171,21 @@ class RecursiveDenseBitBlockTreeSharded if (static_cast(block_start + b) < text.size()) { (*this->leaf_bits_)[bit_index++] = static_cast(text[block_start + b]); +#ifdef BT_DBG + non_padded++; +#endif } else { (*this->leaf_bits_)[bit_index++] = false; +#ifdef BT_DBG + padded++; +#endif } } } +#ifdef BT_DBG + std::cout << "padded: " << padded << std::endl; + std::cout << "non_padded: " << non_padded << std::endl; +#endif if constexpr (recursion_level == 0) { if (levels.size() == 1) { top_level.is_internal.release(); diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp index bac918a..d02486f 100644 --- a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -21,7 +21,7 @@ #pragma once -#include +#include #include #include #include @@ -70,12 +70,8 @@ class RecursiveDenseBitBlockTree { std::vector*> block_tree_offsets_; // std::vector*> block_tree_encoded_; std::vector block_size_lvl_; - std::vector block_per_lvl_; std::vector leaves_; - std::vector compress_map_; - std::vector decompress_map_; - sdsl::int_vector<> compressed_leaves_; std::unique_ptr leaf_bits_; /// @brief For each level and each block, contains the number of 1s up to (and @@ -505,8 +501,6 @@ class RecursiveDenseBitBlockTree { #endif space_usage += block_size_lvl_.size() * sizeof(typename decltype(block_size_lvl_)::value_type); - space_usage += block_per_lvl_.size() * - sizeof(typename decltype(block_per_lvl_)::value_type); if (rank_support) { for (auto& rs : one_ranks_) { From ce8637dc61f99e49cab2093646f9b53b1c339360 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 23 Dec 2023 02:21:15 +0100 Subject: [PATCH 76/92] fix bug in bit mersenne hash comparisons --- examples/build_bt.cpp | 12 ++- .../block_tree/construction/block_tree_fp.hpp | 2 +- .../construction/rec_block_tree_sharded.hpp | 3 +- .../rec_dense_bit_block_tree_sharded.hpp | 2 +- .../pasta/block_tree/rec_bit_block_tree.hpp | 3 + include/pasta/block_tree/rec_block_tree.hpp | 13 +++ .../block_tree/rec_dense_bit_block_tree.hpp | 15 ++++ .../pasta/block_tree/utils/MersenneHash.hpp | 31 +++---- tests/utils/bit_rabin_karp_test.cpp | 90 ++++++++----------- 9 files changed, 95 insertions(+), 76 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 9513aee..5cf3393 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,8 @@ #include #include -#define PAR_SHARDED_SYNC +#define BT_DBG +#define REC_PAR_SHARDED #define REC_DENSE_BIT #if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED @@ -336,18 +337,23 @@ int main(int argc, char** argv) { std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); #ifdef BT_DBG +/* size_t cnt = 0; - for (const auto &b : *bt->leaf_bits_) { + for (const auto& b : *bt->leaf_bits_) { if (b) { cnt++; } } - std::cout << "num ones: " << cnt << "/" << bt->leaf_bits_->size() << " (" << static_cast(cnt) * 100 / bt->leaf_bits_->size() << "%)" << std::endl; + std::cout << "num ones: " << cnt << "/" << bt->leaf_bits_->size() << " (" + << static_cast(cnt) * 100 / bt->leaf_bits_->size() << "%)" + << std::endl; + */ #endif auto elapsed = std::chrono::duration_cast( Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); + std::cout << "\n"; bt->add_bit_rank_support(); auto elapsed_rs = std::chrono::duration_cast( Clock::now() - now) diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 9b8a4ae..d5fd43c 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -20,7 +20,7 @@ #pragma once -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index f6d6091..7624b06 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -1395,7 +1395,8 @@ class RecursiveBlockTreeSharded child_pointer < 0) { std::cout << "pruned node missing pointer" << std::endl; } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); BT_ASSERT(child_pointer >= 0); #endif // Decrement the counter of where the child points diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index 3bf5335..80c2af6 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -1152,7 +1152,7 @@ class RecursiveDenseBitBlockTreeSharded std::cout << "tau: " << this->tau_ << "\nleaf_size: " << this->leaf_size << "\nfinal_num_internals: " << final_num_internals << "\nleaf creation size: " - << final_num_internals * this->leaf_size * this->tau_ + << final_num_internals * this->leaf_size * this->tau_ / 8 << std::endl; size_t padded = 0; size_t non_padded = 0; diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index 621c70a..e2df33a 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -570,6 +570,9 @@ class RecursiveBitBlockTree { // space_usage += leaves_.size() * sizeof(uint8_t); space_usage += sdsl::size_in_bytes(compressed_leaves_); +#ifdef BT_DBG + std::cout << "leaf size: " << sdsl::size_in_bytes(compressed_leaves_) << std::endl; +#endif space_usage += compress_map_.size(); return space_usage; diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index b00eb90..f683126 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -29,6 +29,7 @@ #include #include #include +#include #include namespace pasta { @@ -419,12 +420,24 @@ class RecursiveBlockTree { delta_size = 0; #endif } + + size_t ptr_cnt = 0; + (void)ptr_cnt; + size_t level = 0; + (void)level; for (const auto iv : block_tree_pointers_) { space_usage += (int64_t)sdsl::size_in_bytes(*iv); delta_size += (int64_t)sdsl::size_in_bytes(*iv); +#ifdef BT_DBG + ptr_cnt += iv->size(); + std::cout << "level " << level << " ptrs: " << iv->size() + << " block size: " << block_size_lvl_[level] << "\n"; + level++; +#endif } #ifdef BT_DBG std::cout << "ptrs size: " << delta_size << std::endl; + std::cout << "pointer count: " << ptr_cnt << std::endl; delta_size = 0; #endif diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp index d02486f..0a187e6 100644 --- a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -21,12 +21,17 @@ #pragma once +#include #include +#include +#include #include #include #include #include +#include #include +#include #include #include @@ -488,16 +493,26 @@ class RecursiveDenseBitBlockTree { delta_size += (int64_t)sdsl::size_in_bytes(*iv); ; } + size_t ptr_cnt = 0; + (void)ptr_cnt; #ifdef BT_DBG std::cout << "ptrs size: " << delta_size << std::endl; delta_size = 0; + size_t level = 0; #endif for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); delta_size += (int64_t)sdsl::size_in_bytes(*iv); +#ifdef BT_DBG + ptr_cnt += iv->size(); + std::cout << "level " << level << " ptrs: " << iv->size() + << " block size: " << block_size_lvl_[level] << "\n"; + level++; +#endif } #ifdef BT_DBG std::cout << "offs size: " << delta_size << std::endl; + std::cout << "pounter count: " << ptr_cnt << std::endl; #endif space_usage += block_size_lvl_.size() * sizeof(typename decltype(block_size_lvl_)::value_type); diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 0167c12..09e9230 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -21,6 +21,7 @@ #pragma once +#include #include #include #include @@ -150,6 +151,7 @@ class MersenneHash { ++mersenne_hash_comparisons; #endif if (hash_ != other.hash_) { + //std::cout << "hashes unequal" << std::endl; return false; } @@ -160,19 +162,19 @@ class MersenneHash { if (slice_at(*text_, start_ + pos) != slice_at(*other.text_, other.start_ + pos)) { is_same = false; - goto mersenne_hash_cmp_done; + break; } } - for (size_t i = pos; i < length_; ++i) { - if (get_bit(*text_, start_ + i) != - get_bit(*other.text_, other.start_ + i)) { - is_same = false; - goto mersenne_hash_cmp_done; + if (!is_same) { + for (size_t i = pos; i < length_; ++i) { + if ((*text_)[start_ + i] != (*other.text_)[other.start_ + i]) { + is_same = false; + break; + } } } - mersenne_hash_cmp_done: #ifdef BT_INSTRUMENT if (!is_same) { // The hash is the same but the substring isn't => collision @@ -185,15 +187,6 @@ class MersenneHash { return is_same; }; -private: - static bool get_bit(const pasta::BitVector& v, const size_t bit_index) { - return v[bit_index]; - } - - static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - static uint64_t slice_at(const pasta::BitVector& bv, const size_t i) { const std::span backing = bv.data(); const uint8_t offset = i % 64; @@ -201,10 +194,10 @@ class MersenneHash { // TODO Check if the right shift actually shifts in zeros const uint64_t r = backing[data_index] & (~static_cast(0) << offset); - if (offset == 0) { + if (offset > 0) { const uint64_t l = backing[data_index + 1] & - (~static_cast(0) >> (63 - offset)); - return (l << (63 - offset)) | (r >> offset); + (~static_cast(0) >> (64 - offset)); + return (l << (64 - offset)) | (r >> offset); } return r; } diff --git a/tests/utils/bit_rabin_karp_test.cpp b/tests/utils/bit_rabin_karp_test.cpp index 56f81f3..d7a3061 100644 --- a/tests/utils/bit_rabin_karp_test.cpp +++ b/tests/utils/bit_rabin_karp_test.cpp @@ -1,34 +1,34 @@ -#include -#include -#include -#define asdasdkasld +#include "pasta/block_tree/utils/MersenneHash.hpp" #include +#include +#include #include #include +#include class BitRabinKarpTest : public ::testing::Test { protected: + + static constexpr size_t unique_len = 100000; pasta::BitVector bv; void SetUp() override { std::random_device rd; std::mt19937 gen{rd()}; - std::uniform_int_distribution dist(1); - - constexpr size_t string_length = 100000; - bv.resize(string_length * 2 + 3); - for (size_t i = 0; i < string_length; ++i) { - bv[i] = dist(gen); + std::uniform_int_distribution dist(10); + bv.resize(unique_len * 2 + 3); + for (size_t i = 0; i < unique_len; ++i) { + bv[i] = dist(gen) % 2 == 0; } // Offset it by some amount to check that even misaligned bit sequences // correctly match - bv[string_length] = false; - bv[string_length + 1] = false; - bv[string_length + 2] = false; - for (size_t i = 0; i < string_length; ++i) { - bv[string_length + 3 + i] = static_cast(bv[i]); + bv[unique_len] = false; + bv[unique_len + 1] = false; + bv[unique_len + 2] = false; + for (size_t i = 0; i < unique_len; ++i) { + bv[unique_len + 3 + i] = static_cast(bv[i]); } } @@ -36,7 +36,7 @@ class BitRabinKarpTest : public ::testing::Test { bool compare_ranges(const size_t s1, const size_t s2, const size_t len) const { for (size_t i = 0; i < len; ++i) { - if (get_bit(s1 + i) == get_bit(s2 + i)) { + if (get_bit(s1 + i) != get_bit(s2 + i)) { return false; } } @@ -48,32 +48,35 @@ class BitRabinKarpTest : public ::testing::Test { } }; +TEST_F(BitRabinKarpTest, test_slice) { + for (size_t i = 0; i < bv.size() - 1; i++) { + uint64_t manual_slice = 0; + for (size_t j = 0; j < 64; j++) { + manual_slice |= (static_cast(bv[i + j]) << j); + } + + uint64_t slice = pasta::MersenneHash::slice_at(bv, i); + ASSERT_EQ(manual_slice, slice) + << " bit index " << i + << "\nwith manual slice: " << std::bitset<64>(manual_slice) + << "\nwith hash slice: " << std::bitset<64>(slice); + } +} + TEST_F(BitRabinKarpTest, test_eq) { constexpr std::array sizes = {13, 24, 59, 1220}; for (const size_t size : sizes) { - const size_t half_len = bv.size() / 2; pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); pasta::MersenneRabinKarp rk2(bv, - half_len + 3, + unique_len + 3, size, (1ULL << 61) - 1); - for (size_t i = 0; i < half_len - size; ++i) { + for (size_t i = 0; i < unique_len - size; ++i) { const auto h1 = rk1.current_hash(); const auto h2 = rk2.current_hash(); - if (h1 != h2) { - std::cerr << "h1: "; - for (const auto byte : h1.overlapping_range()) { - std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; - } - std::cerr << "\n"; - std::cerr << "h2: "; - for (const auto byte : h2.overlapping_range()) { - std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; - } - std::cerr << "\n"; - } - ASSERT_TRUE(h1 == h2); + ASSERT_TRUE(h1 == h2) << "offset: " << i << " for half len " << unique_len + << " and size " << size; rk1.next(); rk2.next(); } @@ -84,32 +87,17 @@ TEST_F(BitRabinKarpTest, test_rnd) { constexpr std::array sizes = {13, 24, 59, 1220}; for (const size_t size : sizes) { - const size_t half_len = bv.size() / 2; pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); pasta::MersenneRabinKarp rk2(bv, - half_len, + unique_len, size, (1ULL << 61) - 1); size_t offset = 0; - for (size_t i = 0; i < half_len - size; ++i) { + for (size_t i = 0; i < unique_len - size; ++i) { const auto h1 = rk1.current_hash(); const auto h2 = rk2.current_hash(); - const bool success = - (h1 == h2) == compare_ranges(offset, half_len + offset, size); - if (!success) { - std::cerr << "h1: "; - for (const auto byte : h1.overlapping_range()) { - std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; - } - std::cerr << " offset: " << h1.start_ % 8 << "\n"; - std::cerr << "h2: "; - for (const auto byte : h2.overlapping_range()) { - std::cerr << std::bitset<8>{std::to_integer(byte)} << ", "; - } - std::cerr << " offset: " << h2.start_ % 8 << "\n"; - } - ASSERT_TRUE((h1 == h2) == compare_ranges(offset, half_len + offset, size)) - << "error at offset " << offset << " for half_len " << half_len + ASSERT_EQ((h1 == h2), compare_ranges(offset, unique_len + offset, size)) + << "error at offset " << offset << " for half_len " << unique_len << " and window size " << size; rk1.next(); rk2.next(); From 3903f752d3e7ded3a35ab1a30dbd31fe7dc11a8a Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 23 Dec 2023 02:48:51 +0100 Subject: [PATCH 77/92] make recursive block tree use dense bit block tree --- examples/build_bt.cpp | 1 - .../block_tree/construction/rec_block_tree_sharded.hpp | 6 +++--- include/pasta/block_tree/rec_block_tree.hpp | 6 +++--- 3 files changed, 6 insertions(+), 7 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 5cf3393..2da928c 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -24,7 +24,6 @@ #include #include -#define BT_DBG #define REC_PAR_SHARDED #define REC_DENSE_BIT diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index 7624b06..a0cfff9 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -21,7 +21,7 @@ #pragma once #include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp" +#include "pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp" #include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" @@ -1117,7 +1117,7 @@ class RecursiveBlockTreeSharded if constexpr (recursion_level > 0) { auto* bt = - new RecursiveBitBlockTreeSharded( + new RecursiveDenseBitBlockTreeSharded( *top_level.is_internal, this->tau_, this->s_, @@ -1285,7 +1285,7 @@ class RecursiveBlockTreeSharded if constexpr (recursion_level > 0) { auto* bt = - new RecursiveBitBlockTreeSharded( + new RecursiveDenseBitBlockTreeSharded( *is_internal, this->tau_, this->s_, diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp index f683126..10b5ac1 100644 --- a/include/pasta/block_tree/rec_block_tree.hpp +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -27,7 +27,7 @@ #include #include #include -#include +#include #include #include #include @@ -41,11 +41,11 @@ class RecursiveBlockTree { constexpr static bool types_is_block_tree = recursion_level > 0; using IsInternalType = std::conditional_t, + RecursiveDenseBitBlockTree, pasta::BitVector>; using IsInternalRankType = std::conditional_t, + RecursiveDenseBitBlockTree, pasta::RankSelect>; /// @brief If this is true, then the only levels of the tree start to be From f90bb70c7052a39fb1dfa8fd66e8d9daa89ff85c Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 23 Dec 2023 22:46:16 +0100 Subject: [PATCH 78/92] remove robin hood dependency --- .gitignore | 4 +- .gitmodules | 3 - CMakeLists.txt | 3 - extlib/robin-hood-hashing | 1 - .../construction/bit_block_tree_sharded.hpp | 1608 ----------------- .../block_tree/construction/block_tree_fp.hpp | 4 +- .../construction/block_tree_fp_par_parlay.hpp | 17 - .../construction/block_tree_sharded.hpp | 4 - .../rec_bit_block_tree_sharded.hpp | 1 - .../construction/rec_block_tree_sharded.hpp | 1 - .../pasta/block_tree/utils/MersenneHash.hpp | 2 - 11 files changed, 5 insertions(+), 1643 deletions(-) delete mode 160000 extlib/robin-hood-hashing delete mode 100644 include/pasta/block_tree/construction/bit_block_tree_sharded.hpp diff --git a/.gitignore b/.gitignore index 2c7cafb..93485af 100644 --- a/.gitignore +++ b/.gitignore @@ -5,6 +5,8 @@ compile_commands.json .idea/* perf.data* .pdf -bt_* *.txt +!CMakeLists.txt debug +dev/ +.vscode \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index 17c4003..10308d3 100644 --- a/.gitmodules +++ b/.gitmodules @@ -7,9 +7,6 @@ [submodule "extlib/tlx"] path = extlib/tlx url = https://github.com/tlx/tlx -[submodule "extlib/robin-hood-hashing"] - path = extlib/robin-hood-hashing - url = https://github.com/martinus/robin-hood-hashing [submodule "extlib/growt"] path = extlib/growt url = https://github.com/TooBiased/growt diff --git a/CMakeLists.txt b/CMakeLists.txt index 9bcbd18..c20d6bf 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -72,7 +72,6 @@ if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) if (PASTA_BLOCK_TREE_BENCH) target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) target_compile_definitions(build_bt PRIVATE BT_BENCH) - #target_compile_definitions(build_bt PRIVATE ROBIN_HOOD_LOG_ENABLED) endif () endif () @@ -94,7 +93,6 @@ set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/bit_vector) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/tlx) -add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/robin-hood-hashing) set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode") add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort) @@ -128,7 +126,6 @@ target_link_libraries(pasta_block_tree INTERFACE libsais pasta_bit_vector tlx - robin_hood waitfree-mpsc-queue sdsl #jiffy diff --git a/extlib/robin-hood-hashing b/extlib/robin-hood-hashing deleted file mode 160000 index 7697343..0000000 --- a/extlib/robin-hood-hashing +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 7697343363af4cc3f42cab17be49e6af9ab181e2 diff --git a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp deleted file mode 100644 index 3ed7cfc..0000000 --- a/include/pasta/block_tree/construction/bit_block_tree_sharded.hpp +++ /dev/null @@ -1,1608 +0,0 @@ -/******************************************************************************* - * This file is part of pasta::block_tree - * - * Copyright (C) 2023 Etienne Palanga - * - * pasta::block_tree is free software: you can redistribute it and/or modify - * it under the terms of the GNU General Public License as published by - * the Free Software Foundation, either version 3 of the License, or - * (at your option) any later version. - * - * pasta::block_tree is distributed in the hope that it will be useful, - * but WITHOUT ANY WARRANTY; without even the implied warranty of - * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the - * GNU General Public License for more details. - * - * You should have received a copy of the GNU General Public License - * along with pasta::block_tree. If not, see . - * - ******************************************************************************/ - -#pragma once - -#include "pasta/bit_vector/bit_vector.hpp" -#include "pasta/block_tree/rec_bit_block_tree.hpp" -#include "pasta/block_tree/utils/MersenneHash.hpp" -#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include "pasta/block_tree/utils/byteread.hpp" -#include "pasta/block_tree/utils/sync_sharded_map.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -__extension__ typedef unsigned __int128 uint128_t; - -namespace pasta { - -/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows -/// or just use the block's content itself as a hash, stored in an integer. -enum class UseHash { - /// @brief Use a Rabin-Karp hash - RABIN_KARP, - /// @brief Use the block's content as a hash - IDENTITY -}; - -/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes -/// and a sharded hash map. Small blocks are not RK-hashed but rather use the -/// blocks themselves. -/// @tparam size_type The type used for indices etc. (must be a signed integer) -/// in the sharded hash map. -template -class BitBlockTreeSharded : public BitBlockTree { - using Clock = std::chrono::high_resolution_clock; - using TimePoint = Clock::time_point; - - /// @brief For some block size (in bytes) i, return the number of trailing - /// zeros in a 64 bit integer when zeroing out characters that are not part - /// of the block. - constexpr static uint64_t MASK_TRAILING_ZEROS[9] = - {64, 56, 48, 40, 32, 24, 16, 8, 0}; - - /// @brief Masks used for the identity hash. These depend on endianness - constexpr static std::array masks() { - if constexpr (std::endian::native == std::endian::big) { - return {0, - static_cast(~0) << MASK_TRAILING_ZEROS[1], - static_cast(~0) << MASK_TRAILING_ZEROS[2], - static_cast(~0) << MASK_TRAILING_ZEROS[3], - static_cast(~0) << MASK_TRAILING_ZEROS[4], - static_cast(~0) << MASK_TRAILING_ZEROS[5], - static_cast(~0) << MASK_TRAILING_ZEROS[6], - static_cast(~0) << MASK_TRAILING_ZEROS[7], - static_cast(~0) << MASK_TRAILING_ZEROS[8]}; - } else { - return {0, - static_cast(~0) >> MASK_TRAILING_ZEROS[1], - static_cast(~0) >> MASK_TRAILING_ZEROS[2], - static_cast(~0) >> MASK_TRAILING_ZEROS[3], - static_cast(~0) >> MASK_TRAILING_ZEROS[4], - static_cast(~0) >> MASK_TRAILING_ZEROS[5], - static_cast(~0) >> MASK_TRAILING_ZEROS[6], - static_cast(~0) >> MASK_TRAILING_ZEROS[7], - static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; - } - } - - /// @brief Masks for identity hashes for a block size i (in bytes) - constexpr static std::array HASH_MASKS = masks(); - - /// @brief A marker for a block that has no earlier occurrence - constexpr static size_type NO_EARLIER_OCC = -1; - /// @brief A marker for a block that has been pruned - constexpr static size_type PRUNED = -2; - - /// @brief Base of the polynomial used for the Rabin-Karp hasher - constexpr static size_type SIGMA = 256; - - /// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher - constexpr static uint8_t PRIME_EXPONENT = 107; - // constexpr static uint8_t PRIME_EXPONENT = 89; - // constexpr static uint8_t PRIME_EXPONENT = 61; - /// @brief A mersenne prime used for the Rabin-Karp hasher - constexpr static uint128_t PRIME = pasta::primer(); - - /// @brief A bit vector - using BitVector = pasta::BitVector; - /// @brief A rank data structure for a bit vector - using Rank = pasta::RankSelect; - - /// @brief A sequential hash map used as backing for the sharded hash map. - template - using SeqHashMap = - ankerl::unordered_dense::map>; - // robin_hood::unordered_flat_map>; - // std::unordered_map>; - - /// @brief A rabin karp hasher preconfigured for the current template - /// parameters - using RabinKarp = MersenneRabinKarp; - /// @brief A rabin karp hash for the preconfigured rabin karp hasher - using RabinKarpHash = MersenneHash; - - /// @brief A hash map with rabin karp hashes as keys - template update_fn_type, - template typename seq_map_type = SeqHashMap> - using RabinKarpMap = - SyncShardedMap; - -#define MIX - static uint64_t mix_select(uint64_t key) { -#ifdef MIX - key ^= (key >> 31); - key *= 0x7fb5d329728ea185; - key ^= (key >> 27); - key *= 0x81dadef4bc2dd44d; - key ^= (key >> 33); -#endif - return key; - } - -#ifdef BT_INSTRUMENT -public: - size_t bp_hash_pairs_ns = 0; - size_t bp_scan_pairs_ns = 0; - size_t bp_markings_ns = 0; - size_t bp_bitvec_ns = 0; - - size_t b_hash_blocks_ns = 0; - size_t b_scan_blocks_ns = 0; - size_t b_update_blocks_ns = 0; -#endif - -private: - /// @brief Contains data about a block tree level under construction - struct LevelData { - /// @brief Contains a 1 for each internal block (= block with children) - /// and a 0 for each block that has a back pointer - std::unique_ptr is_internal; - /// @brief Rank data structure for is_internal - std::unique_ptr is_internal_rank; - /// @brief The block from which a back block is copying - std::unique_ptr> pointers; - /// @brief The offset into the block from which the back block is copying - std::unique_ptr> offsets; - /// @brief The number of back blocks pointing to the block - std::unique_ptr> counters; - /// @brief Block start indices - std::unique_ptr> block_starts; - /// @brief The block size on this level - int64_t block_size; - /// @brief The index of the current level. - /// First level is 0, second level is 1 etc. - int64_t level_index; - /// @brief The number of blocks on the current level - int64_t num_blocks; - - LevelData(const int64_t level_index_, - const int64_t block_size_, - const int64_t num_blocks_) - : is_internal(nullptr), - is_internal_rank(nullptr), - pointers(new std::vector()), - offsets(new std::vector()), - counters(new std::vector()), - block_starts(new std::vector()), - block_size(block_size_), - level_index(level_index_), - num_blocks(num_blocks_) {} - - /// @brief Checks whether a block is adjacent in the text - /// to its successor on this level - [[nodiscard]] bool next_is_adjacent(size_t i) const { - return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; - } - }; - - /// @brief Contains data about the occurrences of a hashed block pair - struct PairOccurrences { - /// @brief The first block in the text in which the content appears - size_type first_occ_block; - /// @brief A list of block indices in which the content of the hashed block - /// pair appears - /// - /// We're using an std::list here instead of an std::vector, since the - /// reallocation upon insertion lead to issues during parallel access, when - /// another thread tries to access the vector during reallocation. - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block pair. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the pair's first block. - inline explicit PairOccurrences(size_type first_occ_block_) - : first_occ_block(first_occ_block_), - occurrences() {} - - PairOccurrences(PairOccurrences&&) noexcept = default; - PairOccurrences& operator=(PairOccurrences&&) = default; - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block_pair(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index is an earlier occurrence, update it - /// @param block_index The block index of an occurrence - void update(size_type block_index) { - first_occ_block = std::min(first_occ_block, block_index); - } - }; - - /// @brief Contains data about the occurrences of a hashed block - struct BlockOccurrences { - /// @brief Represents the first occurrence of a block - struct FirstOccurrence { - /// @brief Block index of the first occurrence of the block's content - size_type block; - /// @brief The offset into the block at which that first occurrence occurs - size_type offset; - - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) - : block(first_occ_block_), - offset(first_occ_offset_) {} - }; - - // @brief The block index and offset of the first occurrence of this block's - // content - std::atomic first_occ; - - /// @brief A list of block indices in which the content of the hashed block - /// occurs - std::list occurrences; - - /// @brief Initialize the occurrences of a hashed block. - /// - /// Note, that this only sets the first occurrence to the given block index, - /// but does not add it to the occurrences list. - /// @param first_occ_block_ The block index of the block's first occurrence. - explicit BlockOccurrences(size_type first_occ_block_) - : first_occ({first_occ_block_, 0}), - occurrences() {} - - BlockOccurrences(const BlockOccurrences& other) - : first_occ(other.first_occ.load()), - occurrences(other.occurrences) {} - - BlockOccurrences(BlockOccurrences&& other) noexcept - : first_occ(other.first_occ.load()), - occurrences(std::move(other.occurrences)) {} - - ~BlockOccurrences() = default; - - BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { - first_occ = other.first_occ.load(); - occurrences = std::move(other.occurrences); - return *this; - } - - /// @brief Add a block index to the occurrences. - /// @param block_index The block index to add to the occurrences. - void add_block(size_type block_index) { - occurrences.push_back(block_index); - } - - /// @brief If the given block index and offset are an earlier occurrence, - /// update them - /// @param block_index The block index of an occurrence - /// @param block_offset The offset of that occurrence - void update(size_type block_index, size_type block_offset) { - FirstOccurrence prev_first_occ = this->first_occ.load(); - FirstOccurrence set(block_index, block_offset); - while (block_index < prev_first_occ.block && - !first_occ.compare_exchange_weak(prev_first_occ, set)) { - } - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block pair - struct UpdatePairOccurrences { - /// @brief The block index to add to the occurrences - using InputValue = size_type; - /// @brief Update the occurrences of a hashed block pair by adding the new - /// block index and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index to add to the occurrences - inline static void update(const RabinKarpHash&, - PairOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - } - - /// @brief Initialize the occurrences of a hashed block pair - /// @param input_value The block index of the pair's first block - /// @return The initialized occurrences only containing the given block pair - inline static PairOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - PairOccurrences occurrences(input_value); - occurrences.add_block_pair(input_value); - occurrences.update(input_value); - return occurrences; - } - }; - - /// @brief An update function for the sharded hash map that updates the - /// occurrences of a hashed block - struct UpdateBlockOccurrences { - /// @brief A pair of the block index - /// and offset of the first occurrence of a block - using InputValue = std::pair; - - /// @brief Update the occurrences of a hashed block by adding the new - /// block index and offset and updating the first occurrence if needed - /// @param occurrences A reference to the occurrences in the map - /// @param input_value The new block index and offset to add to the - /// occurrences - inline static void update(const RabinKarpHash&, - BlockOccurrences& occurrences, - InputValue&& input_value) { - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - } - - /// @brief Initialize the occurrences of a hashed block. - /// @param input_value A pair of the block index and offset of one of the - /// block's occurrences - /// @return The initialized occurrences only containing the given block - inline static BlockOccurrences init(const RabinKarpHash&, - InputValue&& input_value) { - BlockOccurrences occurrences(input_value.first); - occurrences.add_block(input_value.first); - occurrences.update(input_value.first, input_value.second); - return occurrences; - } - }; - - /// @brief A map containing hashed block pairs mapped to their occurrences - using BlockPairMap = RabinKarpMap; - /// @brief A map containing hashed blocks mapped to their occurrences - using BlockMap = RabinKarpMap; - - /// @brief Constructs the block tree. - /// @param text The input text. - /// @param threads The number of threads to use for construction - /// @param queue_size The max number of items in each thread's queue for its - /// hash map - void construct(const std::span text, - const size_t threads, - const size_t queue_size) { -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - const size_type text_len = text.size(); - /// The number of characters a block tree with s top-level blocks and arity - /// of strictly tau would exceed over the text size - int64_t padding; - /// The height of the tree - int64_t tree_height; - /// The size of the largest blocks (i.e. the top level blocks) - int64_t top_block_size; - - this->calculate_padding(padding, text_len, tree_height, top_block_size); - - const bool is_padded = padding > 0; - - std::vector levels; - - // Prepare the top level - levels.emplace_back(0, top_block_size, text_len / top_block_size); - LevelData& top_level = levels.back(); - top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); - for (size_type i = 0; i < text_len; i += top_level.block_size) { - top_level.block_starts->push_back(i); - } - top_level.block_size = top_block_size; - top_level.num_blocks = top_level.block_starts->size(); - -#ifdef BT_INSTRUMENT - - const size_t setup_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_BENCH - std::cout << " setup=" << setup_ns; -# endif - - size_t pairs_ns = 0; - size_t blocks_ns = 0; - size_t generate_ns = 0; -#endif -#ifdef BT_DBG - std::cout << "using " << threads << " threads" << std::endl; -#endif - -#ifdef BT_BENCH - std::cout << " queue_capacity=" << queue_size; -#endif - - // Construct the pre-pruned tree level by level - for (size_t level = 0; level < static_cast(tree_height); level++) { -#ifdef BT_DBG - std::cout << "----------------- level " << level << " -----------------" - << std::endl; -#endif - -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - LevelData& current = levels.back(); - if (2 * static_cast(current.block_size) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } -#ifdef BT_INSTRUMENT - pairs_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - if (static_cast(current.block_size) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } -#ifdef BT_INSTRUMENT - blocks_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the next level (if we're not at the last level) - if (level < static_cast(tree_height) - 1) { - levels.push_back(std::move(generate_next_level(text, current))); - } -#ifdef BT_INSTRUMENT - generate_ns += std::chrono::duration_cast( - Clock::now() - now) - .count(); -#endif - } -#ifdef BT_INSTRUMENT -# if defined(BT_DBG) - std::cout << "pairs: " << (pairs_ns / 1'000'000) - << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) - << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) - << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) - << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) - << "ms,\nblocks: " << (blocks_ns / 1'000'000) - << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) - << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) - << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) - << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," - << std::endl; -# elif defined(BT_BENCH) - std::cout << " pairs=" << (pairs_ns / 1'000'000) - << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) - << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) - << " markings=" << (bp_markings_ns / 1'000'000) - << " bitvec=" << (bp_bitvec_ns / 1'000'000) - << " blocks=" << (blocks_ns / 1'000'000) - << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) - << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) - << " update_blocks=" << (b_update_blocks_ns / 1'000'000) - << " generate_next=" << (generate_ns / 1'000'000); - -# endif - now = Clock::now(); -#endif - prune(levels); -#ifdef BT_INSTRUMENT - size_t prune_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -# ifdef BT_DBG - std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; -# elif defined BT_BENCH - std::cout << " prune=" << (prune_ns / 1'000'000); -# endif -#endif - - make_tree(text, levels, padding); -#ifdef BT_INSTRUMENT - size_t make_ns = - std::chrono::duration_cast(Clock::now() - now) - .count(); -# ifdef BT_DBG - std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; -# elif defined BT_BENCH - std::cout << " make=" << (make_ns / 1'000'000); -# endif -#endif - } - - /// @brief Returns the ceiling of x / y for x > 0; - /// - /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c - inline static size_t ceil_div(std::integral auto x, std::integral auto y) { - return 1 + ((x - 1) / y); - } - - [[maybe_unused]] static void - print_aggregate(const char* name, - const tlx::Aggregate& agg, - const size_t div = 1) { - printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", - name, - static_cast(agg.min() / div), - static_cast(agg.max() / div), - agg.avg() / static_cast(div), - agg.standard_deviation(0) / static_cast(div), - static_cast(agg.count())); - } - - /// @brief Scan through the blocks pairwise in order to identify which blocks - /// should be replaced with back blocks. - /// - /// @param text The input string. - /// @param level The data for the current level. - /// @param is_padded `true` iff the last block on this level *does not* end at - /// the exact end of the text. - /// @param threads Number of threads to use - /// @param queue_size The size of the queue to use per thread in the sharded - /// hash map. - /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or - /// use the blocks' contents themselves as hashes. - /// For block sizes greater than 4 bytes, use Rabin-Karp. - /// - template - void scan_block_pairs(const std::span text, - LevelData& level, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - if (level.num_blocks < 4) { - level.is_internal = std::make_unique(level.num_blocks, true); - level.is_internal_rank = std::make_unique(*level.is_internal); - return; - } - - // A map containing hashed block pairs mapped to their indices of the - // pairs' first block respectively - BlockPairMap map(threads, queue_size); - - std::atomic_size_t threads_done = 0; - std::atomic_bool last_done = false; - auto& barrier = map.barrier(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, \ - map, \ - text, \ - now, \ - is_padded, \ - threads_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits, \ - threads, \ - std::cout) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level, map, text, is_padded, threads_done, last_done, barrier) -#endif - { - const size_t thread_id = omp_get_thread_num(); - typename BlockPairMap::Shard shard = map.get_shard(thread_id); - const size_t num_threads = omp_get_num_threads(); - const size_t num_block_pairs = level.num_blocks - 1 - is_padded; - const size_t block_size = level.block_size; - const size_t pair_size = 2 * block_size; - const auto& block_starts = *level.block_starts; - - // Hash every window and determine for all block pairs whether - // they have previous occurrences. - const size_t segment_size = - std::max(1, ceil_div(num_block_pairs, num_threads)); - - // Start and end index of the current thread's segment - const auto start = thread_id * segment_size; - const auto end = - std::min(num_block_pairs, (thread_id + 1) * segment_size); - - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block - if (!level.next_is_adjacent(i)) { - continue; - } - rk.restart(block_starts[i]); - // Move the hasher to the current block pair - RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } else { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start, - block_size); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - } - - if (const size_t thread_order = - threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - // Now, we handle the queue asynchronously - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - { - bp_hash_pairs_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - - if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], pair_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); - } - scan_windows_in_block_pair(rk, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - scan_windows_in_block_pair_identity(text, - block_starts[i], - pair_size, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } - } - -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - bp_scan_pairs_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : map.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Pair Map Loads ", map_loads); - print_aggregate("Pair Map Hits ", scan_hits); - print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); - - BT_ASSERT(map.num_inserts_.load() == map.size()); -# endif -#endif - - level.is_internal = std::make_unique(level.num_blocks); - fill_is_internal(*level.is_internal, map); - level.is_internal_rank = std::make_unique(*level.is_internal); - } - - /// @brief Fills the bit vector `is_internal` based on the values in the - /// given map. - /// @param is_internal An unfilled bit vector with a bit for each block on - /// this level. - /// @param map A map, mapping hashed block pairs to their first occurrence's - /// block index. - void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { - const size_type num_blocks = is_internal.size(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); -#endif - // Set up the packed array holding the markings for each block. - // Each mark is a 2-bit number. - // The MSB is 1 iff the block and its successor have a prior - // occurrence. The LSB is 1 iff the block and its predecessor - // have a prior occurrence. - sdsl::int_vector<2> markings(num_blocks, 0); - map.for_each( - [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { - for (const size_type occ : pair_occs.occurrences) { - if (pair_occs.first_occ_block < occ) { - markings[occ] = markings[occ] | 0b10; - markings[occ + 1] = markings[occ + 1] | 0b01; - } - } - }); -#ifdef BT_INSTRUMENT - bp_markings_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); -#endif - - // Generate the bit vector indicating which blocks are internal - is_internal[0] = true; - is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; - for (size_type i = 0; i < num_blocks - 1; ++i) { - const bool block_is_internal = markings[i] != 0b11; - is_internal[i] = block_is_internal; - } -#ifdef BT_INSTRUMENT - bp_bitvec_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scan through the windows starting in a block and mark - /// them accordingly if they represent the earliest occurrence of some - /// block hash. - /// - /// The supplied `RabinKarp` hasher must be at the start of the block. - /// @param rk A Rabin-Karp hasher whose state is at the start of the block. - /// @param map The map containing the hashes of block pairs mapped to their - /// block indexes at which they occur. - /// @param num_iterations The number of contiguous windows to hash. - /// @param current_block_index The index of the block being currently - /// hashed. - static inline void - scan_windows_in_block_pair(RabinKarp& rk, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg -#endif - ) { - for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { - RabinKarpHash current_hash = rk.current_hash(); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { -#ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); -#else - continue; -#endif - } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - - static inline void - scan_windows_in_block_pair_identity(const std::span& text, - const size_t block_start, - const size_t pair_size, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg -#endif - ) { - const uint64_t HASH_MASK = HASH_MASKS[pair_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_t offset = 0; offset < num_iterations; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash current_hash(text, - mix_select(hash_value), - block_start + offset, - pair_size); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { -#ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); -#else - continue; -#endif - } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - - /// @brief Determine the positions for each block's earliest occurrence if - /// there is any. - /// - /// @param text The input text - /// @param level_data The data for the current level - /// @param is_padded true, iff the last block of the level extends past the - /// end of the text - /// @param threads The number of threads to use during construction. - /// @param queue_size The max number of items in each thread's queues. - /// @tparam use_hash Determines whether to use a rabin karp hash for hashing - /// text windows or to use the block's content as a hash. For any window size - /// greater than 8 bytes, use Rabin-Karp. - template - void scan_blocks(std::span text, - LevelData& level_data, - const bool is_padded, - const size_t threads, - const size_t queue_size) { - const size_t num_blocks = level_data.num_blocks; - - level_data.pointers = - std::make_unique>(num_blocks, NO_EARLIER_OCC); - level_data.offsets = - std::make_unique>(num_blocks, 0); - level_data.counters = - std::make_unique>(num_blocks, 0); - - if (num_blocks <= 2) { - return; - } - - // A map hashing blocks and saving where they occur. - BlockMap links(threads, queue_size); - - // The number of threads finished with hashing blocks - std::atomic_size_t num_done = 0; - // Whether the last thread is done - std::atomic_bool last_done = false; - auto& barrier = links.barrier(); -#ifdef BT_INSTRUMENT - TimePoint now = Clock::now(); - tlx::Aggregate scan_hits; - tlx::Aggregate start_idle_ns; - tlx::Aggregate finish_idle_ns; - tlx::Aggregate total_idle_ns; - tlx::Aggregate handle_queue_ns; - -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, \ - text, \ - links, \ - now, \ - is_padded, \ - num_done, \ - last_done, \ - barrier, \ - start_idle_ns, \ - finish_idle_ns, \ - total_idle_ns, \ - handle_queue_ns, \ - scan_hits) -#else -# pragma omp parallel default(none) num_threads(threads) \ - shared(level_data, text, links, is_padded, num_done, last_done, barrier) -#endif - { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[0], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } else { - const uint64_t HASH_MASK = HASH_MASKS[block_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); - } - } - - if (const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); -#pragma omp barrier -#pragma omp single -#ifdef BT_INSTRUMENT - - { - b_hash_blocks_ns += - std::chrono::duration_cast(Clock::now() - - now) - .count(); - now = Clock::now(); - } - - tlx::Aggregate thread_scan_hits; -#else - { - } -#endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, SIGMA, block_starts[start], block_size, PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); - } - } - } -#ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); - -# pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; -#endif - } -#ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); - now = Clock::now(); - -# ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : links.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); - print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); - - BT_ASSERT(links.num_inserts_.load() == links.size()); -# endif -#endif - - // By this point, the map should contain the first occurrences of - // every respective block's content. We then fill the pointers - // and offsets with this data and increment counters accordingly - links.for_each( - [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; - } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); - -#ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast(Clock::now() - now) - .count(); -#endif - } - - /// @brief Scans through block-sized windows starting inside one block and - /// tries to find blocks with matching hashes in the map. Such blocks - /// will have their earliest occurrence update. - /// @param rk A Rabin-Karp hasher whose current state is at a block start. - /// @param links A map whose keys are hashed blocks and the values - /// are all block indices of blocks matching the hash in ascending order. - /// @param level_data The data for the current level. - /// @param current_block_index The index of the block which the - /// Rabin-Karp hasher is situated in. - static void scan_windows_in_block(RabinKarp& rk, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits -#endif - ) { - for (size_type offset = 0; offset < level_data.block_size; - ++offset, rk.next()) { - RabinKarpHash hash = rk.current_hash(); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { -#ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); -#else - continue; -#endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - - static void - scan_windows_in_block_identity(const std::span& text, - const size_t block_start, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index -#ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits -#endif - ) { - const uint64_t HASH_MASK = HASH_MASKS[level_data.block_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_type offset = 0; offset < level_data.block_size; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash hash(text, - mix_select(hash_value), - block_start + offset, - level_data.block_size); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { -#ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); -#else - continue; -#endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - - /// @brief Generate the block size, number of block and block start indices - /// for the next level. - /// - /// This depends on the current level's block size, number of blocks and - /// is_internal bit vector being filled. - /// - /// @param text The input text. - /// @param level The level data of the current level. - /// @return The level data of the next level. - [[nodiscard]] LevelData - generate_next_level(const std::span text, - const LevelData& level) const { - const size_t block_size = level.block_size; - const size_t num_blocks = level.num_blocks; - const auto& is_internal = *level.is_internal; - const size_t next_block_size = block_size / this->tau_; - - std::vector new_block_starts; - new_block_starts.reserve(num_blocks * this->tau_); - for (size_t i = 0; i < num_blocks; ++i) { - if (!is_internal[i]) { - continue; - } - - // We generate up to tau new blocks for each internal block, - // excluding blocks that start past the end of the text - const auto parent_block_start = (*level.block_starts)[i]; - for (size_t j = 0, current_block_start = parent_block_start; - j < static_cast(this->tau_) && - current_block_start < text.size(); - ++j, current_block_start += next_block_size) { - new_block_starts.push_back(current_block_start); - } - } - - LevelData next_level(level.level_index + 1, - next_block_size, - new_block_starts.size()); - next_level.block_starts = - std::make_unique>(std::move(new_block_starts)); - return next_level; - } - - /// - /// @brief Takes a vector of levels and fills the block tree fields with - /// them. - /// - /// @param[in] levels A vector containing data for each level, with the - /// first entry corresponding to the topmost level. - /// - void make_tree(const std::span text, - std::vector& levels, - const int64_t padding) { - const bool is_padded = padding > 0; - - // Count the current number of internal blocks per level - std::vector new_num_internal(levels.size(), 0); - for (size_t level = 0; level < levels.size(); level++) { - for (size_t block = 0; block < levels[level].is_internal->size(); - block++) { - if ((*levels[level].is_internal)[block]) { - ++new_num_internal[level]; - } - } - } - - // Create first level - bool found_back_block = levels[0].is_internal->size() > - static_cast(new_num_internal[0]) || - !this->CUT_FIRST_LEVELS; - LevelData& top_level = levels.front(); - if (found_back_block) { - const size_t n = top_level.num_blocks; - const size_t num_internal = new_num_internal[0]; - auto pointers = new sdsl::int_vector<>(n - num_internal, 0); - auto offsets = new sdsl::int_vector<>(n - num_internal, 0); - size_t num_back_blocks = 0; - for (size_t i = 0; i < n; i++) { - // if a back block is found, add its pointer and offset - if (!(*top_level.is_internal)[i]) { - (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; - (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; - num_back_blocks++; - } - } - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(top_level.is_internal.release()); - this->block_tree_types_rs_.push_back( - new Rank(*this->block_tree_types_.back())); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(top_level.block_size); - } - top_level.pointers.reset(); - top_level.offsets.reset(); - top_level.counters.reset(); - - // Add level data to the tree - for (size_t level_index = 1; level_index < levels.size(); level_index++) { - LevelData& level = levels[level_index]; - LevelData& previous_level = levels[level_index - 1]; - found_back_block |= static_cast(new_num_internal[level_index]) < - levels[level_index].is_internal->size(); - if (!found_back_block && level_index < levels.size() - 1) { - if (level_index < levels.size() - 1) { - level.is_internal.reset(); - } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - continue; - } - - make_tree_level(levels, - new_num_internal, - level_index, - is_padded, - text.size()); - - // We don't need these anymore - if (level_index < levels.size() - 1) { - level.is_internal.reset(); - } - level.is_internal_rank.reset(); - level.pointers.reset(); - level.offsets.reset(); - level.counters.reset(); - previous_level.block_starts.reset(); - } - - this->leaf_size = levels.back().block_size / this->tau_; - // Construct the leaf string - int64_t leaf_count = 0; - auto& last_is_internal = *levels.back().is_internal; - std::vector& last_block_starts = *levels.back().block_starts; - for (size_t block = 0; block < last_is_internal.size(); block++) { - if (!last_is_internal[block]) { - continue; - } - const size_type block_start = last_block_starts[block]; - // For every leaf on the last level, we have tau leaf blocks - leaf_count += this->tau_; - // Iterate through all characters in this child and - // add them to the leaf string - for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); - b++) { - if (static_cast(block_start + b) < text.size()) { - this->leaves_.push_back(text[block_start + b]); - } else { - this->leaves_.push_back(0); - } - } - } - this->amount_of_leaves = leaf_count; - this->compress_leaves(); - } - - /// @brief Generates a level and adds the relevant data to the block tree. - /// - /// @param levels The vector of levels of the tree. - /// @param level_index The index of the level to generate. This must be - /// strictly greater than 0. - /// @param is_padded Whether there is padding in the last block of the tree - void make_tree_level(std::vector& levels, - const std::vector& new_num_internal, - const size_t level_index, - const bool is_padded, - const size_t text_len) { - LevelData& previous_level = levels[level_index - 1]; - LevelData& level = levels[level_index]; - - size_type new_size = - (new_num_internal[level_index - 1] - is_padded) * this->tau_; - // Determine the number of children the last block generated - if (is_padded) { - const size_type last_block_parent_start = - previous_level.block_starts->back(); - const size_type block_size = level.block_size; - new_size += ceil_div(text_len - last_block_parent_start, block_size); - } - previous_level.block_starts.reset(); - const size_type num_internal = new_num_internal[level_index]; - - // Allocate new vectors for the tree - auto* is_internal = new BitVector(new_size); - auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); - auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); - - // Number of non-pruned blocks before the current block - size_type num_non_pruned = 0; - // Number of back blocks before the current block - size_type num_back_blocks = 0; - // Number of pruned blocks before the current block - size_type num_pruned = 0; - - // We will reuse the allocated memory of the pointers vector to - // store the number of pruned blocks before the block. The - // invariant is that all values up to i are overwritten while all - // values starting after i will still be valid pointers - // This contains the number of pruned blocks before the block i - std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { - const size_type ptr = (*level.pointers)[i]; - prefix_pruned_blocks[i] = num_pruned; - - // If the current block is not pruned, add it to the new tree - if (ptr == PRUNED) { - num_pruned++; - continue; - } - - // Add it to the is_internal bit vector - const bool block_is_internal = (*level.is_internal)[i]; - (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; - - if (block_is_internal) { - continue; - } - - // If it is a back block, add its pointer and offset - const size_type offset = (*level.offsets)[i]; - - (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; - (*offsets)[num_back_blocks] = offset; - ++num_back_blocks; - } - - sdsl::util::bit_compress(*pointers); - sdsl::util::bit_compress(*offsets); - this->block_tree_types_.push_back(is_internal); - this->block_tree_types_rs_.push_back(new Rank(*is_internal)); - this->block_tree_pointers_.push_back(pointers); - this->block_tree_offsets_.push_back(offsets); - this->block_size_lvl_.push_back(level.block_size); - } - - /// @brief Prunes the tree of unnecessary nodes. - /// @param levels The levels of the tre represented as a vector of levels. - void prune(std::vector& levels) { - // We need to traverse the block tree in post order, - // handling children from right to left - for (int block_index = levels[0].num_blocks - 1; block_index >= 0; - --block_index) { - prune_block(levels, 0, block_index); - } - } - - /// @brief Prunes a block and its descendants of unnecessary internal nodes. - /// @param levels The WIP levels of the tree. - /// @param level_index The level of the block to prune. - /// @param block_index The index of the block to prune. - /// @return Whether this block is/stays internal after the pruning process - bool prune_block(std::vector& levels, - const size_t level_index, - const size_t block_index) const { - LevelData& level = levels[level_index]; - BitVector& is_internal = *level.is_internal; - - // If the current block is a back block already, there is nothing - // to prune - if (!is_internal[block_index]) { - return false; - } - - const size_type first_child = - level.is_internal_rank->rank1(block_index) * this->tau_; - - bool has_internal_children = false; - - // On the last level, all blocks just have leaves as children, - // none of which can be pointed to. So only recurse, if we are - // not on the last level. - if (level_index < levels.size() - 1) { - const size_type last_child = - std::min(first_child + this->tau_ - 1, - levels[level_index + 1].is_internal->size() - 1); - // Iterate through children in reverse - for (size_type child = last_child; child >= first_child; --child) { - has_internal_children |= prune_block(levels, level_index + 1, child); - } - } - - // If any of the children is internal, this block stays internal - // as well - if (has_internal_children) { - return true; - } - - const size_type pointer = (*level.pointers)[block_index]; - const size_type offset = (*level.offsets)[block_index]; - const size_type counter = (*level.counters)[block_index]; - // If there is no earlier occurrence or there are blocks pointing - // to this, then this must stay internal - if (pointer == NO_EARLIER_OCC || counter > 0) { - return true; - } - - // Now we know that there is an earlier occurrence, - // and nothing is pointing here. - // We will make this block here into a back block... - is_internal[block_index] = false; - (*level.counters)[pointer] += 1; - (*level.counters)[pointer + 1] += offset > 0; - - if (level_index == levels.size() - 1) { - return false; - } - - // ...and mark the children as pruned - LevelData& child_level = levels[level_index + 1]; - const size_type last_child = - std::min(first_child + this->tau_ - 1, - child_level.is_internal->size() - 1); - for (size_type child = last_child; child >= first_child; --child) { - const size_type child_pointer = (*child_level.pointers)[child]; - const size_type child_offset = (*child_level.offsets)[child]; -#ifdef BT_DBG - if (!(*child_level.is_internal)[child] && child_pointer < 0) { - std::cout << "non-internal node missing pointer" << std::endl; - std::cout << level_index << ", " << block_index << " / " - << child_level.is_internal->size() << std::endl; - } else if (child_pointer == PRUNED && child_pointer < 0) { - std::cout << "pruned node missing pointer" << std::endl; - } - BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); - BT_ASSERT(child_pointer >= 0); -#endif - // Decrement the counter of where the child points - (*child_level.counters)[child_pointer] -= 1; - (*child_level.counters)[child_pointer + 1] -= child_offset > 0; - // Mark the child as pruned - (*child_level.pointers)[child] = PRUNED; - } - - return false; - } - -public: - BitBlockTreeSharded(const pasta::BitVector& text, - const size_t arity, - const size_t root_arity, - const size_t max_leaf_length, - const size_t threads, - const size_t queue_size) { - const auto old = omp_get_max_threads(); - const auto old_dynamic = omp_get_dynamic(); - omp_set_dynamic(0); - omp_set_num_threads(static_cast(threads)); - this->tau_ = arity; - this->s_ = root_arity; - this->max_leaf_length_ = max_leaf_length; - this->num_bits = text.size(); - const std::span bytes(reinterpret_cast(text.data().data()), - ceil_div(text.size(), 8ULL)); - construct(bytes, threads, queue_size); - omp_set_dynamic(old_dynamic); - omp_set_num_threads(old); - } - - ~BitBlockTreeSharded() { - for (auto& rank : this->block_tree_types_rs_) { - delete rank; - } - for (auto& bv : this->block_tree_types_) { - delete bv; - } - for (auto& ptrs : this->block_tree_pointers_) { - delete ptrs; - } - for (auto& offsets : this->block_tree_offsets_) { - delete offsets; - } - } -}; -} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index d5fd43c..4eea4fd 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -24,14 +24,14 @@ #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include +#include __extension__ typedef unsigned __int128 uint128_t; namespace pasta { template> - using HashMap = robin_hood::unordered_map; + using HashMap = ankerl::unordered_dense::map; template class BlockTreeFP : public BlockTree { diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp index a8a69a7..7eab044 100644 --- a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include @@ -58,26 +57,10 @@ class BlockTreeFPParParlay : public BlockTree { using Rank = pasta::RankSelect; /// A concurrent hash map - /*template , - size_t num_submaps = 6, - typename mutex_type = phmap::NullMutex> - using HashMap = phmap::parallel_flat_hash_map< - key_type, - value_type, - hash_type, - phmap::priv::hash_default_eq, - phmap::priv::Allocator< - typename phmap::priv::Pair>, - num_submaps, - mutex_type>;*/ - template > using HashMap = parlay::unordered_map; - // robin_hood::unordered_node_map; /// A rabin karp hasher preconfigured for the current template parameters using RabinKarp = MersenneRabinKarp; diff --git a/include/pasta/block_tree/construction/block_tree_sharded.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp index acde1ed..135f226 100644 --- a/include/pasta/block_tree/construction/block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -24,15 +24,11 @@ #include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" -#include "pasta/block_tree/utils/byteread.hpp" #include "pasta/block_tree/utils/sharded_util.hpp" #include "pasta/block_tree/utils/sync_sharded_map.hpp" #include -#include -#include #include -#include #include #include #include diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index e006fcc..c36d20c 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index a0cfff9..c8b771e 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -33,7 +33,6 @@ #include #include #include -#include #include #include #include diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 09e9230..aa7c0b1 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -21,13 +21,11 @@ #pragma once -#include #include #include #include #include #include -#include #include #include From 2836006b6c7ba63b43141b54611764e6b46769a0 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sun, 24 Dec 2023 01:44:48 +0100 Subject: [PATCH 79/92] fix memory errors in sequential algorithms --- .../block_tree/construction/block_tree_fp.hpp | 317 +++++++++++------- .../construction/block_tree_fp2_seq.hpp | 6 +- tests/block_tree/block_tree_fp_test.cpp | 19 +- 3 files changed, 205 insertions(+), 137 deletions(-) diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 4eea4fd..116f807 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -25,13 +26,16 @@ #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" #include +#include __extension__ typedef unsigned __int128 uint128_t; namespace pasta { - template> - using HashMap = ankerl::unordered_dense::map; +template > +using HashMap = ankerl::unordered_dense::map; template class BlockTreeFP : public BlockTree { @@ -61,12 +65,14 @@ class BlockTreeFP : public BlockTree { /// @return true, if an this block was and stays internal, false otherwise /// bool prune_block( - std::vector> &counter, - std::vector> &pointer, - std::vector> &offset, - std::vector &marked_tree, - std::vector &pruned_tree, size_type i, size_type j, - std::vector> &ranks) { + std::vector>& counter, + std::vector>& pointer, + std::vector>& offset, + std::vector& marked_tree, + std::vector& pruned_tree, + size_type i, + size_type j, + std::vector>& ranks) { // String leaf children can always be pruned, // since they are contained in the leaf string. // Fully padded children don't exist and can be ignored/sanity check @@ -86,9 +92,14 @@ class BlockTreeFP : public BlockTree { // next level. size_type rank_blk = ranks[i].rank1(j); for (size_type k = this->tau_ - 1; k >= 0; k--) { - marked_children |= - prune_block(counter, pointer, offset, marked_tree, pruned_tree, - i + 1, rank_blk * this->tau_ + k, ranks); + marked_children |= prune_block(counter, + pointer, + offset, + marked_tree, + pruned_tree, + i + 1, + rank_blk * this->tau_ + k, + ranks); } // Conditions to be pruned are: // - no internal children, @@ -148,38 +159,43 @@ class BlockTreeFP : public BlockTree { /// content as marked_tree /// @return 0 /// - int32_t pruning_extended(std::vector> &counter, - std::vector> &pointer, - std::vector> &offset, - std::vector &marked_tree, - std::vector &pruned_tree) { + int32_t pruning_extended(std::vector>& counter, + std::vector>& pointer, + std::vector>& offset, + std::vector& marked_tree, + std::vector& pruned_tree) { std::vector> ranks; for (auto bv : marked_tree) { ranks.push_back(pasta::RankSelect(*bv)); } - auto &top_lvl = *marked_tree[0]; + auto& top_lvl = *marked_tree[0]; /// Prune the blocks on the top level from back to front for (size_type j = top_lvl.size() - 1; j >= 0; j--) { - prune_block(counter, pointer, offset, marked_tree, pruned_tree, 0, j, + prune_block(counter, + pointer, + offset, + marked_tree, + pruned_tree, + 0, + j, ranks); } return 0; } - int32_t pruning_simple(std::vector &first_pass_bv, - std::vector> &blk_lvl, - std::vector &bv_pass_2, - std::vector> &pass1_pointer, - std::vector> &pass1_offset, - std::vector> &pass2_pointer, - std::vector> &pass2_offset, - std::vector &pass2_max_pointer, - std::vector &pass2_max_offset, - std::vector &pass2_ones, - int64_t &block_size) { - + int32_t pruning_simple(std::vector& first_pass_bv, + std::vector>& blk_lvl, + std::vector& bv_pass_2, + std::vector>& pass1_pointer, + std::vector>& pass1_offset, + std::vector>& pass2_pointer, + std::vector>& pass2_offset, + std::vector& pass2_max_pointer, + std::vector& pass2_max_offset, + std::vector& pass2_ones, + int64_t& block_size) { for (int64_t i = first_pass_bv.size() - 1; i >= 0; i--) { - auto *bv = new pasta::BitVector(blk_lvl[i].size(), 0); + auto* bv = new pasta::BitVector(blk_lvl[i].size(), 0); size_type marked_counter = 0; if (static_cast(i) != first_pass_bv.size() - 1) { for (uint64_t j = 0; j < bv->size(); j++) { @@ -198,7 +214,6 @@ class BlockTreeFP : public BlockTree { auto offsets = std::vector(); size_type max_pointer = 0; for (size_type j = blk_lvl[i].size() - 1; j >= 0; j--) { - if ((*bv)[j] == 1) { continue; } @@ -241,7 +256,7 @@ class BlockTreeFP : public BlockTree { return 0; } - int32_t init_extended(std::vector &text) { + int32_t init_extended(std::vector& text) { static constexpr uint128_t kPrime = 2305843009213693951ULL; /// The number of characters a block tree with s top-level blocks and arity /// of strictly tau would exceed over the text size @@ -261,14 +276,16 @@ class BlockTreeFP : public BlockTree { std::vector> pass1_offset; /// For each level contains a bit vector containing a 1 for each block that /// is internal and a 0 for each back block - std::vector bv_marked; + std::vector bv_marked; /// For every level and block counts how many back blocks are pointing to /// the block std::vector> counter; std::vector pass2_ones; /// The block size for each level, starting at the top level std::vector block_size_lvl_temp; - this->calculate_padding(added_padding, text.size(), tree_max_height, + this->calculate_padding(added_padding, + text.size(), + tree_max_height, max_blk_size); auto is_padded = added_padding > 0 ? 1 : 0; /// The current block size starting at the top level @@ -283,13 +300,13 @@ class BlockTreeFP : public BlockTree { // we may not divide them further. So the entire block tree just consists of // the current level verbatim, no pointers if (block_size <= this->max_leaf_length_) { - auto *bv = new pasta::BitVector(block_text_inx.size(), 1); + auto* bv = new pasta::BitVector(block_text_inx.size(), 1); this->block_tree_types_rs_.push_back( new pasta::RankSelect(*bv)); auto p0 = new sdsl::int_vector<>(0, 0); auto o0 = new sdsl::int_vector<>(0, 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; sdsl::util::bit_compress(ptr0); sdsl::util::bit_compress(off0); this->block_tree_types_.push_back(bv); @@ -304,7 +321,7 @@ class BlockTreeFP : public BlockTree { while (block_size > this->max_leaf_length_) { block_size_lvl_temp.push_back(block_size); /// Marks whether a block should be internal or not - auto *bv = new pasta::BitVector(block_text_inx.size(), false); + auto* bv = new pasta::BitVector(block_text_inx.size(), false); /// If left[i] == 1, then there is an earlier occurrence of /// block[i]block[i+1] auto left = pasta::BitVector(block_text_inx.size(), false); @@ -315,13 +332,12 @@ class BlockTreeFP : public BlockTree { // Check, whether the last block's end extends past the end of the text auto last_block_padded = static_cast(block_text_inx[block_text_inx.size() - 1] + - block_size) != text.size() - ? 1 - : 0; + block_size) != text.size() ? + 1 : + 0; // map block pair hashes to the text index of their occurrences // collecting duplicates in a vector TODO - HashMap, std::vector> pairs( - 0); + HashMap, std::vector> pairs(0); // map block hashes to their *block* index, // collecting duplicates in a vector TODO HashMap, std::vector> blocks = @@ -332,8 +348,11 @@ class BlockTreeFP : public BlockTree { // Hash the current block and insert it into the block hash map auto index = block_text_inx[i]; MersenneRabinKarp rk_block = - MersenneRabinKarp(text, sigma_, index, - block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + block_size, + kPrime); MersenneHash mh_block = MersenneHash(text, rk_block.hash_, index, block_size); blocks[mh_block].push_back(i); @@ -381,8 +400,11 @@ class BlockTreeFP : public BlockTree { text.size()) { auto index = block_text_inx[i]; MersenneRabinKarp rk_pair = - MersenneRabinKarp(text, sigma_, index, - pair_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + pair_size, + kPrime); MersenneHash mh_pair = MersenneHash(text, rk_pair.hash_, index, pair_size); pairs[mh_pair].push_back(i); @@ -390,7 +412,10 @@ class BlockTreeFP : public BlockTree { } // Find the occurrences of all block pairs' contents MersenneRabinKarp rk_pair_sw = - MersenneRabinKarp(text, sigma_, 0, pair_size, + MersenneRabinKarp(text, + sigma_, + 0, + pair_size, kPrime); // Hash each window in the text of the size of a block pair // and see if it corresponds to an actual block pair @@ -446,8 +471,11 @@ class BlockTreeFP : public BlockTree { } } MersenneRabinKarp rk_first_occ = - MersenneRabinKarp( - text, sigma_, block_text_inx[0], block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + block_text_inx[0], + block_size, + kPrime); // Identify the first occurrence for each block on this level for (int64_t i = 0; static_cast(i) < block_text_inx.size() - 1; i++) { @@ -476,8 +504,11 @@ class BlockTreeFP : public BlockTree { block_size) < text.size(); j++) { // Hash the window and try to find an earlier occurrence - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i] + j, block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i] + j, + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { // The if the current block (b) were i, it would reference @@ -505,8 +536,11 @@ class BlockTreeFP : public BlockTree { } } else { // If the next block is not adjacent, we only hash once - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i], block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i], + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (b != i) { @@ -534,13 +568,16 @@ class BlockTreeFP : public BlockTree { block_size *= this->tau_; // Prune the tree. Doing so will replace the pointers of pruned nodes with // PRUNED - pruning_extended(counter, pass1_pointer, pass1_offset, bv_marked, + pruning_extended(counter, + pass1_pointer, + pass1_offset, + bv_marked, bv_marked); std::vector ones_per_lvl(bv_marked.size(), 0); // count 1s in each lvl; for (uint64_t i = 0; i < bv_marked.size(); i++) { - auto ¤t_lvl = *bv_marked[i]; + auto& current_lvl = *bv_marked[i]; for (uint64_t j = 0; j < bv_marked[i]->size(); j++) { if (current_lvl[j]) { ones_per_lvl[i]++; @@ -548,7 +585,7 @@ class BlockTreeFP : public BlockTree { } } - auto &top_level = *bv_marked[0]; + auto& top_level = *bv_marked[0]; bool found_back_block = top_level.size() != static_cast(ones_per_lvl[0]) || bv_marked.size() == 1; @@ -559,8 +596,8 @@ class BlockTreeFP : public BlockTree { new pasta::RankSelect(top_level)); auto p0 = new sdsl::int_vector<>(top_level.size() - ones_per_lvl[0], 0); auto o0 = new sdsl::int_vector<>(top_level.size() - ones_per_lvl[0], 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; size_type c = 0; for (uint64_t j = 0; j < top_level.size(); j++) { if (!top_level[j]) { @@ -595,17 +632,18 @@ class BlockTreeFP : public BlockTree { // Check if we have found a back block on the current level found_back_block |= new_size != ones_per_lvl[i]; // If there is a back block, we add this level's data to the tree - if (found_back_block || !this->CUT_FIRST_LEVELS) { + if (found_back_block || !this->CUT_FIRST_LEVELS || + i == bv_marked.size() - 1) { // is_internal auto bit_vector = new pasta::BitVector(new_size, 0); - auto &bv_ref = *bit_vector; + auto& bv_ref = *bit_vector; auto p = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); auto o = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); - auto &ptr = *p; - auto &off = *o; + auto& ptr = *p; + auto& off = *o; // Maps block index => number of pruned blocks before this block HashMap blocks_skipped; - auto &lvl_pass1 = *bv_marked[i]; + auto& lvl_pass1 = *bv_marked[i]; // Number of non-pruned blocks so far size_type c = 0; // @@ -636,15 +674,18 @@ class BlockTreeFP : public BlockTree { this->block_tree_pointers_.push_back(p); this->block_tree_offsets_.push_back(o); this->block_size_lvl_.push_back(block_size_lvl_temp[i]); - } else { - // Otherwise, we don't need the data from this level anymore + } + // Delete the old bitvec since we don't need it anymore. + // If this is the last level, we still need the bv for constructing the + // leaves + if (i < bv_marked.size() - 1) { delete bv_marked[i]; } } // Construct the leaf string int64_t leaf_count = 0; - auto &last_level = (*bv_marked[bv_marked.size() - 1]); + auto& last_level = (*bv_marked[bv_marked.size() - 1]); for (uint64_t i = 0; i < last_level.size(); i++) { if (last_level[i] == 1) { // For every leaf on the last level, we have tau leaf blocks @@ -652,7 +693,8 @@ class BlockTreeFP : public BlockTree { // Iterate through all characters in this child and add them to the leaf // string for (uint64_t j = 0; - j < static_cast(this->leaf_size * this->tau_); j++) { + j < static_cast(this->leaf_size * this->tau_); + j++) { if (static_cast(blk_lvl[blk_lvl.size() - 1][i] + j) < text.size()) { this->leaves_.push_back(text[blk_lvl[blk_lvl.size() - 1][i] + j]); @@ -660,12 +702,13 @@ class BlockTreeFP : public BlockTree { } } } + delete &last_level; this->amount_of_leaves = leaf_count; this->compress_leaves(); return 0; } - int32_t init_simple(std::vector &text) { + int32_t init_simple(std::vector& text) { static constexpr uint128_t kPrime = 2305843009213693951ULL; int64_t added_padding = 0; int64_t tree_max_height = 0; @@ -673,15 +716,17 @@ class BlockTreeFP : public BlockTree { std::vector> blk_lvl; std::vector> pass1_pointer; std::vector> pass1_offset; - std::vector bv_pass_1; - std::vector bv_pass_2; + std::vector bv_pass_1; + std::vector bv_pass_2; std::vector> pass2_pointer; std::vector> pass2_offset; std::vector pass2_max_pointer; std::vector pass2_max_offset; std::vector pass2_ones; std::vector block_size_lvl_temp; - this->calculate_padding(added_padding, text.size(), tree_max_height, + this->calculate_padding(added_padding, + text.size(), + tree_max_height, max_blk_size); auto is_padded = added_padding > 0 ? 1 : 0; int64_t block_size = max_blk_size; @@ -690,13 +735,13 @@ class BlockTreeFP : public BlockTree { block_text_inx.push_back(i); } if (block_size <= this->max_leaf_length_) { - auto *bv = new pasta::BitVector(block_text_inx.size(), 1); + auto* bv = new pasta::BitVector(block_text_inx.size(), 1); this->block_tree_types_rs_.push_back( new pasta::RankSelect(*bv)); auto p0 = new sdsl::int_vector<>(0, 0); auto o0 = new sdsl::int_vector<>(0, 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; sdsl::util::bit_compress(ptr0); sdsl::util::bit_compress(off0); this->block_tree_types_.push_back(bv); @@ -711,24 +756,26 @@ class BlockTreeFP : public BlockTree { bool found_back_block = this->max_leaf_length_ * this->tau_ >= block_size; while (block_size > this->max_leaf_length_) { block_size_lvl_temp.push_back(block_size); - auto *bv = new pasta::BitVector(block_text_inx.size(), false); + auto* bv = new pasta::BitVector(block_text_inx.size(), false); auto left = pasta::BitVector(block_text_inx.size(), false); auto right = pasta::BitVector(block_text_inx.size(), false); auto pair_size = 2 * block_size; auto last_block_padded = static_cast(block_text_inx[block_text_inx.size() - 1] + - block_size) != text.size() - ? 1 - : 0; - HashMap, std::vector> pairs( - 0); + block_size) != text.size() ? + 1 : + 0; + HashMap, std::vector> pairs(0); HashMap, std::vector> blocks = HashMap, std::vector>(); for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { auto index = block_text_inx[i]; MersenneRabinKarp rk_block = - MersenneRabinKarp(text, sigma_, index, - block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + block_size, + kPrime); MersenneHash mh_block = MersenneHash(text, rk_block.hash_, index, block_size); blocks[mh_block].push_back(i); @@ -762,8 +809,11 @@ class BlockTreeFP : public BlockTree { text.size()) { auto index = block_text_inx[i]; MersenneRabinKarp rk_pair = - MersenneRabinKarp(text, sigma_, index, - pair_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + pair_size, + kPrime); MersenneHash mh_pair = MersenneHash(text, rk_pair.hash_, index, pair_size); pairs[mh_pair].push_back(i); @@ -771,7 +821,10 @@ class BlockTreeFP : public BlockTree { } // find pairs MersenneRabinKarp rk_pair_sw = - MersenneRabinKarp(text, sigma_, 0, pair_size, + MersenneRabinKarp(text, + sigma_, + 0, + pair_size, kPrime); for (uint64_t i = 0; i < text.size() - pair_size; i++) { MersenneHash mh_sw = @@ -814,8 +867,11 @@ class BlockTreeFP : public BlockTree { } for (uint64_t i = 0; i < block_text_inx.size() - 1; i++) { MersenneRabinKarp rk_first_occ = - MersenneRabinKarp( - text, sigma_, block_text_inx[i], block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + block_text_inx[i], + block_size, + kPrime); bool followed = (i < block_text_inx.size() - 1) && block_text_inx[i] + block_size == block_text_inx[i + 1] && @@ -826,8 +882,11 @@ class BlockTreeFP : public BlockTree { j < static_cast(block_size) && block_text_inx[i] + j + block_size < text.size(); j++) { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i] + j, block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i] + j, + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (static_cast(b) != i) { @@ -840,8 +899,11 @@ class BlockTreeFP : public BlockTree { rk_first_occ.next(); } } else { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i], block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i], + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (static_cast(b) != i) { @@ -864,9 +926,17 @@ class BlockTreeFP : public BlockTree { } this->leaf_size = block_size; block_size *= this->tau_; - pruning_simple(bv_pass_1, blk_lvl, bv_pass_2, pass1_pointer, pass1_offset, - pass2_pointer, pass2_offset, pass2_max_pointer, - pass2_max_offset, pass2_ones, block_size); + pruning_simple(bv_pass_1, + blk_lvl, + bv_pass_2, + pass1_pointer, + pass1_offset, + pass2_pointer, + pass2_offset, + pass2_max_pointer, + pass2_max_offset, + pass2_ones, + block_size); auto size = pass2_pointer[pass2_pointer.size() - 1].size(); found_back_block |= size != 0; if (found_back_block || !this->CUT_FIRST_LEVELS) { @@ -876,12 +946,14 @@ class BlockTreeFP : public BlockTree { *bv_pass_2[bv_pass_2.size() - 1])); auto p1 = new sdsl::int_vector<>( - size, 0, + size, + 0, (8 * sizeof(size_type)) - this->leading_zeros( pass2_max_pointer[pass2_max_pointer.size() - 1])); auto o1 = new sdsl::int_vector<>( - size, 0, + size, + 0, (8 * sizeof(size_type)) - this->leading_zeros( pass2_max_offset[pass2_max_offset.size() - 1])); @@ -909,7 +981,6 @@ class BlockTreeFP : public BlockTree { auto lvl_block_size = block_size_lvl_temp[level]; if (is_padded) { for (size_type j = 0; j < this->tau_; j++) { - if (static_cast(last_block_parent + j * lvl_block_size) < text.size()) { new_size++; @@ -918,7 +989,7 @@ class BlockTreeFP : public BlockTree { } found_back_block |= new_size != pass2_ones[i]; if (found_back_block || !this->CUT_FIRST_LEVELS) { - auto *bit_vector = new pasta::BitVector(new_size, 0); + auto* bit_vector = new pasta::BitVector(new_size, 0); auto pointer = std::vector(); auto offset = std::vector(); size_type pointer_saved = 0; @@ -930,7 +1001,8 @@ class BlockTreeFP : public BlockTree { if ((*bv_pass_1[pass1_i - 1])[j] == 1) { if ((*bv_pass_2[i + 1])[j] == 1) { for (size_type k = 0; - k < this->tau_ && replace * this->tau_ + k < new_size; k++) { + k < this->tau_ && replace * this->tau_ + k < new_size; + k++) { bool x = (*bv_pass_2[i])[(j - skip) * this->tau_ + k]; auto skipper = pointer_skipped + pointer_saved; (*bit_vector)[replace * this->tau_ + k] = x; @@ -956,11 +1028,13 @@ class BlockTreeFP : public BlockTree { } } auto p = new sdsl::int_vector<>( - pointer.size(), 0, + pointer.size(), + 0, (8 * sizeof(size_type)) - this->leading_zeros(pass2_max_pointer[i])); auto o = new sdsl::int_vector<>( - pointer.size(), 0, + pointer.size(), + 0, (8 * sizeof(size_type)) - this->leading_zeros(pass2_max_offset[i])); for (uint64_t j = 0; j < pointer.size(); j++) { (*p)[j] = pointer[j]; @@ -996,9 +1070,13 @@ class BlockTreeFP : public BlockTree { return 0; }; - BlockTreeFP(std::vector &text, size_type tau, - size_type max_leaf_length, size_type s, size_type sigma, - bool cut_first_levels, bool extended_prune) { + BlockTreeFP(std::vector& text, + size_type tau, + size_type max_leaf_length, + size_type s, + size_type sigma, + bool cut_first_levels, + bool extended_prune) { sigma_ = sigma; this->CUT_FIRST_LEVELS = cut_first_levels; this->map_unique_chars(text); @@ -1012,21 +1090,6 @@ class BlockTreeFP : public BlockTree { } }; - ~BlockTreeFP() { - for (auto &bt_t : this->block_tree_types_) { - delete bt_t; - } - for (auto &bt_rs : this->block_tree_types_rs_) { - delete bt_rs; - } - for (auto &bt_p : this->block_tree_pointers_) { - delete bt_p; - } - for (auto &bt_o : this->block_tree_offsets_) { - delete bt_o; - } - }; - private: // magic number to indicate that a block is pruned const int PRUNED = -2; @@ -1035,10 +1098,16 @@ class BlockTreeFP : public BlockTree { }; template -auto *make_block_tree_fp(std::vector &input, size_type const tau, +auto* make_block_tree_fp(std::vector& input, + size_type const tau, size_type const max_leaf_length) { - return new BlockTreeFP(input, tau, max_leaf_length, 1, - 256, true, true); + return new BlockTreeFP(input, + tau, + max_leaf_length, + 1, + 256, + true, + true); } } // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp index 6d41d22..5b0628a 100644 --- a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -431,8 +431,10 @@ class BlockTreeFP2 : public BlockTree { LevelData& previous_level = levels[level_index - 1]; found_back_block |= static_cast(new_num_internal[level_index]) < levels[level_index].is_internal->size(); - if (!found_back_block) { - level.is_internal.reset(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } level.is_internal_rank.reset(); level.pointers.reset(); level.offsets.reset(); diff --git a/tests/block_tree/block_tree_fp_test.cpp b/tests/block_tree/block_tree_fp_test.cpp index 116e366..6d3dfce 100644 --- a/tests/block_tree/block_tree_fp_test.cpp +++ b/tests/block_tree/block_tree_fp_test.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2022 Daniel Meyer * Copyright (C) 2023 Florian Kurpicz + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,22 +20,19 @@ * ******************************************************************************/ -#include -#include - #include - +#include #include +#include +#include class BlockTreeFPTest : public ::testing::Test { - protected: std::vector text; - pasta::BlockTreeFP *bt; + std::unique_ptr> bt; void SetUp() override { - std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dist(0, 15); @@ -45,11 +43,10 @@ class BlockTreeFPTest : public ::testing::Test { text[i] = dist(gen); } - bt = pasta::make_block_tree_fp(text, 2, 1); + bt = std::unique_ptr>( + pasta::make_block_tree_fp(text, 2, 8)); bt->add_rank_support(); } - - void TearDown() override { delete bt; } }; TEST_F(BlockTreeFPTest, access) { @@ -76,7 +73,7 @@ TEST_F(BlockTreeFPTest, select) { } } -int main(int argc, char **argv) { +int main(int argc, char** argv) { ::testing::InitGoogleTest(&argc, argv); return RUN_ALL_TESTS(); } From a7a4f1d96342c6e7aaa99a82dfa87554577368f4 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 28 Dec 2023 18:42:03 +0100 Subject: [PATCH 80/92] fix incorrect commit hash for bit_vector --- CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 0a8e7d5..1c47692 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -51,7 +51,7 @@ FetchContent_Declare( FetchContent_Declare( pasta_bit_vector GIT_REPOSITORY https://github.com/pasta-toolbox/bit_vector.git - GIT_TAG 05acd97 #main + GIT_TAG b4798d5 #main ) FetchContent_MakeAvailable(tlx pasta_bit_vector) From cb11dd1d5a7e8fca8dbb4ab8ed05e78da5097f7d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 29 Dec 2023 17:15:58 +0100 Subject: [PATCH 81/92] fix error occurring for input too large for bit width --- examples/build_bt.cpp | 48 ++++++++++--------- .../pasta/block_tree/utils/sharded_util.hpp | 15 ++++-- 2 files changed, 35 insertions(+), 28 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 2da928c..cc4e600 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -27,6 +27,8 @@ #define REC_PAR_SHARDED #define REC_DENSE_BIT +using SizeType = int64_t; + #if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED constexpr size_t RECURSION_LEVELS = 0; #else @@ -35,37 +37,37 @@ constexpr size_t RECURSION_LEVELS = 0; #if defined REC_DENSE_BIT # include -using BBT = pasta::RecursiveDenseBitBlockTreeSharded; +using BBT = pasta::RecursiveDenseBitBlockTreeSharded; # define BIT_ALGO_NAME "rec_dense_bit" #elif defined REC_BIT # include -using BBT = pasta::RecursiveBitBlockTreeSharded; +using BBT = pasta::RecursiveBitBlockTreeSharded; # define BIT_ALGO_NAME "rec_bit" #endif #ifdef FP # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t, const size_t) { ; - return std::unique_ptr>( - pasta::make_block_tree_fp(text, arity, leaf_length)); + return std::unique_ptr>( + pasta::make_block_tree_fp(text, arity, leaf_length)); } # define ALGO_NAME "fp" #elif defined FP2 # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t, const size_t) { ; - return std::make_unique>(text, + return std::make_unique>(text, arity, 1, leaf_length); @@ -73,15 +75,15 @@ make_bt(std::vector& text, # define ALGO_NAME "fp2" #elif defined LPF # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t) { ; - return std::unique_ptr>( - pasta::make_block_tree_lpf_parallel(text, + return std::unique_ptr>( + pasta::make_block_tree_lpf_parallel(text, arity, leaf_length, true, @@ -90,14 +92,14 @@ make_bt(std::vector& text, # define ALGO_NAME "lpf" #elif defined PAR_SHARDED # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -107,14 +109,14 @@ make_bt(std::vector& text, # define ALGO_NAME "shard" #elif defined PAR_SHARDED_SYNC # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -125,14 +127,14 @@ make_bt(std::vector& text, # define ALGO_NAME "shard_sync" #elif defined PAR_SHARDED_SYNC_SMALL # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { return std::make_unique< - pasta::RecursiveBlockTreeSharded>(text, + pasta::RecursiveBlockTreeSharded>(text, arity, 1, leaf_length, @@ -143,14 +145,14 @@ make_bt(std::vector& text, #elif defined REC_PAR_SHARDED # include std::unique_ptr< - pasta::RecursiveBlockTreeSharded> + pasta::RecursiveBlockTreeSharded> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t queue_size) { return std::make_unique< - pasta::RecursiveBlockTreeSharded>( + pasta::RecursiveBlockTreeSharded>( text, arity, 1, @@ -161,14 +163,14 @@ make_bt(std::vector& text, # define ALGO_NAME "rec_shard" #elif defined PAR_PHMAP # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -178,14 +180,14 @@ make_bt(std::vector& text, # define ALGO_NAME "par_map" #elif defined PAR_PARLAY # include -std::unique_ptr> +std::unique_ptr> make_bt(std::vector& text, const size_t arity, const size_t leaf_length, const size_t threads, const size_t) { ; - return std::make_unique>( + return std::make_unique>( text, arity, 1, @@ -325,7 +327,7 @@ int main(int argc, char** argv) { /* auto bt = std::make_unique< - RecursiveBitBlockTreeSharded>(*bv, + RecursiveBitBlockTreeSharded>(*bv, arity, 1, leaf_length, diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp index e1b9521..f973b4b 100644 --- a/include/pasta/block_tree/utils/sharded_util.hpp +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -161,18 +161,23 @@ struct PairOccurrences { template struct BlockOccurrences { /// @brief Represents the first occurrence of a block - struct FirstOccurrence { + struct [[gnu::packed]] FirstOccurrence { /// @brief Block index of the first occurrence of the block's content - size_type block; + int64_t block : 40; /// @brief The offset into the block at which that first occurrence occurs - size_type offset; + int32_t offset : 24; - inline FirstOccurrence(size_type first_occ_block_, - size_type first_occ_offset_) + inline FirstOccurrence(int64_t first_occ_block_, + int32_t first_occ_offset_) : block(first_occ_block_), offset(first_occ_offset_) {} }; + static_assert(std::atomic::is_always_lock_free, + "first occurrence must be able to be atomically updated"); + static_assert(sizeof(FirstOccurrence) == 8, + "should be size of computer word"); + // @brief The block index and offset of the first occurrence of this block's // content std::atomic first_occ; From 6ed08070c52a8374140213d13e7bee5adcaafe93 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Tue, 2 Jan 2024 13:43:03 +0100 Subject: [PATCH 82/92] do not read entire text to memory at once --- examples/build_bt.cpp | 53 ++++++++++++++++++++++++------------------- 1 file changed, 30 insertions(+), 23 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index cc4e600..4dab905 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -20,6 +20,7 @@ #include #include +#include #include #include #include @@ -37,7 +38,8 @@ constexpr size_t RECURSION_LEVELS = 0; #if defined REC_DENSE_BIT # include -using BBT = pasta::RecursiveDenseBitBlockTreeSharded; +using BBT = + pasta::RecursiveDenseBitBlockTreeSharded; # define BIT_ALGO_NAME "rec_dense_bit" #elif defined REC_BIT # include @@ -68,9 +70,9 @@ make_bt(std::vector& text, const size_t) { ; return std::make_unique>(text, - arity, - 1, - leaf_length); + arity, + 1, + leaf_length); } # define ALGO_NAME "fp2" #elif defined LPF @@ -84,10 +86,10 @@ make_bt(std::vector& text, ; return std::unique_ptr>( pasta::make_block_tree_lpf_parallel(text, - arity, - leaf_length, - true, - threads)); + arity, + leaf_length, + true, + threads)); } # define ALGO_NAME "lpf" #elif defined PAR_SHARDED @@ -135,11 +137,11 @@ make_bt(std::vector& text, const size_t queue_size) { return std::make_unique< pasta::RecursiveBlockTreeSharded>(text, - arity, - 1, - leaf_length, - threads, - queue_size); + arity, + 1, + leaf_length, + threads, + queue_size); } # define ALGO_NAME "shard_sync_small" #elif defined REC_PAR_SHARDED @@ -281,31 +283,36 @@ int main(int argc, char** argv) { std::unique_ptr bv; std::vector text; { - std::string input; + const size_t input_size = + std::ifstream(file, std::ios::binary | std::ios::ate).tellg(); std::ifstream t(file); - std::stringstream buffer; - buffer << t.rdbuf(); - input = buffer.str(); if (make_bv) { if (one_chars.empty()) { // Interpret each character as 8 bits - bv = std::make_unique(input.size() * 8); + bv = std::make_unique(input_size * 8); std::span bytes = std::as_writable_bytes(bv->data()); - for (size_t i = 0; i < input.size(); ++i) { - bytes[i] = std::byte{static_cast(input[i])}; + for (size_t i = 0; i < input_size; ++i) { + char next_byte; + t >> next_byte; + bytes[i] = std::byte{static_cast(next_byte)}; } } else { // Interpret each character as a bit - bv = std::make_unique(input.size()); + bv = std::make_unique(input_size); std::array is_one{}; for (char c : one_chars) { is_one[static_cast(c)] = true; } - for (size_t i = 0; i < input.size(); ++i) { - (*bv)[i] = is_one[static_cast(input[i])]; + for (size_t i = 0; i < input_size; ++i) { + char next_byte; + t >> next_byte; + (*bv)[i] = is_one[static_cast(next_byte)]; } } } else { + std::stringstream buffer; + buffer << t.rdbuf(); + std::string input = buffer.str(); text = std::vector(input.begin(), input.end()); } } From 6a6ed99a184dd6c677578c865549e4249c101d3d Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Wed, 3 Jan 2024 21:47:12 +0100 Subject: [PATCH 83/92] fix integer overflow in select queries --- examples/build_bt.cpp | 43 +++++++++++++++---- .../pasta/block_tree/rec_bit_block_tree.hpp | 7 +-- .../block_tree/rec_dense_bit_block_tree.hpp | 19 +++----- 3 files changed, 46 insertions(+), 23 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 4dab905..93edc53 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -361,7 +361,6 @@ int main(int argc, char** argv) { Clock::now() - now) .count(); const size_t no_rs_space = bt->print_space_usage(); - std::cout << "\n"; bt->add_bit_rank_support(); auto elapsed_rs = std::chrono::duration_cast( Clock::now() - now) @@ -376,22 +375,27 @@ int main(int argc, char** argv) { return 0; } + std::cerr << "Start verification...\n"; + FlatRankSelect<> frs(*bv); #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif + std::cerr << "Access queries... " << std::flush; #pragma omp parallel for for (size_t i = 0; i < bv->size(); ++i) { const bool c = bt->access(i); if (c != (*bv)[i]) { std::osyncstream(std::cerr) - << "Access error at position " << i + << "\nAccess error at position " << i << "\nExpected: " << std::boolalpha << (*bv)[i] << "\nActual: " << c << std::noboolalpha << std::endl; exit(1); } } + std::cerr << "successful\n"; + std::cerr << "Rank 1 queries... " << std::flush; #pragma omp parallel for for (size_t i = 0; i < bv->size(); i++) { const size_t bt_rank = bt->rank1(i); @@ -399,38 +403,58 @@ int main(int argc, char** argv) { if (bv_rank != bt_rank) { std::osyncstream(std::cerr) - << "Rank error at position " << i << "\nExpected: " << bv_rank + << "\nRank one error at position " << i << "\nExpected: " << bv_rank << "\nActual: " << bt_rank << std::endl; throw std::runtime_error("oof"); } } + + std::cerr << "successful\n"; + std::cerr << "Rank 0 queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 0; i < bv->size(); i++) { + const size_t bt_rank = bt->rank0(i); + const size_t bv_rank = frs.rank0(i); + + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "\nRank zero error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + const size_t num_zeros = frs.rank0(bv->size()); const size_t num_ones = frs.rank1(bv->size()); + std::cerr << "successful\n"; + std::cerr << "Select 1 queries... " << std::flush; #pragma omp parallel for for (size_t i = 1; i <= num_ones; i++) { const size_t bv_rank = frs.select1(i); const size_t bt_rank = bt->select1(i); if (bv_rank != bt_rank) { - std::osyncstream(std::cerr) - << "Select one error at position " << i << "\nExpected: " << bv_rank - << "\nActual: " << bt_rank << std::endl; + std::osyncstream(std::cerr) << "\nSelect one error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; throw std::runtime_error("oof"); } } + std::cerr << "successful\n"; + std::cerr << "Select 0 queries... " << std::flush; #pragma omp parallel for for (size_t i = 1; i <= num_zeros; i++) { const size_t bv_rank = frs.select0(i); const size_t bt_rank = bt->select0(i); if (bv_rank != bt_rank) { - std::osyncstream(std::cerr) << "Select zero error at position " << i + std::osyncstream(std::cerr) << "\nSelect zero error at position " << i << "\nExpected: " << bv_rank << "\nActual: " << bt_rank << std::endl; throw std::runtime_error("oof"); } } - + std::cerr << "successful" << std::endl; } else { std::cout << " algo=" << ALGO_NAME; // Make text block tree @@ -447,10 +471,12 @@ int main(int argc, char** argv) { return 0; } + std::cerr << "Start verification...\n"; #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif + std::cerr << "Access queries... " << std::flush; #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); @@ -463,6 +489,7 @@ int main(int argc, char** argv) { } } } + std::cerr << "successful" << std::endl; return 0; } diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index e2df33a..406541f 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -276,7 +276,7 @@ class RecursiveBitBlockTree { pos += 8; ++byte_offset; } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + for (size_t bit = 0; bit < 8 && rank > 0; ++bit) { pos++; rank -= ((1 << bit) & byte) > 0; } @@ -389,7 +389,7 @@ class RecursiveBitBlockTree { pos += 8; byte_offset++; } else { - for (uint8_t bit = 0; bit < 8 && rank > 0; ++bit) { + for (size_t bit = 0; bit < 8 && rank > 0; ++bit) { pos++; rank -= ((1 << bit) & byte) == 0; } @@ -571,7 +571,8 @@ class RecursiveBitBlockTree { // space_usage += leaves_.size() * sizeof(uint8_t); space_usage += sdsl::size_in_bytes(compressed_leaves_); #ifdef BT_DBG - std::cout << "leaf size: " << sdsl::size_in_bytes(compressed_leaves_) << std::endl; + std::cout << "leaf size: " << sdsl::size_in_bytes(compressed_leaves_) + << std::endl; #endif space_usage += compress_map_.size(); diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp index 0a187e6..d1afbeb 100644 --- a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -21,17 +21,13 @@ #pragma once -#include #include #include -#include -#include #include #include #include #include #include -#include #include #include @@ -265,13 +261,12 @@ class RecursiveDenseBitBlockTree { current_block = block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - for (uint8_t bit = 0; rank > 0; ++bit, ++pos) { + for (size_t bit = 0; rank > 0; ++bit, ++pos) { rank -= (*leaf_bits_)[current_block * leaf_size + bit]; } return pos; } - /// FIXME DOES NOT WORK YET [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { const auto& top_is_internal = *block_tree_types_[0]; const auto& top_is_internal_rank = *block_tree_types_rs_[0]; @@ -365,7 +360,7 @@ class RecursiveDenseBitBlockTree { current_block = block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; - for (uint8_t bit = 0; rank > 0; ++bit, ++pos) { + for (size_t bit = 0; rank > 0; ++bit, ++pos) { rank -= !(*leaf_bits_)[current_block * leaf_size + bit]; } return pos; @@ -460,11 +455,11 @@ class RecursiveDenseBitBlockTree { return bit_index - rank1(bit_index); } - size_t print_space_usage() const { + [[nodiscard]] size_t print_space_usage() const { size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + sizeof(leaf_size); - auto delta_size = 0; + size_t delta_size = 0; for (const auto* bt : block_tree_types_) { if constexpr (types_is_block_tree) { space_usage += bt->print_space_usage(); @@ -489,8 +484,8 @@ class RecursiveDenseBitBlockTree { } delta_size = 0; for (const auto iv : block_tree_pointers_) { - space_usage += (int64_t)sdsl::size_in_bytes(*iv); - delta_size += (int64_t)sdsl::size_in_bytes(*iv); + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); ; } size_t ptr_cnt = 0; @@ -502,7 +497,7 @@ class RecursiveDenseBitBlockTree { #endif for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); - delta_size += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); #ifdef BT_DBG ptr_cnt += iv->size(); std::cout << "level " << level << " ptrs: " << iv->size() From 614697625a930766e4832f53bcff14d0659cf5c0 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 4 Jan 2024 21:52:11 +0100 Subject: [PATCH 84/92] prevent ankerl from adding a mixing step, hurting performance --- .../construction/rec_dense_bit_block_tree_sharded.hpp | 8 ++++---- include/pasta/block_tree/utils/MersenneHash.hpp | 3 ++- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index 80c2af6..96bbfd8 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -163,7 +163,7 @@ class RecursiveDenseBitBlockTreeSharded size_t generate_ns = 0; #endif #ifdef BT_DBG - std::cout << "using " << threads << " threads" << std::endl; + std::cerr << "using " << threads << " threads" << std::endl; #endif #ifdef BT_BENCH @@ -173,7 +173,7 @@ class RecursiveDenseBitBlockTreeSharded // Construct the pre-pruned tree level by level for (size_t level = 0; level < static_cast(tree_height); level++) { #ifdef BT_DBG - std::cout << "----------------- level " << level << " -----------------" + std::cerr << "----------------- level " << level << " -----------------" << std::endl; #endif @@ -235,7 +235,7 @@ class RecursiveDenseBitBlockTreeSharded } #ifdef BT_INSTRUMENT # if defined(BT_DBG) - std::cout << "pairs: " << (pairs_ns / 1'000'000) + std::cerr << "pairs: " << (pairs_ns / 1'000'000) << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) @@ -268,7 +268,7 @@ class RecursiveDenseBitBlockTreeSharded .count(); now = Clock::now(); # ifdef BT_DBG - std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; + std::cerr << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; # elif defined BT_BENCH std::cout << " prune=" << (prune_ns / 1'000'000); # endif diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index aa7c0b1..8eccbe6 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -41,7 +41,7 @@ static std::atomic_size_t mersenne_hash_equals = 0; static std::atomic_size_t mersenne_hash_collisions = 0; void print_hash_data() { - std::cout << "comparisons: " << mersenne_hash_comparisons + std::cerr << "comparisons: " << mersenne_hash_comparisons << ", equals: " << mersenne_hash_equals << ", collisions: " << mersenne_hash_collisions << ", percent equals: " @@ -205,6 +205,7 @@ class MersenneHash { template struct std::hash> { + //using is_avalanching = void; typename pasta::MersenneHash::uint128_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; From 1fc02d34a8123b8620041e60dc87f22f12369886 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 4 Jan 2024 22:06:15 +0100 Subject: [PATCH 85/92] update unordered_dense dependency --- extlib/unordered_dense | 2 +- include/pasta/block_tree/utils/MersenneHash.hpp | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/extlib/unordered_dense b/extlib/unordered_dense index 729896c..231e48c 160000 --- a/extlib/unordered_dense +++ b/extlib/unordered_dense @@ -1 +1 @@ -Subproject commit 729896c7ba8bbd9da5573679270133086d05b5dd +Subproject commit 231e48c9426bd21c273669e5fdcd042c146975cf diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index 8eccbe6..a39c9bb 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -205,7 +205,7 @@ class MersenneHash { template struct std::hash> { - //using is_avalanching = void; + using is_avalanching = void; typename pasta::MersenneHash::uint128_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; From 6b721d968584b772dd5024c9fe3de462ef6532c9 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Thu, 4 Jan 2024 22:27:17 +0100 Subject: [PATCH 86/92] replace lists with vectors In algorithms using the sharded map, the occurrences are only accessed by a single thread each. Therefore vectors are slightly faster. --- include/pasta/block_tree/utils/sharded_util.hpp | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp index f973b4b..c494eda 100644 --- a/include/pasta/block_tree/utils/sharded_util.hpp +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -130,7 +130,7 @@ struct PairOccurrences { /// We're using an std::list here instead of an std::vector, since the /// reallocation upon insertion lead to issues during parallel access, when /// another thread tries to access the vector during reallocation. - std::list occurrences; + std::vector occurrences; /// @brief Initialize the occurrences of a hashed block pair. /// @@ -184,7 +184,7 @@ struct BlockOccurrences { /// @brief A list of block indices in which the content of the hashed block /// occurs - std::list occurrences; + std::vector occurrences; /// @brief Initialize the occurrences of a hashed block. /// @@ -351,4 +351,4 @@ struct Keep { } }; -} // namespace pasta \ No newline at end of file +} // namespace pasta From f8b63ed9d32d49e4c1728525b3c758a1045dd602 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Fri, 5 Jan 2024 22:23:29 +0100 Subject: [PATCH 87/92] fix BT_DBG macro --- .../construction/rec_block_tree_sharded.hpp | 363 +++++++++--------- .../rec_dense_bit_block_tree_sharded.hpp | 271 +++---------- .../pasta/block_tree/utils/MersenneHash.hpp | 3 +- .../block_tree/utils/MersenneRabinKarp.hpp | 3 +- .../block_tree/utils/sync_sharded_map.hpp | 6 +- 5 files changed, 233 insertions(+), 413 deletions(-) diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index c8b771e..1dea9aa 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -148,12 +148,10 @@ class RecursiveBlockTreeSharded top_level.num_blocks = top_level.block_starts->size(); #ifdef BT_INSTRUMENT - +# ifdef BT_BENCH const size_t setup_ns = std::chrono::duration_cast(Clock::now() - now) .count(); - -# ifdef BT_BENCH std::cout << " setup=" << setup_ns; # endif @@ -386,8 +384,8 @@ class RecursiveBlockTreeSharded const size_t pair_size = 2 * block_size; const auto& block_starts = *level.block_starts; - // Hash every window and determine for all block pairs whether - // they have previous occurrences. + // Hash every window and determine for all block pairs + // whether they have previous occurrences. const size_t segment_size = std::max(1, ceil_div(num_block_pairs, num_threads)); @@ -403,16 +401,17 @@ class RecursiveBlockTreeSharded pair_size, internal::sharded::PRIME); for (size_t i = start; i < end; ++i) { - // If the next block is not adjacent, we cannot hash the pair - // starting at the current block + // If the next block is not adjacent, we cannot hash the + // pair starting at the current block if (!level.next_is_adjacent(i)) { continue; } rk.restart(block_starts[i]); // Move the hasher to the current block pair RabinKarpHash hash = rk.current_hash(); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry + // Try to find the hash in the map, insert a new entry if + // it doesn't exist, and add the current block to the + // entry shard.insert(hash, i); } } else { @@ -427,8 +426,9 @@ class RecursiveBlockTreeSharded internal::sharded::mix_select(hash_value), block_start, block_size); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry + // Try to find the hash in the map, insert a new entry if + // it doesn't exist, and add the current block to the + // entry shard.insert(hash, i); } } @@ -736,7 +736,7 @@ class RecursiveBlockTreeSharded finish_idle_ns, \ total_idle_ns, \ handle_queue_ns, \ - scan_hits, + scan_hits, \ internal::sharded::HASH_MASKS) #else # pragma omp parallel default(none) num_threads(threads) \ @@ -750,200 +750,185 @@ class RecursiveBlockTreeSharded internal::sharded::HASH_MASKS) #endif { - const size_t num_threads = omp_get_num_threads(); - const size_t thread_id = omp_get_thread_num(); - typename BlockMap::Shard shard = links.get_shard(thread_id); - const size_t block_size = - std::min(level_data.block_size, text.size()); - const std::vector& block_starts = - *level_data.block_starts; - // Number of total iterations the for loop should do - const size_t num_total_iterations = - level_data.num_blocks - is_padded - 1; - // The number of iterations each thread should do - const size_t segment_size = - ceil_div(num_total_iterations, num_threads); - // The start and end index of the current thread's segment - const size_t start = thread_id * segment_size; - const size_t end = - std::min(num_total_iterations, - (thread_id + 1) * segment_size); - - // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - internal::sharded::SIGMA, - block_starts[0], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } else { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[block_size / - sizeof(input_type)]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const input_type* block_start_ptr = - text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & - HASH_MASK; - RabinKarpHash hash( - text, - internal::sharded::mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); - } - } - - if (const size_t thread_order = - num_done.fetch_add(1, std::memory_order_acq_rel) + 1; - thread_order == num_threads) { - last_done.store(true, std::memory_order_release); - } - - while (!last_done.load(std::memory_order::acquire)) { - shard.handle_queue_sync(false); - } - barrier.arrive_and_drop(); - shard.handle_queue(); + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); #pragma omp barrier #pragma omp single #ifdef BT_INSTRUMENT - { - b_hash_blocks_ns += - std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); - } + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } - tlx::Aggregate thread_scan_hits; + tlx::Aggregate thread_scan_hits; #else { } #endif - // Hash every window and find the first occurrences for every - // block. - if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - internal::sharded::SIGMA, - block_starts[start], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != - block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } - } - } + ); + } + } + } #ifdef BT_INSTRUMENT - auto& start_idle = shard.start_idle_ns(); - auto& finish_idle = shard.finish_idle_ns(); - auto& handle_queue = shard.handle_queue_ns(); + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); # pragma omp critical - { - start_idle_ns.add(start_idle.sum()); - finish_idle_ns.add(finish_idle.sum()); - total_idle_ns.add(start_idle.sum() + finish_idle.sum()); - handle_queue_ns.add(handle_queue.sum()); - scan_hits += thread_scan_hits; - }; + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; #endif - } + } #ifdef BT_INSTRUMENT - b_scan_blocks_ns += - std::chrono::duration_cast( - Clock::now() - now) - .count(); - now = Clock::now(); + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); # ifdef BT_DBG - tlx::Aggregate map_loads; - - for (size_t load : links.map_loads()) { - map_loads.add(load); - } - - print_aggregate("Block Map Loads ", map_loads); - print_aggregate("Block Map Hits ", scan_hits); - print_aggregate("Block Idle (ms) ", - total_idle_ns, - 1'000'000); - print_aggregate("Block Handle Queue (ms)", - finish_idle_ns, - 1'000'000); - - BT_ASSERT(links.num_inserts_.load() == links.size()); + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); # endif #endif - // By this point, the map should contain the first occurrences - // of every respective block's content. We then fill the - // pointers and offsets with this data and increment counters - // accordingly - links.for_each([&level_data](const RabinKarpHash&, - const BlockOccurrences& occs) { - auto first_occ = occs.first_occ.load(); - for (const size_type occ : occs.occurrences) { - if (occ == first_occ.block || - (first_occ.offset > 0 && occ == first_occ.block + 1)) { - continue; - } - - (*level_data.pointers)[occ] = first_occ.block; - (*level_data.offsets)[occ] = first_occ.offset; - const bool is_back_block = !(*level_data.is_internal)[occ]; - (*level_data.counters)[first_occ.block] += 1; - (*level_data.counters)[first_occ.block + 1] += - is_back_block && (first_occ.offset > 0); - } - }); + // By this point, the map should contain the first occurrences + // of every respective block's content. We then fill the + // pointers and offsets with this data and increment counters + // accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); #ifdef BT_INSTRUMENT - b_update_blocks_ns += - std::chrono::duration_cast( - Clock::now() - now) - .count(); + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); #endif } @@ -1115,14 +1100,14 @@ class RecursiveBlockTreeSharded sdsl::util::bit_compress(*offsets); if constexpr (recursion_level > 0) { - auto* bt = - new RecursiveDenseBitBlockTreeSharded( - *top_level.is_internal, - this->tau_, - this->s_, - this->max_leaf_length_, - threads, - queue_size); + auto* bt = new RecursiveDenseBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); this->block_tree_types_.push_back(bt); this->block_tree_types_.back()->add_bit_rank_support(threads); this->block_tree_types_rs_.push_back(bt); diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index 96bbfd8..8a44c39 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -150,11 +150,11 @@ class RecursiveDenseBitBlockTreeSharded #ifdef BT_INSTRUMENT +# ifdef BT_BENCH const size_t setup_ns = std::chrono::duration_cast(Clock::now() - now) .count(); -# ifdef BT_BENCH std::cout << " setup=" << setup_ns; # endif @@ -181,38 +181,22 @@ class RecursiveDenseBitBlockTreeSharded TimePoint now = Clock::now(); #endif LevelData& current = levels.back(); - if (2 * static_cast(current.block_size) > 8) { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_block_pairs(text, - current, - is_padded, - threads, - queue_size); - } + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); #ifdef BT_INSTRUMENT pairs_ns += std::chrono::duration_cast( Clock::now() - now) .count(); now = Clock::now(); #endif - if (static_cast(current.block_size) > 8) { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } else { - scan_blocks(text, - current, - is_padded, - threads, - queue_size); - } + scan_blocks(text, + current, + is_padded, + threads, + queue_size); #ifdef BT_INSTRUMENT blocks_ns += std::chrono::duration_cast( Clock::now() - now) @@ -399,22 +383,7 @@ class RecursiveDenseBitBlockTreeSharded // doesn't exist, and add the current block to the entry shard.insert(hash, i); } - } /*else { - const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start, - block_size); - // Try to find the hash in the map, insert a new entry if it - // doesn't exist, and add the current block to the entry - shard.insert(hash, i); - } - }*/ + } if (const size_t thread_order = threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -446,45 +415,26 @@ class RecursiveDenseBitBlockTreeSharded #endif if (start < static_cast(num_block_pairs)) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - block_starts[start], - pair_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - if (block_starts[i] != static_cast(rk.init_)) { - rk.restart(block_starts[i]); - } - scan_windows_in_block_pair(rk, - map, - block_size, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); + RabinKarp rk(text, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; } - } else { - for (size_t i = start; i < end; ++i) { - if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { - continue; - } - scan_windows_in_block_pair_identity(text, - block_starts[i], - pair_size, - map, - block_size, - i + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } + ); } } @@ -615,46 +565,7 @@ class RecursiveDenseBitBlockTreeSharded occurrences.update(current_block_index); } } - /* - static inline void - scan_windows_in_block_pair_identity(const pasta::BitVector& text, - const size_t block_start, - const size_t pair_size, - BlockPairMap& map, - const size_t num_iterations, - const size_type current_block_index - #ifdef BT_INSTRUMENT - , - tlx::Aggregate& agg - #endif - ) { - const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_t offset = 0; offset < num_iterations; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash current_hash(text, - internal::sharded::mix_select(hash_value), - block_start + offset, - pair_size); - // Find the hash of the current window among the hashed block - // pairs. - auto found = map.find(current_hash); - if (found == map.end()) { - #ifdef BT_INSTRUMENT - agg.add(0); - continue; - } else { - agg.add(100); - #else - continue; - #endif - } - PairOccurrences& occurrences = found->second; - occurrences.update(current_block_index); - } - } - */ + /// @brief Determine the positions for each block's earliest occurrence if /// there is any. /// @@ -739,31 +650,12 @@ class RecursiveDenseBitBlockTreeSharded (thread_id + 1) * segment_size); // Hash each block and store their hashes in the map - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - block_starts[0], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - rk.restart(block_starts[i]); - RabinKarpHash hash = rk.current_hash(); - shard.insert(hash, {i, 0}); - } - } /*else { - const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[block_size]; - for (size_t i = start; i < end; ++i) { - const size_t block_start = block_starts[i]; - const uint8_t* block_start_ptr = text.data() + block_start; - const uint64_t hash_value = - pasta::copy_le(block_start_ptr) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start, - block_size); - - shard.insert(hash, {i, 0}); - } - }*/ + RabinKarp rk(text, block_starts[0], block_size, internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } if (const size_t thread_order = num_done.fetch_add(1, std::memory_order_acq_rel) + 1; @@ -796,44 +688,26 @@ class RecursiveDenseBitBlockTreeSharded // Hash every window and find the first occurrences for every // block. if (start < block_starts.size() - is_padded) { - if constexpr (use_hash == UseHash::RABIN_KARP) { - RabinKarp rk(text, - block_starts[start], - block_size, - internal::sharded::PRIME); - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - if (static_cast(rk.init_) != block_starts[i]) { - rk.restart(block_starts[i]); - } - scan_windows_in_block(rk, - links, - level_data, - i -#ifdef BT_INSTRUMENT - , - thread_scan_hits -#endif - ); + RabinKarp rk(text, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; } - } else { - for (size_t i = start; i < end; ++i) { - if (!level_data.next_is_adjacent(i)) { - continue; - } - scan_windows_in_block_identity(text, - block_starts[i], - links, - level_data, - i + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i #ifdef BT_INSTRUMENT - , - thread_scan_hits + , + thread_scan_hits #endif - ); - } + ); } } #ifdef BT_INSTRUMENT @@ -938,45 +812,6 @@ class RecursiveDenseBitBlockTreeSharded occurrences.update(current_block_index, offset); } } - /* - static void - scan_windows_in_block_identity(const std::span& text, - const size_t block_start, - BlockMap& links, - LevelData& level_data, - const size_type current_block_index - #ifdef BT_INSTRUMENT - , - tlx::Aggregate& hits - #endif - ) { - const uint64_t HASH_MASK = - internal::sharded::HASH_MASKS[level_data.block_size]; - const uint8_t* block_start_ptr = text.data() + block_start; - for (size_type offset = 0; offset < level_data.block_size; ++offset) { - const uint64_t hash_value = - pasta::copy_le(block_start_ptr + offset) & HASH_MASK; - RabinKarpHash hash(text, - internal::sharded::mix_select(hash_value), - block_start + offset, - level_data.block_size); - // Find all blocks in the multimap that match our hash - auto found = links.find(hash); - if (found == links.end()) { - #ifdef BT_INSTRUMENT - hits.add(0.0); - continue; - } else { - hits.add(100.0); - #else - continue; - #endif - } - BlockOccurrences& occurrences = found->second; - occurrences.update(current_block_index, offset); - } - } - */ /// @brief Generate the block size, number of block and block start indices /// for the next level. diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index a39c9bb..4ae7c6e 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -144,7 +144,7 @@ class MersenneHash { constexpr MersenneHash& operator=(const MersenneHash& other) = default; constexpr MersenneHash& operator=(MersenneHash&& other) = default; - [[gnu::noinline]] bool operator==(const MersenneHash& other) const { + inline bool operator==(const MersenneHash& other) const { #ifdef BT_INSTRUMENT ++mersenne_hash_comparisons; #endif @@ -205,7 +205,6 @@ class MersenneHash { template struct std::hash> { - using is_avalanching = void; typename pasta::MersenneHash::uint128_t operator()(const pasta::MersenneHash& hS) const { return hS.hash_; diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 8b48583..1d3a5bf 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -274,8 +274,7 @@ class MersenneRabinKarp { return get_bit(idx); } - [[nodiscard, gnu::noinline]] constexpr bool - get_bit(const size_t bit_index) const { + [[nodiscard]] constexpr bool get_bit(const size_t bit_index) const { return text_[bit_index]; } }; diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp index 735b876..8628c6f 100644 --- a/include/pasta/block_tree/utils/sync_sharded_map.hpp +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -32,6 +32,8 @@ #include #include +#include + namespace pasta { enum Whereabouts { NOWHERE, IN_MAP, IN_QUEUE }; @@ -140,7 +142,7 @@ class SyncShardedMap { SeqHashMap& map_; Queue& task_queue_; std::atomic_size_t& task_count_; -#ifdef BT_INSTRUMENT +#if defined BT_INSTRUMENT || defined BT_DBG tlx::Aggregate start_idle_ns_; tlx::Aggregate handle_queue_ns_; tlx::Aggregate finish_idle_ns_; @@ -153,7 +155,7 @@ class SyncShardedMap { map_(sharded_map_.map_[thread_id]), task_queue_(sharded_map_.task_queue_[thread_id]), task_count_(sharded_map.task_count_[thread_id]) -#ifdef BT_INSTRUMENT +#if defined BT_INSTRUMENT || defined BT_DBG , start_idle_ns_(), handle_queue_ns_(), From 6424a81c5cead7b7449ba7780e7669b9a356b551 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 6 Jan 2024 14:41:34 +0100 Subject: [PATCH 88/92] fix segfault caused by not finding prev occurrence --- .../rec_bit_block_tree_sharded.hpp | 23 +++++++++++-------- .../pasta/block_tree/rec_bit_block_tree.hpp | 14 ++++------- 2 files changed, 19 insertions(+), 18 deletions(-) diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp index c36d20c..b7c149b 100644 --- a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -139,7 +139,8 @@ class RecursiveBitBlockTreeSharded // Prepare the top level levels.emplace_back(0, top_block_size, text_len / top_block_size); LevelData& top_level = levels.back(); - top_level.block_starts->reserve(internal::sharded::ceil_div(text_len, top_level.block_size)); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); for (size_type i = 0; i < text_len; i += top_level.block_size) { top_level.block_starts->push_back(i); } @@ -378,8 +379,9 @@ class RecursiveBitBlockTreeSharded // Hash every window and determine for all block pairs whether // they have previous occurrences. - const size_t segment_size = - std::max(1, internal::sharded::ceil_div(num_block_pairs, num_threads)); + const size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, num_threads)); // Start and end index of the current thread's segment const auto start = thread_id * segment_size; @@ -744,9 +746,10 @@ class RecursiveBitBlockTreeSharded std::min(level_data.block_size, text.size()); const std::vector& block_starts = *level_data.block_starts; // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + const size_t num_total_iterations = level_data.num_blocks - is_padded; // The number of iterations each thread should do - const size_t segment_size = internal::sharded::ceil_div(num_total_iterations, num_threads); + const size_t segment_size = + internal::sharded::ceil_div(num_total_iterations, num_threads); // The start and end index of the current thread's segment const size_t start = thread_id * segment_size; const size_t end = std::min(num_total_iterations, @@ -1200,7 +1203,9 @@ class RecursiveBitBlockTreeSharded const size_type last_block_parent_start = previous_level.block_starts->back(); const size_type block_size = level.block_size; - new_size += internal::sharded::ceil_div(text_len - last_block_parent_start, block_size); + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); } previous_level.block_starts.reset(); const size_type num_internal = new_num_internal[level_index]; @@ -1223,20 +1228,20 @@ class RecursiveBitBlockTreeSharded // values starting after i will still be valid pointers // This contains the number of pruned blocks before the block i std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { + for (size_type i = 0; i < level.num_blocks; ++i) { const size_type ptr = (*level.pointers)[i]; prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree if (ptr == internal::sharded::PRUNED) { - num_pruned++; + ++num_pruned; continue; } // Add it to the is_internal bit vector const bool block_is_internal = (*level.is_internal)[i]; (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; + ++num_non_pruned; if (block_is_internal) { continue; diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp index 406541f..775e219 100644 --- a/include/pasta/block_tree/rec_bit_block_tree.hpp +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -510,11 +510,11 @@ class RecursiveBitBlockTree { return bit_index - rank1(bit_index); } - size_t print_space_usage() const { + [[nodiscard]] size_t print_space_usage() const { size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + sizeof(leaf_size); - auto delta_size = 0; + size_t delta_size = 0; for (const auto* bt : block_tree_types_) { if constexpr (types_is_block_tree) { space_usage += bt->print_space_usage(); @@ -539,8 +539,8 @@ class RecursiveBitBlockTree { } delta_size = 0; for (const auto iv : block_tree_pointers_) { - space_usage += (int64_t)sdsl::size_in_bytes(*iv); - delta_size += (int64_t)sdsl::size_in_bytes(*iv); + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); ; } #ifdef BT_DBG @@ -549,7 +549,7 @@ class RecursiveBitBlockTree { #endif for (const auto iv : block_tree_offsets_) { space_usage += sdsl::size_in_bytes(*iv); - delta_size += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); } #ifdef BT_DBG std::cout << "offs size: " << delta_size << std::endl; @@ -586,8 +586,6 @@ class RecursiveBitBlockTree { } rank_support = true; - // FIXME For the last level where block_tree_types_ is a bitvec, using - // multiple threads doesn't work for some reason if constexpr (recursion_level == 0) { threads = 1; } @@ -603,8 +601,6 @@ class RecursiveBitBlockTree { block_tree_pointers_[level]->size()); } - // FIXME: breaks if parallelism is used - // #pragma omp parallel for default(none) num_threads(threads) for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { bit_rank_block(0, block); } From 71e0e032f8034623b387bf08c65617e9bd1f86e5 Mon Sep 17 00:00:00 2001 From: Etienne Palanga Date: Sat, 6 Jan 2024 14:53:29 +0100 Subject: [PATCH 89/92] small fix to sharded algorithm --- .../block_tree/construction/rec_block_tree_sharded.hpp | 2 +- .../construction/rec_dense_bit_block_tree_sharded.hpp | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp index 1dea9aa..6089d6e 100644 --- a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -757,7 +757,7 @@ class RecursiveBlockTreeSharded std::min(level_data.block_size, text.size()); const std::vector& block_starts = *level_data.block_starts; // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + const size_t num_total_iterations = level_data.num_blocks - is_padded; // The number of iterations each thread should do const size_t segment_size = ceil_div(num_total_iterations, num_threads); // The start and end index of the current thread's segment diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp index 8a44c39..0ff72cf 100644 --- a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -640,7 +640,7 @@ class RecursiveDenseBitBlockTreeSharded std::min(level_data.block_size, text.size()); const std::vector& block_starts = *level_data.block_starts; // Number of total iterations the for loop should do - const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + const size_t num_total_iterations = level_data.num_blocks - is_padded; // The number of iterations each thread should do const size_t segment_size = internal::sharded::ceil_div(num_total_iterations, num_threads); @@ -1077,20 +1077,20 @@ class RecursiveDenseBitBlockTreeSharded // values starting after i will still be valid pointers // This contains the number of pruned blocks before the block i std::vector& prefix_pruned_blocks = *level.pointers; - for (size_type i = 0; i < level.num_blocks; i++) { + for (size_type i = 0; i < level.num_blocks; ++i) { const size_type ptr = (*level.pointers)[i]; prefix_pruned_blocks[i] = num_pruned; // If the current block is not pruned, add it to the new tree if (ptr == internal::sharded::PRUNED) { - num_pruned++; + ++num_pruned; continue; } // Add it to the is_internal bit vector const bool block_is_internal = (*level.is_internal)[i]; (*is_internal)[num_non_pruned] = block_is_internal; - num_non_pruned++; + ++num_non_pruned; if (block_is_internal) { continue; From 22ca931c0bc0382cb50130f686b86151d27cdf72 Mon Sep 17 00:00:00 2001 From: Florian Kurpicz Date: Thu, 29 Aug 2024 11:00:38 +0200 Subject: [PATCH 90/92] Add verification to RESULT output --- examples/build_bt.cpp | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp index 93edc53..519e12c 100644 --- a/examples/build_bt.cpp +++ b/examples/build_bt.cpp @@ -465,31 +465,29 @@ int main(int argc, char** argv) { std::cout << " rec=" << RECURSION_LEVELS; std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); - std::cout << std::endl; if (!verify) { + std::cout << std::endl; return 0; } - std::cerr << "Start verification...\n"; + // std::cerr << "Start verification...\n"; #if defined BT_INSTRUMENT && defined BT_DBG pasta::print_hash_data(); #endif - std::cerr << "Access queries... " << std::flush; + // std::cerr << "Access queries... " << std::flush; #pragma omp parallel for for (size_t i = 0; i < text.size(); ++i) { const auto c = bt->access(i); if (c != text[i]) { - std::osyncstream(std::cerr) - << "Error at position " << i - << "\nExpected: " << static_cast(text[i]) - << "\nActual: " << static_cast(c) << std::endl; - exit(1); + #pragma omp critical + std::cout << " verification=failed" << std::endl; + std::exit(-1); } } } - std::cerr << "successful" << std::endl; + std::cout << " verification=passed" << std::endl; return 0; } From 42002bf0e0a47f84cd43252614c47f050d39e114 Mon Sep 17 00:00:00 2001 From: Florian Kurpicz Date: Mon, 13 Apr 2026 21:32:57 +0200 Subject: [PATCH 91/92] Update ctest.yml --- .github/workflows/ctest.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml index afcaff6..f7bfef6 100644 --- a/.github/workflows/ctest.yml +++ b/.github/workflows/ctest.yml @@ -10,7 +10,6 @@ jobs: strategy: matrix: os: [ubuntu-latest] - compiler: [{cpp: g++-10, c: gcc-10}] runs-on: ${{ matrix.os }} From 6606b4aa6725c2529ddecbdc2ba564edc7335d27 Mon Sep 17 00:00:00 2001 From: Florian Kurpicz Date: Mon, 13 Apr 2026 21:43:16 +0200 Subject: [PATCH 92/92] Update ctest.yml --- .github/workflows/ctest.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml index f7bfef6..3d51387 100644 --- a/.github/workflows/ctest.yml +++ b/.github/workflows/ctest.yml @@ -27,9 +27,6 @@ jobs: - name: Configure CMake run: cmake --preset=release -DPASTA_BLOCK_TREE_BUILD_TESTS=ON - env: - CC: gcc-10 - CXX: g++-10 - name: Build run: cmake --build ${{github.workspace}}/build/