diff --git a/.github/workflows/ctest.yml b/.github/workflows/ctest.yml index afcaff6..3d51387 100644 --- a/.github/workflows/ctest.yml +++ b/.github/workflows/ctest.yml @@ -10,7 +10,6 @@ jobs: strategy: matrix: os: [ubuntu-latest] - compiler: [{cpp: g++-10, c: gcc-10}] runs-on: ${{ matrix.os }} @@ -28,9 +27,6 @@ jobs: - name: Configure CMake run: cmake --preset=release -DPASTA_BLOCK_TREE_BUILD_TESTS=ON - env: - CC: gcc-10 - CXX: g++-10 - name: Build run: cmake --build ${{github.workspace}}/build/ diff --git a/.gitignore b/.gitignore index 42afabf..93485af 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,12 @@ -/build \ No newline at end of file +/build* +/cmake-build-* +.cache +compile_commands.json +.idea/* +perf.data* +.pdf +*.txt +!CMakeLists.txt +debug +dev/ +.vscode \ No newline at end of file diff --git a/.gitmodules b/.gitmodules index e832482..0322647 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,6 +1,37 @@ [submodule "extlib/libsais"] path = extlib/libsais url = https://github.com/IlyaGrebnov/libsais +[submodule "extlib/growt"] + path = extlib/growt + url = https://github.com/TooBiased/growt +[submodule "extlib/parallel-hashmap"] + path = extlib/parallel-hashmap + url = https://github.com/greg7mdp/parallel-hashmap +[submodule "extlib/waitfree-mpsc-queue"] + path = extlib/waitfree-mpsc-queue + url = https://github.com/dbittman/waitfree-mpsc-queue +[submodule "extlib/Jiffy"] + path = extlib/Jiffy + url = https://github.com/DolevAdas/Jiffy +[submodule "fastwfc"] + path = extlib/fastwfc/fast-wait-free-queue + url = https://github.com/chaoran/fast-wait-free-queue +[submodule "extlib/Jiffy-1"] + path = extlib/Jiffy-1 + url = https://github.com/quininer/Jiffy-1 + branch = fix-atomic +[submodule "extlib/sdsl-lite"] + path = extlib/sdsl-lite + url = https://github.com/Skadic/sdsl-lite +[submodule "extlib/--force"] + path = extlib/--force + url = https://github.com/rizkg/BBHash +[submodule "extlib/parlayhash"] + path = extlib/parlayhash + url = https://github.com/cmuparlay/parlayhash +[submodule "extlib/unordered_dense"] + path = extlib/unordered_dense + url = https://github.com/martinus/unordered_dense [submodule "extlib/doxygen-awesome-css"] path = extlib/doxygen-awesome-css url = https://github.com/jothepro/doxygen-awesome-css.git diff --git a/CMakeLists.txt b/CMakeLists.txt index 2f3ef8a..1c47692 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -24,15 +24,23 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR) set(CMAKE_CXX_STANDARD 20) set(CMAKE_CXX_STANDARD_REQUIRED ON) -include(FetchContent) - project(pasta_block_tree) +# Generate compile_commands.json +set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "") + ## Build tests option(PASTA_BLOCK_TREE_BUILD_TESTS "Build blocktree's tests." OFF) option(PASTA_BLOCK_TREE_BUILD_EXAMPLES - "Build blocktree's benchmarks." OFF) + "Build blocktree's benchmarks." OFF) +option(PASTA_BLOCK_TREE_DEBUG + "Print debug information" OFF) +option(PASTA_BLOCK_TREE_BENCH + "Enable outputting benchmark information" OFF) + +include(ExternalProject) +include(FetchContent) FetchContent_Declare( tlx @@ -49,8 +57,7 @@ FetchContent_Declare( FetchContent_MakeAvailable(tlx pasta_bit_vector) # Optional test -if(PASTA_BLOCK_TREE_BUILD_TESTS) - include(FetchContent) +if (PASTA_BLOCK_TREE_BUILD_TESTS) FetchContent_Declare( googletest GIT_REPOSITORY https://github.com/google/googletest.git @@ -59,27 +66,80 @@ if(PASTA_BLOCK_TREE_BUILD_TESTS) set(gtest_force_shared_crt ON CACHE BOOL "" FORCE) FetchContent_MakeAvailable(googletest) enable_testing() - add_subdirectory(tests) include(GoogleTest) -endif() -if(PASTA_BLOCK_TREE_BUILD_EXAMPLES) - add_executable(example - examples/block_tree_construction.cpp) - target_link_libraries(example - pasta_block_tree) -endif() + add_subdirectory(tests) +endif () +if (PASTA_BLOCK_TREE_BUILD_EXAMPLES) + add_executable(block_tree_construction + examples/block_tree_construction.cpp) + target_link_libraries(block_tree_construction + pasta_block_tree) + + add_executable(build_bt + examples/build_bt.cpp) + target_link_libraries(build_bt + pasta_block_tree) + if (PASTA_BLOCK_TREE_DEBUG) + target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) + target_compile_definitions(build_bt PRIVATE BT_DBG) + endif () + + if (PASTA_BLOCK_TREE_BENCH) + target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT) + target_compile_definitions(build_bt PRIVATE BT_BENCH) + endif () +endif () set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE) add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais) +set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode") +add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort) + +add_library(waitfree-mpsc-queue + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c) +target_include_directories(waitfree-mpsc-queue PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue) + +add_library(jiffy INTERFACE) +target_include_directories(jiffy INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy) +add_library(jiffy1 INTERFACE) +target_include_directories(jiffy1 INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy-1) + add_library(pasta_block_tree INTERFACE) target_include_directories(pasta_block_tree INTERFACE - ${CMAKE_CURRENT_SOURCE_DIR}/include) + ${CMAKE_CURRENT_SOURCE_DIR}/include) + +# sdsl +file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp) +add_library(sdsl STATIC ${sdsl_sources}) +target_include_directories(sdsl SYSTEM PUBLIC + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include) +target_include_directories(sdsl SYSTEM PUBLIC + ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include) +target_link_libraries(sdsl PUBLIC divsufsort64) +set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w") target_link_libraries(pasta_block_tree INTERFACE - libsais - pasta_bit_vector - sdsl - tlx) + libsais + pasta_bit_vector + tlx + waitfree-mpsc-queue + sdsl + #jiffy + jiffy1 + # libzstd_static +) + +target_include_directories(pasta_block_tree INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt) +target_include_directories(pasta_block_tree INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parallel-hashmap/parallel_hashmap) +target_include_directories(pasta_block_tree SYSTEM INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parlayhash/include) +target_include_directories(pasta_block_tree SYSTEM INTERFACE + ${CMAKE_CURRENT_SOURCE_DIR}/extlib/unordered_dense/include) ################################################################################ diff --git a/CMakePresets.json b/CMakePresets.json index 7282ac3..df26352 100644 --- a/CMakePresets.json +++ b/CMakePresets.json @@ -1,65 +1,124 @@ { - "version": 2, - "cmakeMinimumRequired": { - "major": 3, - "minor": 20, - "patch": 0 + "version": 2, + "cmakeMinimumRequired": { + "major": 3, + "minor": 20, + "patch": 0 + }, + "configurePresets": [ + { + "name": "default", + "displayName": "Default", + "description": "Default build options", + "hidden": true, + "generator": "Ninja", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", + "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", + "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler", + "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -static-libasan -fsanitize=address -fsanitize=leak -fsanitize=undefined" + } }, - "configurePresets": [ - { - "name": "default", - "displayName": "Default", - "description": "Default build options", - "hidden": true, - "generator": "Ninja", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always", - "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3", - "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3", - "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address" - } - }, - { - "name": "release", - "displayName": "Release", - "inherits": "default", - "binaryDir": "${sourceDir}/build", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Release" - } - }, - { - "name": "relwithdeb", - "displayName": "ReleaseWithDebugInfo", - "inherits": "default", - "binaryDir": "${sourceDir}/build_with_debug_info", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "RelWithDebInfo" - } - }, - { - "name": "debug", - "displayName": "Debug", - "inherits": "default", - "binaryDir": "${sourceDir}/debug", - "cacheVariables": { - "CMAKE_BUILD_TYPE": "Debug" - } - } - ], - "buildPresets": [ - { - "name": "release", - "configurePreset": "release" - }, - { - "name": "relwithdeb", - "configurePreset": "relwithdeb" - }, - { - "name": "debug", - "configurePreset": "debug" - } - ] + { + "name": "release", + "displayName": "Release", + "inherits": "default", + "binaryDir": "${sourceDir}/build", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Release" + } + }, + { + "name": "relwithdeb", + "displayName": "ReleaseWithDebugInfo", + "inherits": "default", + "binaryDir": "${sourceDir}/build_with_debug_info", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "RelWithDebInfo" + } + }, + { + "name": "debug", + "displayName": "Debug", + "inherits": "default", + "binaryDir": "${sourceDir}/debug", + "cacheVariables": { + "CMAKE_BUILD_TYPE": "Debug" + } + }, + { + "name": "ninja-multi", + "displayName": "Ninja Multi-Config", + "description": "Default build using Ninja Multi-Config generator", + "generator": "Ninja Multi-Config", + "binaryDir": "${sourceDir}/build_multi", + "inherits": "default", + "cacheVariables": { + "PASTA_BLOCK_TREE_BUILD_TESTS": "ON", + "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON" + } + }, + { + "name": "bench", + "displayName": "Behcnmarking Configuration", + "description": "Release build with extended debug information", + "generator": "Ninja", + "binaryDir": "${sourceDir}/build_bench", + "inherits": "default", + "cacheVariables": { + "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON", + "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -lprofiler", + "CMAKE_BUILD_TYPE": "Release", + "PASTA_BLOCK_TREE_BENCH": "ON" + } + } + ], + "buildPresets": [ + { + "name": "release", + "configurePreset": "release" + }, + { + "name": "relwithdeb", + "configurePreset": "relwithdeb" + }, + { + "name": "debug", + "configurePreset": "debug" + }, + { + "name": "release-multi", + "configurePreset": "ninja-multi", + "configuration": "Release" + }, + { + "name": "relwithdeb-multi", + "configurePreset": "ninja-multi", + "configuration": "RelWithDebInfo" + }, + { + "name": "debug-multi", + "configurePreset": "ninja-multi", + "configuration": "Debug" + }, + { + "name": "bench", + "configurePreset": "bench" + } + ], + "testPresets": [ + { + "name": "default", + "configurePreset": "ninja-multi", + "output": { + "outputOnFailure": true + }, + "execution": { + "noTestsAction": "error", + "stopOnFailure": true + }, + "configuration": "RelWithDebInfo" + } + ] } diff --git a/examples/build_bt.cpp b/examples/build_bt.cpp new file mode 100644 index 0000000..519e12c --- /dev/null +++ b/examples/build_bt.cpp @@ -0,0 +1,495 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#include +#include +#include +#include +#include +#include + +#define REC_PAR_SHARDED +#define REC_DENSE_BIT + +using SizeType = int64_t; + +#if defined REC_BIT || defined REC_DENSE_BIT || defined REC_PAR_SHARDED +constexpr size_t RECURSION_LEVELS = 0; +#else +constexpr size_t RECURSION_LEVELS = 0; +#endif + +#if defined REC_DENSE_BIT +# include +using BBT = + pasta::RecursiveDenseBitBlockTreeSharded; +# define BIT_ALGO_NAME "rec_dense_bit" +#elif defined REC_BIT +# include +using BBT = pasta::RecursiveBitBlockTreeSharded; +# define BIT_ALGO_NAME "rec_bit" +#endif + +#ifdef FP +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t, + const size_t) { + ; + return std::unique_ptr>( + pasta::make_block_tree_fp(text, arity, leaf_length)); +} +# define ALGO_NAME "fp" +#elif defined FP2 +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t, + const size_t) { + ; + return std::make_unique>(text, + arity, + 1, + leaf_length); +} +# define ALGO_NAME "fp2" +#elif defined LPF +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t) { + ; + return std::unique_ptr>( + pasta::make_block_tree_lpf_parallel(text, + arity, + leaf_length, + true, + threads)); +} +# define ALGO_NAME "lpf" +#elif defined PAR_SHARDED +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "shard" +#elif defined PAR_SHARDED_SYNC +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "shard_sync" +#elif defined PAR_SHARDED_SYNC_SMALL +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + return std::make_unique< + pasta::RecursiveBlockTreeSharded>(text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "shard_sync_small" +#elif defined REC_PAR_SHARDED +# include +std::unique_ptr< + pasta::RecursiveBlockTreeSharded> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t queue_size) { + return std::make_unique< + pasta::RecursiveBlockTreeSharded>( + text, + arity, + 1, + leaf_length, + threads, + queue_size); +} +# define ALGO_NAME "rec_shard" +#elif defined PAR_PHMAP +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "par_map" +#elif defined PAR_PARLAY +# include +std::unique_ptr> +make_bt(std::vector& text, + const size_t arity, + const size_t leaf_length, + const size_t threads, + const size_t) { + ; + return std::make_unique>( + text, + arity, + 1, + leaf_length, + threads); +} +# define ALGO_NAME "par_parlay" +#endif + +#include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +int main(int argc, char** argv) { + using namespace pasta; + + tlx::CmdlineParser cp; + cp.set_description( + "Build a block tree for a given input text or bit vector."); + + std::string file; + cp.add_param_string("file", + file, + "The path to the file which to build a block tree from"); + + size_t arity = 0; + cp.add_param_size_t("arity", arity, "The arity of the block tree"); + size_t leaf_length = 0; + cp.add_param_size_t( + "leaf", + leaf_length, + "The maximum number of characters saved verbatim per leaf block."); + + size_t threads = 1; + cp.add_size_t('t', + "threads", + threads, + "The number of threads to use for parallel algorithms (ignored " + "for sequential algorithms)"); + size_t queue_size = 1024; + cp.add_size_t( + 'q', + "queue_size", + queue_size, + "The size of each thread's queue used for sharded hash map algorithms"); + + bool make_bv = false; + cp.add_bool('b', + "bitvec", + make_bv, + "Whether to interpret the input as a bitvector. If \"-o\" is not " + "used, each byte of the input file will be interpreted as 8 bits " + "of the bit vector respectively. In each byte, the least " + "significant bit is index 0."); + + std::string one_chars; + cp.add_string( + 'o', + "one_chars", + one_chars, + "A string to be used with the \"-b\" flag. If used, each character of " + "the input represents one bit of the bit vector. It will be a 1 if this " + "parameter string contains the respective character, 0 otherwise."); + + bool verify = false; + cp.add_bool( + 'v', + "verify", + verify, + "Verify whether all queries on the produced block tree are correct."); + + if (!cp.process(argc, argv)) { + return 1; + } + +#ifdef BT_DBG + std::cout << "building block tree with parameters:" + << "\narity: " << arity << "\nmax leaf length: " << leaf_length + << "\nusing " << threads << " threads" << std::endl; +#endif + + if (!std::filesystem::exists(file)) { + std::cerr << "File does not exist" << std::endl; + exit(1); + } + + std::unique_ptr bv; + std::vector text; + { + const size_t input_size = + std::ifstream(file, std::ios::binary | std::ios::ate).tellg(); + std::ifstream t(file); + if (make_bv) { + if (one_chars.empty()) { + // Interpret each character as 8 bits + bv = std::make_unique(input_size * 8); + std::span bytes = std::as_writable_bytes(bv->data()); + for (size_t i = 0; i < input_size; ++i) { + char next_byte; + t >> next_byte; + bytes[i] = std::byte{static_cast(next_byte)}; + } + } else { + // Interpret each character as a bit + bv = std::make_unique(input_size); + std::array is_one{}; + for (char c : one_chars) { + is_one[static_cast(c)] = true; + } + for (size_t i = 0; i < input_size; ++i) { + char next_byte; + t >> next_byte; + (*bv)[i] = is_one[static_cast(next_byte)]; + } + } + } else { + std::stringstream buffer; + buffer << t.rdbuf(); + std::string input = buffer.str(); + text = std::vector(input.begin(), input.end()); + } + } + + std::cout << "RESULT" + << " file=" << std::filesystem::path(file).filename().string() + << " threads=" << threads << " arity=" << arity + << " leaf_length=" << leaf_length; + if (make_bv) { + std::cout << " input_size=" << bv->size() / 8; + } else { + std::cout << " input_size=" << text.size(); + } + TimePoint now = Clock::now(); + + if (make_bv) { + std::cout << " algo=" << BIT_ALGO_NAME; + // Make bit vector block tree + + /* + auto bt = std::make_unique< + RecursiveBitBlockTreeSharded>(*bv, + arity, + 1, + leaf_length, + threads, + queue_size); + */ + auto bt = + std::make_unique(*bv, arity, 1, leaf_length, threads, queue_size); + +#ifdef BT_DBG +/* + size_t cnt = 0; + for (const auto& b : *bt->leaf_bits_) { + if (b) { + cnt++; + } + } + std::cout << "num ones: " << cnt << "/" << bt->leaf_bits_->size() << " (" + << static_cast(cnt) * 100 / bt->leaf_bits_->size() << "%)" + << std::endl; + */ +#endif + auto elapsed = std::chrono::duration_cast( + Clock::now() - now) + .count(); + const size_t no_rs_space = bt->print_space_usage(); + bt->add_bit_rank_support(); + auto elapsed_rs = std::chrono::duration_cast( + Clock::now() - now) + .count(); + const size_t rs_space = bt->print_space_usage(); + std::cout << " rec=" << RECURSION_LEVELS; + std::cout << " time=" << elapsed << " space=" << no_rs_space + << " time_rs=" << elapsed_rs << " space_rs=" << rs_space; + std::cout << std::endl; + + if (!verify) { + return 0; + } + + std::cerr << "Start verification...\n"; + + FlatRankSelect<> frs(*bv); + +#if defined BT_INSTRUMENT && defined BT_DBG + pasta::print_hash_data(); +#endif + std::cerr << "Access queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 0; i < bv->size(); ++i) { + const bool c = bt->access(i); + if (c != (*bv)[i]) { + std::osyncstream(std::cerr) + << "\nAccess error at position " << i + << "\nExpected: " << std::boolalpha << (*bv)[i] << "\nActual: " << c + << std::noboolalpha << std::endl; + exit(1); + } + } + std::cerr << "successful\n"; + std::cerr << "Rank 1 queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 0; i < bv->size(); i++) { + const size_t bt_rank = bt->rank1(i); + const size_t bv_rank = frs.rank1(i); + + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) + << "\nRank one error at position " << i << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + + std::cerr << "successful\n"; + std::cerr << "Rank 0 queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 0; i < bv->size(); i++) { + const size_t bt_rank = bt->rank0(i); + const size_t bv_rank = frs.rank0(i); + + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "\nRank zero error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + + const size_t num_zeros = frs.rank0(bv->size()); + const size_t num_ones = frs.rank1(bv->size()); + + std::cerr << "successful\n"; + std::cerr << "Select 1 queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 1; i <= num_ones; i++) { + const size_t bv_rank = frs.select1(i); + const size_t bt_rank = bt->select1(i); + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "\nSelect one error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + + std::cerr << "successful\n"; + std::cerr << "Select 0 queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 1; i <= num_zeros; i++) { + const size_t bv_rank = frs.select0(i); + const size_t bt_rank = bt->select0(i); + if (bv_rank != bt_rank) { + std::osyncstream(std::cerr) << "\nSelect zero error at position " << i + << "\nExpected: " << bv_rank + << "\nActual: " << bt_rank << std::endl; + throw std::runtime_error("oof"); + } + } + std::cerr << "successful" << std::endl; + } else { + std::cout << " algo=" << ALGO_NAME; + // Make text block tree + auto bt = make_bt(text, arity, leaf_length, threads, queue_size); + auto elapsed = std::chrono::duration_cast( + Clock::now() - now) + .count(); + + std::cout << " rec=" << RECURSION_LEVELS; + std::cout << " time=" << elapsed << " space=" << bt->print_space_usage(); + + if (!verify) { + std::cout << std::endl; + return 0; + } + + // std::cerr << "Start verification...\n"; +#if defined BT_INSTRUMENT && defined BT_DBG + pasta::print_hash_data(); +#endif + + // std::cerr << "Access queries... " << std::flush; +#pragma omp parallel for + for (size_t i = 0; i < text.size(); ++i) { + const auto c = bt->access(i); + if (c != text[i]) { + #pragma omp critical + std::cout << " verification=failed" << std::endl; + std::exit(-1); + } + } + } + std::cout << " verification=passed" << std::endl; + + return 0; +} + +/******************************************************************************/ diff --git a/extlib/Jiffy b/extlib/Jiffy new file mode 160000 index 0000000..82cb6fb --- /dev/null +++ b/extlib/Jiffy @@ -0,0 +1 @@ +Subproject commit 82cb6fbff6b6ff28b6cfc1e2c1231e85570d03bb diff --git a/extlib/Jiffy-1 b/extlib/Jiffy-1 new file mode 160000 index 0000000..4121cd2 --- /dev/null +++ b/extlib/Jiffy-1 @@ -0,0 +1 @@ +Subproject commit 4121cd27ecf62889e1cf22e62da3a72cbd0f8e7d diff --git a/extlib/growt b/extlib/growt new file mode 160000 index 0000000..0c1148e --- /dev/null +++ b/extlib/growt @@ -0,0 +1 @@ +Subproject commit 0c1148ebcdfd4c04803be79706533ad09cc81d37 diff --git a/extlib/parallel-hashmap b/extlib/parallel-hashmap new file mode 160000 index 0000000..df7935a --- /dev/null +++ b/extlib/parallel-hashmap @@ -0,0 +1 @@ +Subproject commit df7935aca33afdae9218ea57f57a35dab3eec8fe diff --git a/extlib/parlayhash b/extlib/parlayhash new file mode 160000 index 0000000..dda80fc --- /dev/null +++ b/extlib/parlayhash @@ -0,0 +1 @@ +Subproject commit dda80fcb90fb6f8ff5b5a7a730b202a7ceca6200 diff --git a/extlib/sdsl-lite b/extlib/sdsl-lite new file mode 160000 index 0000000..513f9eb --- /dev/null +++ b/extlib/sdsl-lite @@ -0,0 +1 @@ +Subproject commit 513f9ebe87ee9d3cfe8dbed5133d639767e1722c diff --git a/extlib/unordered_dense b/extlib/unordered_dense new file mode 160000 index 0000000..231e48c --- /dev/null +++ b/extlib/unordered_dense @@ -0,0 +1 @@ +Subproject commit 231e48c9426bd21c273669e5fdcd042c146975cf diff --git a/extlib/waitfree-mpsc-queue b/extlib/waitfree-mpsc-queue new file mode 160000 index 0000000..020ba22 --- /dev/null +++ b/extlib/waitfree-mpsc-queue @@ -0,0 +1 @@ +Subproject commit 020ba2262c48b24828bed98e5cc63c5529ca24ce diff --git a/include/pasta/block_tree/block_tree.hpp b/include/pasta/block_tree/block_tree.hpp index 3281cd3..c69e202 100644 --- a/include/pasta/block_tree/block_tree.hpp +++ b/include/pasta/block_tree/block_tree.hpp @@ -20,24 +20,26 @@ #pragma once -#include +#include #include #include -#include -#include -#include #include #include #include -#include #include #include #include namespace pasta { -template class BlockTree { +template +class BlockTree { public: + /// @brief If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. bool CUT_FIRST_LEVELS = true; size_type tau_; size_type max_leaf_length_; @@ -45,11 +47,11 @@ template class BlockTree { size_type leaf_size = 0; size_type amount_of_leaves = 0; bool rank_support = false; - std::vector block_tree_types_; - std::vector *> + std::vector block_tree_types_; + std::vector*> block_tree_types_rs_; - std::vector *> block_tree_pointers_; - std::vector *> block_tree_offsets_; + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; // std::vector*> block_tree_encoded_; std::vector block_size_lvl_; std::vector block_per_lvl_; @@ -59,7 +61,7 @@ template class BlockTree { std::vector decompress_map_; sdsl::int_vector<> compressed_leaves_; - std::unordered_map chars_index_; + ankerl::unordered_dense::map chars_index_; std::vector chars_; size_type u_chars_; std::vector> top_level_c_ranks_; @@ -73,10 +75,10 @@ template class BlockTree { int64_t child; for (size_type i = 0; static_cast(i) < block_tree_types_.size(); i++) { - auto &lvl = *block_tree_types_[i]; - auto &lvl_rs = *block_tree_types_rs_[i]; - auto &lvl_ptr = *block_tree_pointers_[i]; - auto &lvl_off = *block_tree_offsets_[i]; + auto& lvl = *block_tree_types_[i]; + auto& lvl_rs = *block_tree_types_rs_[i]; + auto& lvl_ptr = *block_tree_pointers_[i]; + auto& lvl_off = *block_tree_offsets_[i]; if (lvl[blk_pointer] == 0) { size_type blk = lvl_rs.rank0(blk_pointer); off = off + lvl_off[blk]; @@ -91,23 +93,22 @@ template class BlockTree { off = off % block_size; blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; } - return compressed_leaves_[blk_pointer * leaf_size + off]; + return decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; }; int64_t select(input_type c, size_type j) { auto c_index = chars_index_[c]; - auto &top_level = *block_tree_types_[0]; + auto& top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; size_type current_block = (j - 1) / block_size_lvl_[0]; size_type end_block = c_ranks_[c_index][0].size() - 1; int64_t block_size = block_size_lvl_[0]; // find first level block containing the jth occurrence of c with a bin // search while (current_block != end_block) { - size_type m = current_block + (end_block - current_block) / 2; size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; @@ -134,10 +135,10 @@ template class BlockTree { int64_t blk = top_level_rs.rank0(current_block); current_block = top_level_ptr[blk]; int64_t g = top_level_off[blk]; - int64_t rank_d = (current_block == 0) - ? c_ranks_[c_index][0][0] - : c_ranks_[c_index][0][current_block] - - c_ranks_[c_index][0][current_block - 1]; + int64_t rank_d = (current_block == 0) ? + c_ranks_[c_index][0][0] : + c_ranks_[c_index][0][current_block] - + c_ranks_[c_index][0][current_block - 1]; rank_d -= pointer_c_ranks_[c_index][0][blk]; if (rank_d < j) { j -= rank_d; @@ -150,11 +151,11 @@ template class BlockTree { } uint64_t i = 1; while (i < block_tree_types_.size()) { - auto ¤t_level = *block_tree_types_[i]; - auto ¤t_level_rs = *block_tree_types_rs_[i]; - auto ¤t_level_ptr = *block_tree_pointers_[i]; - auto ¤t_level_off = *block_tree_offsets_[i]; - auto &prev_level_rs = *block_tree_types_rs_[i - 1]; + auto& current_level = *block_tree_types_[i]; + auto& current_level_rs = *block_tree_types_rs_[i]; + auto& current_level_ptr = *block_tree_pointers_[i]; + auto& current_level_off = *block_tree_offsets_[i]; + auto& prev_level_rs = *block_tree_types_rs_[i - 1]; current_block = prev_level_rs.rank1(current_block) * tau_; block_size /= tau_; int64_t k = current_block; @@ -167,10 +168,10 @@ template class BlockTree { int64_t blk = current_level_rs.rank0(current_block); current_block = current_level_ptr[blk]; int64_t g = current_level_off[blk]; - int64_t rank_d = (current_block % tau_ == 0) - ? c_ranks_[c_index][i][current_block] - : c_ranks_[c_index][i][current_block] - - c_ranks_[c_index][i][current_block - 1]; + int64_t rank_d = (current_block % tau_ == 0) ? + c_ranks_[c_index][i][current_block] : + c_ranks_[c_index][i][current_block] - + c_ranks_[c_index][i][current_block - 1]; rank_d -= pointer_c_ranks_[c_index][i][blk]; if (rank_d < j) { j -= rank_d; @@ -195,10 +196,10 @@ template class BlockTree { } int64_t rank_base(input_type c, size_type index) { - pasta::BitVector &top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; int64_t c_index = chars_index_[c]; int64_t block_size = block_size_lvl_[0]; int64_t blk_pointer = index / block_size; @@ -219,10 +220,10 @@ template class BlockTree { blk_pointer = top_level_ptr[blk]; child = blk_pointer; if (to >= block_size) { - int64_t adder = (child == 0) - ? c_ranks_[c_index][0][blk_pointer] - : c_ranks_[c_index][0][blk_pointer] - - c_ranks_[c_index][0][blk_pointer - 1]; + int64_t adder = (child == 0) ? + c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; rank += adder; blk_pointer++; off = to - block_size; @@ -253,8 +254,8 @@ template class BlockTree { child = blk_pointer % tau_; if (to >= block_size) { - auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] - : c_ranks_[c_index][i][blk_pointer] - + auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - c_ranks_[c_index][i][blk_pointer - 1]; rank += adder; blk_pointer++; @@ -280,10 +281,10 @@ template class BlockTree { } int64_t rank(input_type c, size_type index) { - pasta::BitVector &top_level = *block_tree_types_[0]; - auto &top_level_rs = *block_tree_types_rs_[0]; - auto &top_level_ptr = *block_tree_pointers_[0]; - auto &top_level_off = *block_tree_offsets_[0]; + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; int64_t c_index = chars_index_[c]; int64_t block_size = block_size_lvl_[0]; int64_t blk_pointer = index / block_size; @@ -303,8 +304,8 @@ template class BlockTree { blk_pointer = top_level_ptr[blk]; child = blk_pointer; if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] - : c_ranks_[c_index][0][blk_pointer] - + rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - c_ranks_[c_index][0][blk_pointer - 1]; blk_pointer++; off = off - block_size; @@ -333,8 +334,8 @@ template class BlockTree { blk_pointer = (*block_tree_pointers_[i])[blk]; child = blk_pointer % tau_; if (off >= block_size) { - rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] - : c_ranks_[c_index][i][blk_pointer] - + rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - c_ranks_[c_index][i][blk_pointer - 1]; blk_pointer++; child = blk_pointer % tau_; @@ -401,12 +402,14 @@ template class BlockTree { void compress_leaves() { compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); for (size_t i = 0; i < this->leaves_.size(); ++i) { compress_map_[this->leaves_[i]] = 1; } - for (size_t i = 0, cur_val = 0; i < this->compress_map_.size(); ++i) { - size_t tmp = compress_map_[i]; - compress_map_[i] = cur_val; + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; cur_val += tmp; } @@ -526,25 +529,59 @@ template class BlockTree { return 0; } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. inline size_type leading_zeros(int32_t val) { return __builtin_clz(static_cast(val) | 1); } + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. inline size_type leading_zeros(int64_t val) { return __builtin_clzll(static_cast(val) | 1); } - void calculate_padding(int64_t &padding, int64_t text_length, int64_t &height, - int64_t &blk_size) { + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. int64_t tmp_padding = this->s_ * this->tau_; int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) blk_size = tau_; + // While the tree does not cover the entire text, add a level while (tmp_padding < text_length) { tmp_padding *= this->tau_; blk_size *= this->tau_; h++; } + // once the tree has enough levels to cover the entire text, we set the + // tree's values height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. padding = tmp_padding - text_length; } @@ -582,8 +619,8 @@ template class BlockTree { c_ranks_[chars_index_[c]][i][j] = rank_c; return rank_c; } - size_type part_rank_block(input_type c, size_type i, size_type j, - size_type g) { + size_type + part_rank_block(input_type c, size_type i, size_type j, size_type g) { if (static_cast(j) >= block_tree_types_[i]->size()) { return 0; } @@ -630,7 +667,6 @@ template class BlockTree { return rank_c; } size_type rank_leaf(input_type c, size_type leaf_index, size_type i) { - if (static_cast(leaf_index * leaf_size) >= compressed_leaves_.size()) { return 0; @@ -647,7 +683,7 @@ template class BlockTree { return result; } - size_type map_unique_chars(std::vector &text) { + size_type map_unique_chars(const std::vector& text) { this->u_chars_ = 0; input_type i = 0; for (auto a : text) { @@ -662,7 +698,7 @@ template class BlockTree { }; size_type find_next_smallest_index_binary_search(size_type i, - std::vector &pVector) { + std::vector& pVector) { int64_t l = 0; int64_t r = pVector.size(); while (l < r) { @@ -677,7 +713,7 @@ template class BlockTree { }; int64_t find_next_smallest_index_linear_scan(size_type i, - std::vector &pVector) { + std::vector& pVector) { int64_t b = 0; while (b < pVector.size() && i >= pVector[b]) { b++; diff --git a/include/pasta/block_tree/construction/block_tree_fp.hpp b/include/pasta/block_tree/construction/block_tree_fp.hpp index 325d890..116f807 100644 --- a/include/pasta/block_tree/construction/block_tree_fp.hpp +++ b/include/pasta/block_tree/construction/block_tree_fp.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,28 +21,61 @@ #pragma once -#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" #include "pasta/block_tree/utils/MersenneHash.hpp" #include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include +#include + __extension__ typedef unsigned __int128 uint128_t; namespace pasta { +template > +using HashMap = ankerl::unordered_dense::map; + template class BlockTreeFP : public BlockTree { public: size_type const_size = 0; size_type sigma_ = 0; + /// + /// @brief Prune this block and all of its children. + /// + /// This will set the pointers for each node that is pruned to PRUNED (== -2) + /// and will increase and decrease pointers for each new back block or pruned + /// back block respectively. + /// + /// @param[in] counter For each level (starting at the topmost level) and + /// block, contains the number of back blocks pointing to the block + /// @param[in] pointer For each level (starting at the topmost level) and + /// block, contains the earliest block index in which the content of this + /// @param[in] offset For each level (starting at the topmost level) and + /// block, contains the character offset at which this block's content can + /// @param[in] marked_tree For each level (starting at the topmost level) and + /// block, has a 1 if the block is internal, 0 if it is a back block. + /// @param[out] pruned_tree The resulting tree after pruning, with the same + /// content as marked_tree (as far as I can see this is unused) + /// @param[in] i The current level, 0 being the top level + /// @param[in] j The current block being pruned + /// @param[in] ranks Rank data structures for each bit vector in marked_tree. + /// @return true, if an this block was and stays internal, false otherwise + /// bool prune_block( - std::vector> &counter, - std::vector> &pointer, - std::vector> &offset, - std::vector &marked_tree, - std::vector &pruned_tree, size_type i, size_type j, - std::vector> &ranks) { - // string leaf children can always be pruned - // fully padded children don't exist and can be ignored/ sanity check + std::vector>& counter, + std::vector>& pointer, + std::vector>& offset, + std::vector& marked_tree, + std::vector& pruned_tree, + size_type i, + size_type j, + std::vector>& ranks) { + // String leaf children can always be pruned, + // since they are contained in the leaf string. + // Fully padded children don't exist and can be ignored/sanity check // assumes short circuit evaluation as compiler behaviour if (static_cast(i) >= marked_tree.size() || static_cast(j) >= marked_tree[i]->size()) { @@ -49,31 +83,45 @@ class BlockTreeFP : public BlockTree { } bool marked_children = false; - // we already incremented counters for unmarked blocks during a previous - // step and now only need to consider marked blocks + // We already incremented counters for back blocks during construction + // and now only need to consider internal blocks if ((*marked_tree[i])[j] == 1) { - // inverse postorder dfs - // prune_block returns true if a marked block stays marked false otherwise + // inverse postorder dfs. We handle children from back to front. + // We need the rank over the internal nodes of this level, + // because only the internal nodes generated children on the + // next level. size_type rank_blk = ranks[i].rank1(j); for (size_type k = this->tau_ - 1; k >= 0; k--) { - marked_children |= - prune_block(counter, pointer, offset, marked_tree, pruned_tree, - i + 1, rank_blk * this->tau_ + k, ranks); + marked_children |= prune_block(counter, + pointer, + offset, + marked_tree, + pruned_tree, + i + 1, + rank_blk * this->tau_ + k, + ranks); } - // conditions to be pruned are no marked children, no pointers pointing to - // me and a former occurrence in S + // Conditions to be pruned are: + // - no internal children, + // - no pointers pointing to me + // - an earlier occurrence in the text if (!marked_children && counter[i][j] == 0 && pointer[i][j] != NO_FORMER_OCC) { - + // This block is no longer internal (*marked_tree[i])[j] = 0; + // Since this is a back block now, we need to increment the counters for + // the blocks this back block now points to counter[i][pointer[i][j]]++; if (offset[i][j] > 0) { counter[i][pointer[i][j] + 1]++; } + // We only remove children if we're not at the last level if (static_cast(i + 1) < counter.size()) { - // remove all of its children by decrementing counters and marking - // them as PRUNED + // Remove all of its children by decrementing counters + // and marking them as PRUNED for (size_type k = this->tau_ - 1; k >= 0; k--) { + // If this child node actually exists, + // decrement the counters of the blocks the child points to and set if (static_cast(rank_blk * this->tau_) + k < counter[i + 1].size()) { auto ptr_child = pointer[i + 1][rank_blk * this->tau_ + k]; @@ -81,47 +129,73 @@ class BlockTreeFP : public BlockTree { if (offset[i + 1][rank_blk * this->tau_ + k] > 0) { counter[i + 1][ptr_child + 1]--; } + // Set this child's pointer to PRUNED pointer[i + 1][rank_blk * this->tau_ + k] = PRUNED; } } } } } + // If this node has internal children, has other blocks pointing to itself + // or has no earlier occurrence, then it remains internal return marked_children || counter[i][j] > 0 || pointer[i][j] == NO_FORMER_OCC; }; - int32_t pruning_extended(std::vector> &counter, - std::vector> &pointer, - std::vector> &offset, - std::vector &marked_tree, - std::vector &pruned_tree) { + /// + /// @brief Prunes the block tree + /// + /// @param[in] counter For each level (starting at the topmost level) and + /// block, contains the number of back blocks pointing to the block + /// @param[in] pointer For each level (starting at the topmost level) and + /// block, contains the earliest block index in which the content of this + /// block can be found. + /// @param[in] offset For each level (starting at the topmost level) and + /// block, contains the character offset at which this block's content can + /// be found in the block the back-pointer points to. + /// @param[in] marked_tree For each level (starting at the topmost level) and + /// block, has a 1 if the block is internal, 0 if it is a back block. + /// @param[out] pruned_tree The resulting tree after pruning, with the same + /// content as marked_tree + /// @return 0 + /// + int32_t pruning_extended(std::vector>& counter, + std::vector>& pointer, + std::vector>& offset, + std::vector& marked_tree, + std::vector& pruned_tree) { std::vector> ranks; for (auto bv : marked_tree) { ranks.push_back(pasta::RankSelect(*bv)); } - auto &top_lvl = *marked_tree[0]; + auto& top_lvl = *marked_tree[0]; + /// Prune the blocks on the top level from back to front for (size_type j = top_lvl.size() - 1; j >= 0; j--) { - prune_block(counter, pointer, offset, marked_tree, pruned_tree, 0, j, + prune_block(counter, + pointer, + offset, + marked_tree, + pruned_tree, + 0, + j, ranks); } return 0; } - int32_t pruning_simple(std::vector &first_pass_bv, - std::vector> &blk_lvl, - std::vector &bv_pass_2, - std::vector> &pass1_pointer, - std::vector> &pass1_offset, - std::vector> &pass2_pointer, - std::vector> &pass2_offset, - std::vector &pass2_max_pointer, - std::vector &pass2_max_offset, - std::vector &pass2_ones, - int64_t &block_size) { - + int32_t pruning_simple(std::vector& first_pass_bv, + std::vector>& blk_lvl, + std::vector& bv_pass_2, + std::vector>& pass1_pointer, + std::vector>& pass1_offset, + std::vector>& pass2_pointer, + std::vector>& pass2_offset, + std::vector& pass2_max_pointer, + std::vector& pass2_max_offset, + std::vector& pass2_ones, + int64_t& block_size) { for (int64_t i = first_pass_bv.size() - 1; i >= 0; i--) { - auto *bv = new pasta::BitVector(blk_lvl[i].size(), 0); + auto* bv = new pasta::BitVector(blk_lvl[i].size(), 0); size_type marked_counter = 0; if (static_cast(i) != first_pass_bv.size() - 1) { for (uint64_t j = 0; j < bv->size(); j++) { @@ -140,7 +214,6 @@ class BlockTreeFP : public BlockTree { auto offsets = std::vector(); size_type max_pointer = 0; for (size_type j = blk_lvl[i].size() - 1; j >= 0; j--) { - if ((*bv)[j] == 1) { continue; } @@ -183,35 +256,57 @@ class BlockTreeFP : public BlockTree { return 0; } - int32_t init_extended(std::vector &text) { + int32_t init_extended(std::vector& text) { static constexpr uint128_t kPrime = 2305843009213693951ULL; + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size int64_t added_padding = 0; + /// The height of the tree int64_t tree_max_height = 0; + /// The size of the largest blocks (i.e. the top level blocks) int64_t max_blk_size = 0; + /// For each level (starting at the top) contains the text start indices of + /// each block on the level std::vector> blk_lvl; + /// For each level and each block, contains the first block index at which + /// the content of this block appears std::vector> pass1_pointer; + /// For each level and each block, contains the offset at which + /// the content of this block appears in its back-pointed block std::vector> pass1_offset; - std::vector bv_marked; + /// For each level contains a bit vector containing a 1 for each block that + /// is internal and a 0 for each back block + std::vector bv_marked; + /// For every level and block counts how many back blocks are pointing to + /// the block std::vector> counter; std::vector pass2_ones; + /// The block size for each level, starting at the top level std::vector block_size_lvl_temp; - this->calculate_padding(added_padding, text.size(), tree_max_height, + this->calculate_padding(added_padding, + text.size(), + tree_max_height, max_blk_size); auto is_padded = added_padding > 0 ? 1 : 0; + /// The current block size starting at the top level int64_t block_size = max_blk_size; + /// The text start indices of each block on the current level std::vector block_text_inx; for (uint64_t i = 0; i < text.size(); i += block_size) { block_text_inx.push_back(i); } + // if the blocks on the current level are already below the max leaf length, + // we may not divide them further. So the entire block tree just consists of + // the current level verbatim, no pointers if (block_size <= this->max_leaf_length_) { - auto *bv = new pasta::BitVector(block_text_inx.size(), 1); + auto* bv = new pasta::BitVector(block_text_inx.size(), 1); this->block_tree_types_rs_.push_back( new pasta::RankSelect(*bv)); auto p0 = new sdsl::int_vector<>(0, 0); auto o0 = new sdsl::int_vector<>(0, 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; sdsl::util::bit_compress(ptr0); sdsl::util::bit_compress(off0); this->block_tree_types_.push_back(bv); @@ -225,36 +320,59 @@ class BlockTreeFP : public BlockTree { } while (block_size > this->max_leaf_length_) { block_size_lvl_temp.push_back(block_size); - auto *bv = new pasta::BitVector(block_text_inx.size(), false); + /// Marks whether a block should be internal or not + auto* bv = new pasta::BitVector(block_text_inx.size(), false); + /// If left[i] == 1, then there is an earlier occurrence of + /// block[i]block[i+1] auto left = pasta::BitVector(block_text_inx.size(), false); + /// If right[i] == 1, then there is an earlier occurrence of + /// block[i-1]block[i] auto right = pasta::BitVector(block_text_inx.size(), false); auto pair_size = 2 * block_size; + // Check, whether the last block's end extends past the end of the text auto last_block_padded = static_cast(block_text_inx[block_text_inx.size() - 1] + - block_size) != text.size() - ? 1 - : 0; - std::unordered_map, std::vector> pairs( - 0); - std::unordered_map, std::vector> blocks = - std::unordered_map, std::vector>(); + block_size) != text.size() ? + 1 : + 0; + // map block pair hashes to the text index of their occurrences + // collecting duplicates in a vector TODO + HashMap, std::vector> pairs(0); + // map block hashes to their *block* index, + // collecting duplicates in a vector TODO + HashMap, std::vector> blocks = + HashMap, std::vector>(); + // iterate through all blocks on the current level, skipping over the last + // block if it is padded for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { + // Hash the current block and insert it into the block hash map auto index = block_text_inx[i]; MersenneRabinKarp rk_block = - MersenneRabinKarp(text, sigma_, index, - block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + block_size, + kPrime); MersenneHash mh_block = MersenneHash(text, rk_block.hash_, index, block_size); blocks[mh_block].push_back(i); } - std::vector pointers(block_text_inx.size(), -1); + // Back pointers, offsets and the incoming-pointer-counters which we need + // for pruning later + std::vector pointers(block_text_inx.size(), NO_FORMER_OCC); std::vector offsets(block_text_inx.size(), 0); std::vector counters(block_text_inx.size(), 0); + // If a block pair is larger than the whole text, then there is + // nothing really to do on this level. There cannot be any back pointers if (static_cast(pair_size) > text.size()) { block_size /= this->tau_; + // Start indices of blocks for the next level std::vector new_blocks(0); for (uint64_t i = 0; i < block_text_inx.size(); i++) { + // All of the current blocks are internal (*bv)[i] = 1; + // We split all blocks on the current level into tau sub-blocks but + // only create blocks that start before the end of the text for (size_type j = 0; j < this->tau_; j++) { if (static_cast(block_text_inx[i] + (j * block_size)) < text.size()) { @@ -262,6 +380,7 @@ class BlockTreeFP : public BlockTree { } } } + /// There are no pointers, offsets or counters on this level std::vector p(block_text_inx.size(), -1); std::vector o(block_text_inx.size(), 0); std::vector c(block_text_inx.size(), 0); @@ -273,27 +392,39 @@ class BlockTreeFP : public BlockTree { counter.push_back(c); continue; } + // Iterate through the block pairs and add them to the pair hash table + // along with their index *if they are consecutive* for (uint64_t i = 0; i < block_text_inx.size() - 1; i++) { if (block_text_inx[i] + block_size == block_text_inx[i + 1] && static_cast(block_text_inx[i] + pair_size) <= text.size()) { auto index = block_text_inx[i]; MersenneRabinKarp rk_pair = - MersenneRabinKarp(text, sigma_, index, - pair_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + pair_size, + kPrime); MersenneHash mh_pair = MersenneHash(text, rk_pair.hash_, index, pair_size); pairs[mh_pair].push_back(i); } } - // find pairs + // Find the occurrences of all block pairs' contents MersenneRabinKarp rk_pair_sw = - MersenneRabinKarp(text, sigma_, 0, pair_size, + MersenneRabinKarp(text, + sigma_, + 0, + pair_size, kPrime); + // Hash each window in the text of the size of a block pair + // and see if it corresponds to an actual block pair for (uint64_t i = 0; i < text.size() - pair_size; i++) { MersenneHash mh_sw = MersenneHash(text, rk_pair_sw.hash_, i, pair_size); if (pairs.find(mh_sw) != pairs.end()) { + // If the current hash corresponds to a hashed block pair, + // we update for those pairs, that they have an earlier occurrence for (auto b : pairs[mh_sw]) { if (i != static_cast(block_text_inx[b])) { left[b] = 1; @@ -308,18 +439,28 @@ class BlockTreeFP : public BlockTree { auto new_block_size = block_size / this->tau_; std::vector new_blocks(0); for (uint64_t i = 0; i < block_text_inx.size(); i++) { + /// This is true <=> the current block is adjacent + /// to its predecessor and successor bool surrounded = (i > 0 && i < block_text_inx.size() - 1) && block_text_inx[i] + old_block_size == block_text_inx[i + 1] && block_text_inx[i - 1] + old_block_size == block_text_inx[i]; + /// marked <=> not internal (ONLY HERE IT SEEMS) bool marked = false; + // if the block is adjacent to its predecessor and successor, then it + // must have a previous occurrence both as a left and right part of a + // block pair in order to be a back block. + // Otherwise, only consecutive neighboring blocks need to be considered. if (surrounded) { marked = left[i] && right[i]; } else { marked = left[i] || right[i]; } + // If either the block is internal or extends past the text end (i.e. is + // padded), then we create tau child nodes for this block if (!(marked) || static_cast(block_text_inx[i] + old_block_size) >= text.size()) { + // This block is internal (*bv)[i] = 1; for (size_type j = 0; j < this->tau_; j++) { if (static_cast(block_text_inx[i] + @@ -330,35 +471,60 @@ class BlockTreeFP : public BlockTree { } } MersenneRabinKarp rk_first_occ = - MersenneRabinKarp( - text, sigma_, block_text_inx[0], block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + block_text_inx[0], + block_size, + kPrime); + // Identify the first occurrence for each block on this level for (int64_t i = 0; static_cast(i) < block_text_inx.size() - 1; i++) { + // This is true <=> + // This is not the last block, + // this block is adjacent to the next and + // the next block is internal + // We need this, because the hasher overlaps the next block as well. bool followed = (static_cast(i) < block_text_inx.size() - 1) && block_text_inx[i] + block_size == block_text_inx[i + 1] && (*bv)[i + 1] == 1; + // If this block is internal if ((*bv)[i] == 1) { + // If the hasher's current position is currently not at the current + // block index, move it there if (rk_first_occ.init_ != static_cast(block_text_inx[i])) { rk_first_occ.restart(block_text_inx[i]); } if (followed) { + // We iterate through every window that starts in this block + // and ends before the end of the text. + // j is the offset into the current block for (int64_t j = 0; j < block_size && static_cast(block_text_inx[i] + j + block_size) < text.size(); j++) { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i] + j, block_size); + // Hash the window and try to find an earlier occurrence + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i] + j, + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { - // b cant be i and if j>0 then b cant follow on i (j>0) -> b > - // i + 1 (a -> b <=> not a or b) + // The if the current block (b) were i, it would reference + // itself. If j > 0 then the occurrence overlaps the block i + // + 1. Therefore, in that case b must be a block *past* i+1 if (b > i && (j <= 0 || b > i + 1)) { + // If all is well, set the back pointer and offsets pointers[b] = i; offsets[b] = j; if ((*bv)[b] == 0) { + // If b is a back block, then we have another block + // pointing to i counters[i]++; if (j > 0) { + // if the offset is greater than 0, the copied area + // extends into the next block counters[i + 1]++; } } @@ -369,8 +535,12 @@ class BlockTreeFP : public BlockTree { rk_first_occ.next(); } } else { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i], block_size); + // If the next block is not adjacent, we only hash once + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i], + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (b != i) { @@ -383,6 +553,7 @@ class BlockTreeFP : public BlockTree { } } } + // Add the values calculated on this level pass1_pointer.push_back(pointers); pass1_offset.push_back(offsets); counter.push_back(counters); @@ -392,15 +563,21 @@ class BlockTreeFP : public BlockTree { block_size = new_block_size; bv_marked.push_back(bv); } + // By this point, the first pass is done and we have an unpruned block tree this->leaf_size = block_size; block_size *= this->tau_; - pruning_extended(counter, pass1_pointer, pass1_offset, bv_marked, + // Prune the tree. Doing so will replace the pointers of pruned nodes with + // PRUNED + pruning_extended(counter, + pass1_pointer, + pass1_offset, + bv_marked, bv_marked); std::vector ones_per_lvl(bv_marked.size(), 0); // count 1s in each lvl; for (uint64_t i = 0; i < bv_marked.size(); i++) { - auto ¤t_lvl = *bv_marked[i]; + auto& current_lvl = *bv_marked[i]; for (uint64_t j = 0; j < bv_marked[i]->size(); j++) { if (current_lvl[j]) { ones_per_lvl[i]++; @@ -408,18 +585,19 @@ class BlockTreeFP : public BlockTree { } } - auto &top_level = *bv_marked[0]; + auto& top_level = *bv_marked[0]; bool found_back_block = top_level.size() != static_cast(ones_per_lvl[0]) || bv_marked.size() == 1; + // If there is a back block on the first level, add its values to the tree if (found_back_block || !this->CUT_FIRST_LEVELS) { this->block_tree_types_.push_back(&top_level); this->block_tree_types_rs_.push_back( new pasta::RankSelect(top_level)); auto p0 = new sdsl::int_vector<>(top_level.size() - ones_per_lvl[0], 0); auto o0 = new sdsl::int_vector<>(top_level.size() - ones_per_lvl[0], 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; size_type c = 0; for (uint64_t j = 0; j < top_level.size(); j++) { if (!top_level[j]) { @@ -436,11 +614,14 @@ class BlockTreeFP : public BlockTree { } else { delete bv_marked[0]; } - for (uint64_t i = 1; i < bv_marked.size(); i++) { + for (uint64_t i = 1; i < bv_marked.size(); i++) { + // If the previous level is padded, we need to be careful, since the last + // block possibly does not generate exactly tau children size_type new_size = (ones_per_lvl[i - 1] - is_padded) * this->tau_; auto last_block_parent = blk_lvl[i - 1][blk_lvl[i - 1].size() - 1]; auto lvl_block_size = block_size_lvl_temp[i]; + // Determine the number of children the last block generated if (is_padded) { for (uint64_t j = 0; j < static_cast(this->tau_); j++) { if (last_block_parent + j * lvl_block_size < text.size()) { @@ -448,23 +629,34 @@ class BlockTreeFP : public BlockTree { } } } + // Check if we have found a back block on the current level found_back_block |= new_size != ones_per_lvl[i]; - if (found_back_block || !this->CUT_FIRST_LEVELS) { + // If there is a back block, we add this level's data to the tree + if (found_back_block || !this->CUT_FIRST_LEVELS || + i == bv_marked.size() - 1) { + // is_internal auto bit_vector = new pasta::BitVector(new_size, 0); - auto &bv_ref = *bit_vector; + auto& bv_ref = *bit_vector; auto p = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); auto o = new sdsl::int_vector<>(bv_ref.size() - ones_per_lvl[i], 0); - auto &ptr = *p; - auto &off = *o; - std::unordered_map blocks_skipped; - auto &lvl_pass1 = *bv_marked[i]; + auto& ptr = *p; + auto& off = *o; + // Maps block index => number of pruned blocks before this block + HashMap blocks_skipped; + auto& lvl_pass1 = *bv_marked[i]; + // Number of non-pruned blocks so far size_type c = 0; + // size_type c_u = 0; for (uint64_t j = 0; j < lvl_pass1.size(); j++) { blocks_skipped[j] = j - c; - if (pass1_pointer[i][j] != -2) { + // If the current block is not pruned, add it to the new tree + if (pass1_pointer[i][j] != PRUNED) { + // Add it to the is_internal bit vector bv_ref[c] = (bool)lvl_pass1[j]; + // If it is a back block, add its pointer and offset if (!lvl_pass1[j]) { + // We need to ignore the pruned blocks ptr[c_u] = pass1_pointer[i][j] - blocks_skipped[pass1_pointer[i][j]]; off[c_u] = pass1_offset[i][j]; @@ -473,6 +665,7 @@ class BlockTreeFP : public BlockTree { c++; } } + // Add the new data to the tree this->block_tree_types_.push_back(&bv_ref); this->block_tree_types_rs_.push_back( new pasta::RankSelect(bv_ref)); @@ -481,18 +674,27 @@ class BlockTreeFP : public BlockTree { this->block_tree_pointers_.push_back(p); this->block_tree_offsets_.push_back(o); this->block_size_lvl_.push_back(block_size_lvl_temp[i]); - } else { + } + // Delete the old bitvec since we don't need it anymore. + // If this is the last level, we still need the bv for constructing the + // leaves + if (i < bv_marked.size() - 1) { delete bv_marked[i]; } } + // Construct the leaf string int64_t leaf_count = 0; - auto &last_level = (*bv_marked[bv_marked.size() - 1]); + auto& last_level = (*bv_marked[bv_marked.size() - 1]); for (uint64_t i = 0; i < last_level.size(); i++) { if (last_level[i] == 1) { + // For every leaf on the last level, we have tau leaf blocks leaf_count += this->tau_; + // Iterate through all characters in this child and add them to the leaf + // string for (uint64_t j = 0; - j < static_cast(this->leaf_size * this->tau_); j++) { + j < static_cast(this->leaf_size * this->tau_); + j++) { if (static_cast(blk_lvl[blk_lvl.size() - 1][i] + j) < text.size()) { this->leaves_.push_back(text[blk_lvl[blk_lvl.size() - 1][i] + j]); @@ -500,12 +702,13 @@ class BlockTreeFP : public BlockTree { } } } + delete &last_level; this->amount_of_leaves = leaf_count; this->compress_leaves(); return 0; } - int32_t init_simple(std::vector &text) { + int32_t init_simple(std::vector& text) { static constexpr uint128_t kPrime = 2305843009213693951ULL; int64_t added_padding = 0; int64_t tree_max_height = 0; @@ -513,15 +716,17 @@ class BlockTreeFP : public BlockTree { std::vector> blk_lvl; std::vector> pass1_pointer; std::vector> pass1_offset; - std::vector bv_pass_1; - std::vector bv_pass_2; + std::vector bv_pass_1; + std::vector bv_pass_2; std::vector> pass2_pointer; std::vector> pass2_offset; std::vector pass2_max_pointer; std::vector pass2_max_offset; std::vector pass2_ones; std::vector block_size_lvl_temp; - this->calculate_padding(added_padding, text.size(), tree_max_height, + this->calculate_padding(added_padding, + text.size(), + tree_max_height, max_blk_size); auto is_padded = added_padding > 0 ? 1 : 0; int64_t block_size = max_blk_size; @@ -530,13 +735,13 @@ class BlockTreeFP : public BlockTree { block_text_inx.push_back(i); } if (block_size <= this->max_leaf_length_) { - auto *bv = new pasta::BitVector(block_text_inx.size(), 1); + auto* bv = new pasta::BitVector(block_text_inx.size(), 1); this->block_tree_types_rs_.push_back( new pasta::RankSelect(*bv)); auto p0 = new sdsl::int_vector<>(0, 0); auto o0 = new sdsl::int_vector<>(0, 0); - auto &ptr0 = *p0; - auto &off0 = *o0; + auto& ptr0 = *p0; + auto& off0 = *o0; sdsl::util::bit_compress(ptr0); sdsl::util::bit_compress(off0); this->block_tree_types_.push_back(bv); @@ -551,24 +756,26 @@ class BlockTreeFP : public BlockTree { bool found_back_block = this->max_leaf_length_ * this->tau_ >= block_size; while (block_size > this->max_leaf_length_) { block_size_lvl_temp.push_back(block_size); - auto *bv = new pasta::BitVector(block_text_inx.size(), false); + auto* bv = new pasta::BitVector(block_text_inx.size(), false); auto left = pasta::BitVector(block_text_inx.size(), false); auto right = pasta::BitVector(block_text_inx.size(), false); auto pair_size = 2 * block_size; auto last_block_padded = static_cast(block_text_inx[block_text_inx.size() - 1] + - block_size) != text.size() - ? 1 - : 0; - std::unordered_map, std::vector> pairs( - 0); - std::unordered_map, std::vector> blocks = - std::unordered_map, std::vector>(); + block_size) != text.size() ? + 1 : + 0; + HashMap, std::vector> pairs(0); + HashMap, std::vector> blocks = + HashMap, std::vector>(); for (uint64_t i = 0; i < block_text_inx.size() - last_block_padded; i++) { auto index = block_text_inx[i]; MersenneRabinKarp rk_block = - MersenneRabinKarp(text, sigma_, index, - block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + block_size, + kPrime); MersenneHash mh_block = MersenneHash(text, rk_block.hash_, index, block_size); blocks[mh_block].push_back(i); @@ -602,8 +809,11 @@ class BlockTreeFP : public BlockTree { text.size()) { auto index = block_text_inx[i]; MersenneRabinKarp rk_pair = - MersenneRabinKarp(text, sigma_, index, - pair_size, kPrime); + MersenneRabinKarp(text, + sigma_, + index, + pair_size, + kPrime); MersenneHash mh_pair = MersenneHash(text, rk_pair.hash_, index, pair_size); pairs[mh_pair].push_back(i); @@ -611,7 +821,10 @@ class BlockTreeFP : public BlockTree { } // find pairs MersenneRabinKarp rk_pair_sw = - MersenneRabinKarp(text, sigma_, 0, pair_size, + MersenneRabinKarp(text, + sigma_, + 0, + pair_size, kPrime); for (uint64_t i = 0; i < text.size() - pair_size; i++) { MersenneHash mh_sw = @@ -654,8 +867,11 @@ class BlockTreeFP : public BlockTree { } for (uint64_t i = 0; i < block_text_inx.size() - 1; i++) { MersenneRabinKarp rk_first_occ = - MersenneRabinKarp( - text, sigma_, block_text_inx[i], block_size, kPrime); + MersenneRabinKarp(text, + sigma_, + block_text_inx[i], + block_size, + kPrime); bool followed = (i < block_text_inx.size() - 1) && block_text_inx[i] + block_size == block_text_inx[i + 1] && @@ -666,8 +882,11 @@ class BlockTreeFP : public BlockTree { j < static_cast(block_size) && block_text_inx[i] + j + block_size < text.size(); j++) { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i] + j, block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i] + j, + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (static_cast(b) != i) { @@ -680,8 +899,11 @@ class BlockTreeFP : public BlockTree { rk_first_occ.next(); } } else { - MersenneHash mh_first_occ = MersenneHash( - text, rk_first_occ.hash_, block_text_inx[i], block_size); + MersenneHash mh_first_occ = + MersenneHash(text, + rk_first_occ.hash_, + block_text_inx[i], + block_size); if (blocks.find(mh_first_occ) != blocks.end()) { for (auto b : blocks[mh_first_occ]) { if (static_cast(b) != i) { @@ -704,9 +926,17 @@ class BlockTreeFP : public BlockTree { } this->leaf_size = block_size; block_size *= this->tau_; - pruning_simple(bv_pass_1, blk_lvl, bv_pass_2, pass1_pointer, pass1_offset, - pass2_pointer, pass2_offset, pass2_max_pointer, - pass2_max_offset, pass2_ones, block_size); + pruning_simple(bv_pass_1, + blk_lvl, + bv_pass_2, + pass1_pointer, + pass1_offset, + pass2_pointer, + pass2_offset, + pass2_max_pointer, + pass2_max_offset, + pass2_ones, + block_size); auto size = pass2_pointer[pass2_pointer.size() - 1].size(); found_back_block |= size != 0; if (found_back_block || !this->CUT_FIRST_LEVELS) { @@ -716,12 +946,14 @@ class BlockTreeFP : public BlockTree { *bv_pass_2[bv_pass_2.size() - 1])); auto p1 = new sdsl::int_vector<>( - size, 0, + size, + 0, (8 * sizeof(size_type)) - this->leading_zeros( pass2_max_pointer[pass2_max_pointer.size() - 1])); auto o1 = new sdsl::int_vector<>( - size, 0, + size, + 0, (8 * sizeof(size_type)) - this->leading_zeros( pass2_max_offset[pass2_max_offset.size() - 1])); @@ -749,7 +981,6 @@ class BlockTreeFP : public BlockTree { auto lvl_block_size = block_size_lvl_temp[level]; if (is_padded) { for (size_type j = 0; j < this->tau_; j++) { - if (static_cast(last_block_parent + j * lvl_block_size) < text.size()) { new_size++; @@ -758,19 +989,20 @@ class BlockTreeFP : public BlockTree { } found_back_block |= new_size != pass2_ones[i]; if (found_back_block || !this->CUT_FIRST_LEVELS) { - auto *bit_vector = new pasta::BitVector(new_size, 0); + auto* bit_vector = new pasta::BitVector(new_size, 0); auto pointer = std::vector(); auto offset = std::vector(); size_type pointer_saved = 0; size_type pointer_skipped = 0; - std::unordered_map blocks_skipped; + HashMap blocks_skipped; size_type skip = 0; size_type replace = 0; for (uint64_t j = 0; j < bv_pass_1[pass1_i - 1]->size(); j++) { if ((*bv_pass_1[pass1_i - 1])[j] == 1) { if ((*bv_pass_2[i + 1])[j] == 1) { for (size_type k = 0; - k < this->tau_ && replace * this->tau_ + k < new_size; k++) { + k < this->tau_ && replace * this->tau_ + k < new_size; + k++) { bool x = (*bv_pass_2[i])[(j - skip) * this->tau_ + k]; auto skipper = pointer_skipped + pointer_saved; (*bit_vector)[replace * this->tau_ + k] = x; @@ -796,11 +1028,13 @@ class BlockTreeFP : public BlockTree { } } auto p = new sdsl::int_vector<>( - pointer.size(), 0, + pointer.size(), + 0, (8 * sizeof(size_type)) - this->leading_zeros(pass2_max_pointer[i])); auto o = new sdsl::int_vector<>( - pointer.size(), 0, + pointer.size(), + 0, (8 * sizeof(size_type)) - this->leading_zeros(pass2_max_offset[i])); for (uint64_t j = 0; j < pointer.size(); j++) { (*p)[j] = pointer[j]; @@ -836,9 +1070,13 @@ class BlockTreeFP : public BlockTree { return 0; }; - BlockTreeFP(std::vector &text, size_type tau, - size_type max_leaf_length, size_type s, size_type sigma, - bool cut_first_levels, bool extended_prune) { + BlockTreeFP(std::vector& text, + size_type tau, + size_type max_leaf_length, + size_type s, + size_type sigma, + bool cut_first_levels, + bool extended_prune) { sigma_ = sigma; this->CUT_FIRST_LEVELS = cut_first_levels; this->map_unique_chars(text); @@ -852,21 +1090,6 @@ class BlockTreeFP : public BlockTree { } }; - ~BlockTreeFP() { - for (auto &bt_t : this->block_tree_types_) { - delete bt_t; - } - for (auto &bt_rs : this->block_tree_types_rs_) { - delete bt_rs; - } - for (auto &bt_p : this->block_tree_pointers_) { - delete bt_p; - } - for (auto &bt_o : this->block_tree_offsets_) { - delete bt_o; - } - }; - private: // magic number to indicate that a block is pruned const int PRUNED = -2; @@ -875,10 +1098,16 @@ class BlockTreeFP : public BlockTree { }; template -auto *make_block_tree_fp(std::vector &input, size_type const tau, +auto* make_block_tree_fp(std::vector& input, + size_type const tau, size_type const max_leaf_length) { - return new BlockTreeFP(input, tau, max_leaf_length, 1, - 256, true, true); + return new BlockTreeFP(input, + tau, + max_leaf_length, + 1, + 256, + true, + true); } } // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp new file mode 100644 index 0000000..5b0628a --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp2_seq.hpp @@ -0,0 +1,720 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" + +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFP2 : public BlockTree { + using BitVector = pasta::BitVector; + using Rank = pasta::RankSelect; + + template > + using HashMap = ankerl::unordered_dense::map; + + using RabinKarp = MersenneRabinKarp; + using RabinKarpHash = MersenneHash; + + template + using RabinKarpMap = HashMap; + + using LevelData = internal::sharded::LevelData; + + void construct(const std::vector& text) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded); + scan_blocks(text, current, is_padded); + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); + } + } + + prune(levels); + make_tree(text, levels, padding); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + static void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded) { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const size_t pair_size = 2 * block_size; + + if (num_blocks < 4) { + level.is_internal = std::make_unique(num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap> map(num_blocks - 1); + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + + { + RabinKarp rk(text, + internal::sharded::SIGMA, + 0, + pair_size, + internal::sharded::PRIME); + for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + rk.restart((*level.block_starts)[i]); + RabinKarpHash hash = rk.current_hash(); + map[hash].push_back(i); + } + } + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + RabinKarp rk(text, + internal::sharded::SIGMA, + 0, + pair_size, + internal::sharded::PRIME); + for (size_t i = 0; i < num_blocks - 1 - is_padded; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, markings, block_size); + } + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(num_blocks); + auto& is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_t i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } + level.is_internal_rank = std::make_unique(is_internal); + } + + /// @brief Scan through the windows starting in a block and mark them + /// accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// index. + /// @param markings A vector storing the marks on a block. Marks are 2-bit + /// integers. + /// If the MSB is set, that means that the content of the block and its + /// successor has an earlier occurrence. If the LSB being set means that + /// the content of the block and its predecessor has an earlier occurrence. + /// @param block_size The size of blocks on the current level. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap>& map, + sdsl::int_vector<2>& markings, + const size_t block_size) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found_hash_ptr = map.find(current_hash); + if (found_hash_ptr == map.end()) { + continue; + } + auto& [block_pair_hash, block_indices] = *found_hash_ptr; + + // If the hash we found is just the first block pair in the hash map, + // then the first block pair has no earlier occurrence. + // So we want to skip that one + bool skip_first = current_hash.start_ == block_pair_hash.start_; + + // For all pairs with an earlier occurrence, + for (size_t i = skip_first; i < block_indices.size(); i++) { + auto block_index = block_indices[i]; + markings[block_index] = markings[block_index] | 0b10; + markings[block_index + 1] = markings[block_index + 1] | 0b01; + } + map.erase(found_hash_ptr); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + static void scan_blocks(const std::vector& s, + LevelData& level_data, + const bool is_padded) { + const size_t block_size = level_data.block_size; + const size_t num_blocks = level_data.num_blocks; + const std::vector& block_starts = *level_data.block_starts; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap> links(num_blocks); + for (size_t i = 0; i < num_blocks - is_padded; ++i) { + const RabinKarpHash hash = RabinKarp(s, + internal::sharded::SIGMA, + block_starts[i], + block_size, + internal::sharded::PRIME) + .current_hash(); + links[hash].push_back(i); + } + + // Hash every window and find the first occurrences for every block. + RabinKarp rk(s, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t current_block_index = 0; + current_block_index < num_blocks - is_padded - 1; + ++current_block_index) { + if (static_cast(rk.init_) != block_starts[current_block_index]) { + rk.restart(block_starts[current_block_index]); + } + + if (level_data.next_is_adjacent(current_block_index)) { + scan_windows_in_block(rk, links, level_data, current_block_index); + continue; + } + } + } + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap>& links, + LevelData& level_data, + const size_type current_block_index) { + const BitVector& is_internal = *level_data.is_internal; + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + const auto& [block_hash, found_blocks] = *found; + const size_t num_found_blocks = found_blocks.size(); + // In this case, we are hashing an actual block right now (not just an + // arbitrary window). As a result, the first block in the vector is the + // block we are currently hashing in + for (size_t i = 0; i < num_found_blocks; ++i) { + const size_type block_index = found_blocks[i]; + if (block_index == current_block_index || + (offset > 0 && block_index == current_block_index + 1)) { + continue; + } + (*level_data.pointers)[block_index] = current_block_index; + (*level_data.offsets)[block_index] = offset; + // We increment the counter for the block that is being pointed to + // if the current block is actually a back block + // If the offset is greater than 0, + // then it also overlaps into the next block + const bool is_back_block = !is_internal[block_index]; + (*level_data.counters)[current_block_index] += is_back_block; + (*level_data.counters)[current_block_index + 1] += + is_back_block && (offset > 0); + } + links.erase(found); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + BlockTreeFP2(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length) { + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text); + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp new file mode 100644 index 0000000..7eab044 --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_parlay.hpp @@ -0,0 +1,1090 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFPParParlay : public BlockTree { + constexpr static size_type NO_EARLIER_OCC = -1; + constexpr static size_type PRUNED = -2; + + static constexpr size_type SIGMA = 256; + static constexpr uint128_t K_PRIME = 2305843009213693951ULL; + static constexpr uint8_t MERSENNE_EXPONENT = 61; + + using BitVector = pasta::BitVector; + using Rank = pasta::RankSelect; + + /// A concurrent hash map + template > + using HashMap = parlay::unordered_map; + + /// A rabin karp hasher preconfigured for the current template parameters + using RabinKarp = MersenneRabinKarp; + /// A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// A hash map with rabin karp hashes as keys + template + using RabinKarpMap = + HashMap>; + +#ifdef BT_DBG +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: +#endif + /// @brief Contains data about a block tree level under construction + struct LevelData { + /// Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// The block from which a back block is copying + std::unique_ptr> pointers; + /// The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// The number of back blocks pointing to the block + std::unique_ptr> counters; + /// Block start indices + std::unique_ptr> block_starts; + /// The block size on this level + size_type block_size; + /// The index of the current level. First level is 0, second level is 1 etc. + size_type level_index; + /// The number of blocks on the current level + size_type num_blocks; + + inline LevelData(size_type level_index_, + size_type block_size_, + size_type num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] inline bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + static_cast(block_size) == + (*block_starts)[i + 1]; + } + + /// @brief Checks whether a block is adjacent in the text + /// to its predecessor on this level + [[nodiscard]] inline bool prev_is_adjacent(size_t i) const { + return (*block_starts)[i - 1] + static_cast(block_size) == + (*block_starts)[i]; + } + }; + + void construct(const std::vector& text, const size_t threads) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_DBG + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { + // std::cout << "level " << level << std::endl; + +#ifdef BT_DBG + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads); +#ifdef BT_DBG + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + +#endif + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_DBG + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_DBG + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +#endif + prune(levels); +#ifdef BT_DBG + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif + make_tree(text, levels, padding); +#ifdef BT_DBG + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::list occurrences; + + inline PairOccurrences() + : first_occ_block(std::numeric_limits::max()), + occurrences() {} + + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + inline PairOccurrences(const PairOccurrences&) = default; + + inline PairOccurrences& operator=(const PairOccurrences&) = default; + + inline void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + inline void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } + }; + static_assert(std::is_copy_assignable(), + "Must be copy-assignable for parlay"); + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap map(level.num_blocks); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded) +#endif + { + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_blocks = level.num_blocks; + const size_t num_block_pairs = num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; +#pragma omp single + for (size_t i = 0; i < num_block_pairs; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + RabinKarp rk(text, SIGMA, block_starts[i], pair_size, K_PRIME); + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + map.upsert(hash, [i](std::optional occs) { + if (!occs) { + occs.emplace(i); + } + occs->add_block(i); + occs->update(i); + return *occs; + }); + /* + std::optional occs = map.find(hash); + if (occs) { + occs->add_block(i); + occs->update(i); + } else { + PairOccurrences p(i); + map.insert(hash, p); + }*/ + } +#pragma omp barrier +#ifdef BT_DBG +# pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, omp_get_num_threads())); + const size_t thread_id = omp_get_thread_num(); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, SIGMA, block_starts[start], pair_size, K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + +#ifdef BT_DBG + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(level.num_blocks, 0); + parlay::sequence> entries = + map.entries(); + for (auto& [hash, pair_occs] : entries) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + map.remove(hash); + } +#ifdef BT_DBG + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(level.num_blocks); + + auto& is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[level.num_blocks - 1] = markings[level.num_blocks - 1] != 0b01; + for (size_type i = 0; i < level.num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_DBG + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Scan through the windows starting in a block and mark + /// them + /// accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param current_block_index The index of the block being currently hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap& map, + const size_t block_size, + const size_type current_block_index) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (!found) { + continue; + } + found->update(current_block_index); + } + } + + struct BlockOccurrences { + struct FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + size_type block; + /// @brief The offset into the block at which that first occurrence occurs + size_type offset; + + inline FirstOccurrence(size_type first_occ_block_, + size_type first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + // The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::list occurrences; + std::mutex list_mutex; + + explicit BlockOccurrences() + : first_occ({std::numeric_limits::max(), 0}), + occurrences(), + list_mutex() {} + + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences(), + list_mutex() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences), + list_mutex() {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)), + list_mutex() {} + + inline BlockOccurrences& operator=(const BlockOccurrences& other) { + first_occ = other.first_occ.load(); + occurrences = other.occurrences; + new (&list_mutex) std::mutex; + return *this; + } + + inline void add_block(size_type block_index) { + const std::lock_guard lock(list_mutex); + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_index The offset of that occurrence + inline void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + //||(first_occ_block == block_index && block_offset < first_occ_offset)) { + } + }; + static_assert(std::is_copy_assignable(), + "Must be copy-assignable for parlay"); + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = + std::make_unique>(num_blocks, NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap links(std::max(num_blocks / 20, 4)); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded) +#endif + { + const std::vector& block_starts = *level_data.block_starts; +#pragma omp single + for (size_type i = 0; i < level_data.num_blocks - is_padded; ++i) { + const RabinKarp rk(text, + SIGMA, + block_starts[i], + level_data.block_size, + K_PRIME); + const RabinKarpHash hash = rk.current_hash(); + links.upsert(hash, [i](std::optional occs) { + if (!occs) { + occs.emplace(i); + } + occs->add_block(i); + occs->update(i, 0); + return *occs; + }); + + // ptr->second.add_block(i); + // ptr->second.update(i, 0); + } +#pragma omp barrier + +#ifdef BT_DBG +# pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + + const size_t thread_id = omp_get_thread_num(); + const size_t segment_size = + ceil_div(num_total_iterations, omp_get_num_threads()); + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + SIGMA, + block_starts[start], + level_data.block_size, + K_PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } +#ifdef BT_DBG + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + parlay::sequence> entries = + links.entries(); + for (std::pair entry : entries) { + // The occurrences of all blocks with a given hash + const BlockOccurrences& occs = entry.second; + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + } +#ifdef BT_DBG + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (!found) { + continue; + } + found->update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = PRUNED; + } + + return false; + } + +public: + BlockTreeFPParParlay(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParParlay() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } +}; // namespace pasta + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp new file mode 100644 index 0000000..41ddf94 --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_phmap.hpp @@ -0,0 +1,943 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" + +#include +#include +#include +#include +#include + +using Clock = std::chrono::high_resolution_clock; +using TimePoint = Clock::time_point; +using Duration = Clock::duration; + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +template +class BlockTreeFPParPH : public BlockTree { + using BitVector = pasta::BitVector; + using Rank = pasta::RankSelect; + + /// A concurrent hash map + /*template , + size_t num_submaps = 6, + typename mutex_type = phmap::NullMutex> + using HashMap = phmap::parallel_flat_hash_map< + key_type, + value_type, + hash_type, + phmap::priv::hash_default_eq, + phmap::priv::Allocator< + typename phmap::priv::Pair>, + num_submaps, + mutex_type>;*/ + + template > + using HashMap = ankerl::unordered_dense::map; + + /// A rabin karp hasher preconfigured for the current template parameters + using RabinKarp = MersenneRabinKarp; + /// A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// A hash map with rabin karp hashes as keys + template + using RabinKarpMap = + HashMap>; + + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + +#ifdef BT_DBG +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: +#endif + + void construct(const std::vector& text, const size_t threads) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_DBG + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "level " << level << std::endl; + + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG + std::cout << "scanned block pairs " << std::endl; + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads); +#ifdef BT_DBG + std::cout << "scanned blocks " << std::endl; + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + +#endif + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_DBG + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } +#ifdef BT_DBG + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +#endif + prune(levels); +#ifdef BT_DBG + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif + make_tree(text, levels, padding); +#ifdef BT_DBG + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should + /// be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + RabinKarpMap map(level.num_blocks); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded) +#endif + { + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_blocks = level.num_blocks; + const size_t num_block_pairs = num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; +#pragma omp single + { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = 0; i < num_block_pairs; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + auto ptr = map.find(hash); + if (ptr == map.end()) { + auto [insert_ptr, _] = map.insert({hash, PairOccurrences(i)}); + ptr = insert_ptr; + } + ptr->second.add_block_pair(i); + ptr->second.update(i); + } + } +#pragma omp barrier +#ifdef BT_DBG +# pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, omp_get_num_threads())); + const size_t thread_id = omp_get_thread_num(); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + +#ifdef BT_DBG + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(level.num_blocks, 0); + for (auto it = map.begin(); it != map.end(); ++it) { + const PairOccurrences& pair_occs = it->second; + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + } +#ifdef BT_DBG + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + level.is_internal = std::make_unique(level.num_blocks); + + auto& is_internal = *level.is_internal; + is_internal[0] = true; + is_internal[level.num_blocks - 1] = markings[level.num_blocks - 1] != 0b01; + for (size_type i = 0; i < level.num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_DBG + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Scan through the windows starting in a block and mark + /// them + /// accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param current_block_index The index of the block being currently hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + RabinKarpMap& map, + const size_t block_size, + const size_type current_block_index) { + for (size_t offset = 0; offset < block_size; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + continue; + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map with hashed slices as keys, which map to a vector of links, + // describing a link between a (potential) back block to their source block. + // In addition to the vector, there is a boolean which denotes whether a + // hash has already been processed + RabinKarpMap links(num_blocks); + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded) +#endif + { + const std::vector& block_starts = *level_data.block_starts; +#pragma omp single + for (size_type i = 0; i < level_data.num_blocks - is_padded; ++i) { + const RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[i], + level_data.block_size, + internal::sharded::PRIME); + const RabinKarpHash hash = rk.current_hash(); + auto ptr = links.find(hash); + if (ptr == links.end()) { + auto [insert_ptr, _] = links.emplace(hash, BlockOccurrences(i)); + ptr = insert_ptr; + } + + ptr->second.add_block(i); + ptr->second.update(i, 0); + } +#pragma omp barrier + +#ifdef BT_DBG +# pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + + const size_t thread_id = omp_get_thread_num(); + const size_t segment_size = + internal::sharded::ceil_div(num_total_iterations, + omp_get_num_threads()); + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + level_data.block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } +#ifdef BT_DBG + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + for (auto it = links.cbegin(); it != links.cend(); ++it) { + // The occurrences of all blocks with a given hash + const BlockOccurrences& occs = it->second; + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + } +#ifdef BT_DBG + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find earlier occurrences of blocks. Non-internal blocks will + /// have their respective m_source_blocks and m_offsets entries populated. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param current_block_internal_index The index of the block which the + /// Rabin-Karp hasher is situated in only with respect to *internal blocks* + /// on the current level, disregarding back blocks. + /// @param num_hashes The number of times the Rabin-Karp hasher should hash. + static void scan_windows_in_block(RabinKarp& rk, + RabinKarpMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + found->second.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector. + /// + /// @param text The input text. + /// @param level The level data of the previous level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left. + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are not on the + // last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing to this, + // then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + BlockTreeFPParPH(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParPH() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + }; + } + return true; + } +}; // namespace pasta + +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp new file mode 100644 index 0000000..c3539ab --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sharded.hpp @@ -0,0 +1,1013 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/mpsc_queue/jiffy.hpp" +#include "pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp" +#include "pasta/block_tree/utils/sharded_map.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#define BT_FILL_THRESHOLD 0.5 +#define BT_QUEUE_CAPACITY 163840 + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template typename queue_type = JiffyQueue> +class BlockTreeFPParSharded : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A concurrent queue for communication between threads in the + /// sharded hash map. + template + requires MpscQueue, elem_type> + using Queue = queue_type; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + ShardedMap; + + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + +#ifdef BT_DBG +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; + +private: +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text, const size_t threads) { + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_DBG + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "level " << level << std::endl; + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads); +#ifdef BT_DBG + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + scan_blocks(text, current, is_padded, threads); + +#ifdef BT_DBG + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_DBG + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } +#ifdef BT_DBG + TimePoint now = Clock::now(); + + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +#endif + prune(levels); +#ifdef BT_DBG + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +#endif + make_tree(text, levels, padding); +#ifdef BT_DBG + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(BT_FILL_THRESHOLD, threads, BT_QUEUE_CAPACITY); + + std::atomic_size_t num_threads_finished = 0; + +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, now, is_padded, num_threads_finished) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, num_threads_finished) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether they have + // previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair starting + // at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + // Move the hasher to the current block pair + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[i], + pair_size, + internal::sharded::PRIME); + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it doesn't + // exist, and add the current block to the entry + shard.insert(hash, i); + if (shard.should_handle_queue()) { + shard.handle_queue(); + } + } + num_threads_finished.fetch_add(1); + // Threads might be done before the others with the loop. So essentially, + // we need all threads to wait for the others to finish the loop and + // handle thread events that might come in from the other threads + do { + shard.handle_queue(); + } while (num_threads_finished.load() < num_threads); +#pragma omp barrier +#ifdef BT_DBG +# pragma omp single + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair(rk, map, block_size, i); + } + } + } + +#ifdef BT_DBG + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the given + /// map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_DBG + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior occurrence. + // The LSB is 1 iff the block and its predecessor have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); + +#ifdef BT_DBG + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_DBG + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some block + /// hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block pairs. + auto found = map.find(current_hash); + if (found == map.end()) { + // TODO count how often this actually happens + continue; + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + std::cout << "\n pointers have " << level_data.pointers->size() + << std::endl; + ; + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(BT_FILL_THRESHOLD, threads, BT_QUEUE_CAPACITY); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_threads_finished = 0; +#ifdef BT_DBG + TimePoint now = Clock::now(); +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, now, is_padded, num_threads_finished) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_threads_finished) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = level_data.block_size; + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + for (size_t i = start; i < end; ++i) { + const RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[i], + block_size, + internal::sharded::PRIME); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + // The thread checks whether it should handle the inserts in its queue + if (shard.should_handle_queue()) { + shard.handle_queue(); + } + } + num_threads_finished.fetch_add(1); + // Threads might be done with the loop before others. So essentially, + // we need all threads to wait for the others to finish the loop and + // handle thread events that might come in from the other threads + do { + shard.handle_queue(); + } while (num_threads_finished.load() < num_threads); +#ifdef BT_DBG +# pragma omp single + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } +#endif + + // Hash every window and find the first occurrences for every block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, links, level_data, i); + } + } + } +#ifdef BT_DBG + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // By this point, the map should contain the first occurrences of every + // respective block's content. We then fill the pointers and offsets with + // this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_DBG + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks will + /// have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { + continue; + } + found->second.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with them. + /// + /// @param[in] levels A vector containing data for each level, with the first + /// entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to store + // the number of pruned blocks before the block. + // The invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + std::cout << "\n" << level.pointers->size() << std::endl; + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + BlockTreeFPParSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParSharded() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + /// @brief Validates that a back-pointer actually points to the same text + /// content. + /// @param text The input text. + /// @param level_index The index of the current level. + /// @param block_index The block index. + /// @param block_start The start index of the block's content in the text. + /// @param source_start The start index of the source block's content in the + /// text. + /// @param source_pointer The block index of the source block. + /// @param source_offset The offset from which the block copies out of the + /// source block. + /// @param block_size The block size. + /// @return `true`, iff the pointer is valid. false otherwise + bool debug_validate_pointer(const std::vector& text, + const size_type level_index, + const size_type block_index, + const size_type block_start, + const size_type source_start, + const size_type source_pointer, + const size_type source_offset, + const size_type block_size) const { + if (source_start + block_size > block_start) { + std::cerr << "source overlapping block on level " << level_index + << ":\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + for (size_type i = 0; i < block_size; i++) { + if (text[block_start + i] != text[source_start + i]) { + std::cerr << "source block mismatch on level " << level_index << ": " + << "\n\tBlock Start: " << block_start + << "\n\tSource Start: " << source_start + << "\n\tBlock Size: " << block_size + << "\n\tBlock: " << block_index + << "\n\tSource Block: " << source_pointer + << "\n\tSource Offset: " << source_offset << std::endl; + return false; + } + } + return true; + } +}; // namespace pasta + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp new file mode 100644 index 0000000..c75731d --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_fp_par_sync_sharded.hpp @@ -0,0 +1,1171 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// @tparam queue_type The type of queue to use for communication +/// in the sharded hash map. +template +class BlockTreeFPParShardedSync : public BlockTree { + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + // std::unordered_map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type> + using RabinKarpMap = + SyncShardedMap; + + using LevelData = internal::sharded::LevelData; + using PairOccurrences = internal::sharded::PairOccurrences; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + +private: + /// @brief Constructs the block tree. + /// @param text The input text. + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, current, is_padded, threads, queue_size); +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); + } +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] void print_aggregate(const char* name, + const tlx::Aggregate& agg, + size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// + /// @return The block start indices for the next level of the tree + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param s The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + // Hash each block and store their hashes in the map + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + + const bool is_last_thread = thread_order == num_threads; + + if (is_last_thread) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + const RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + new_num_internal[level]++; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + level.is_internal.reset(); + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + num_back_blocks++; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + BlockTreeFPParShardedSync(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeFPParShardedSync() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/block_tree_sharded.hpp b/include/pasta/block_tree/construction/block_tree_sharded.hpp new file mode 100644 index 0000000..135f226 --- /dev/null +++ b/include/pasta/block_tree/construction/block_tree_sharded.hpp @@ -0,0 +1,1387 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class BlockTreeSharded : public BlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size * sizeof(input_type)) > + 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size * sizeof(input_type)) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding); +#ifdef BT_INSTRUMENT + size_t make_ns = std::chrono::duration_cast( + Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void print_aggregate( + const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which + /// blocks should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end + /// at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings + /// or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector & is_internal, BlockPairMap & map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void scan_windows_in_block_pair( + RabinKarp & rk, + BlockPairMap & map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void scan_windows_in_block_pair_identity( + const std::vector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window + /// size greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = + level_data.num_blocks - is_padded - 1; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp & rk, + BlockMap & links, + LevelData & level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void scan_windows_in_block_identity( + const std::vector& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size / + sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData generate_next_level( + const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(top_level.is_internal.release()); + this->block_tree_types_rs_.push_back( + new Rank(*this->block_tree_types_.back())); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size()); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; + b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector & levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector & levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector & levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = std::min( + first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == PRUNED && child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + + public: + BlockTreeSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } + + ~BlockTreeSharded() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + for (auto& bv : this->block_tree_types_) { + delete bv; + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + }; +} // namespace pasta diff --git a/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp new file mode 100644 index 0000000..b7c149b --- /dev/null +++ b/include/pasta/block_tree/construction/rec_bit_block_tree_sharded.hpp @@ -0,0 +1,1408 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_bit_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveBitBlockTreeSharded + : public RecursiveBitBlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = + MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::span text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_BENCH + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size) > 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::span text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::span& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[pair_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(std::span text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded; + // The number of iterations each thread should do + const size_t segment_size = + internal::sharded::ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = internal::sharded::HASH_MASKS[block_size]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const uint8_t* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::span& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size]; + const uint8_t* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::span text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::span text, + std::vector& levels, + const int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels.size() <= 1 || + levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + if constexpr (recursion_level == 0) { + if (levels.size() == 1) { + top_level.is_internal.release(); + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; ++i) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + ++num_pruned; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + ++num_non_pruned; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + auto& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + RecursiveBitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->num_bits_ = text.size(); + const std::span bytes(reinterpret_cast(text.data().data()), + internal::sharded::ceil_div(text.size(), 8ULL)); + construct(bytes, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp new file mode 100644 index 0000000..6089d6e --- /dev/null +++ b/include/pasta/block_tree/construction/rec_block_tree_sharded.hpp @@ -0,0 +1,1416 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp" +#include "pasta/block_tree/rec_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/byteread.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam input_type The type of the characters in the input string +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveBlockTreeSharded + : public RecursiveBlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const std::vector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve(ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT +# ifdef BT_BENCH + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cout << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cout << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + if (2 * static_cast(current.block_size * sizeof(input_type)) > + 8) { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + if (static_cast(current.block_size * sizeof(input_type)) > 8) { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } else { + scan_blocks(text, + current, + is_padded, + threads, + queue_size); + } +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } + +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cout << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cout << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + /// @brief Returns the ceiling of x / y for x > 0; + /// + /// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c + inline static size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + ((x - 1) / y); + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const std::vector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs + // whether they have previous occurrences. + const size_t segment_size = + std::max(1, ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the + // pair starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if + // it doesn't exist, and add the current block to the + // entry + shard.insert(hash, i); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size * sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + // Try to find the hash in the map, insert a new entry if + // it doesn't exist, and add the current block to the + // entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + scan_windows_in_block_pair_identity(text, + block_starts[i], + pair_size, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + static inline void + scan_windows_in_block_pair_identity(const std::vector& text, + const size_t block_start, + const size_t pair_size, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[pair_size / sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_t offset = 0; offset < num_iterations; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash current_hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + pair_size); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const std::vector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + internal::sharded::HASH_MASKS) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + internal::sharded::HASH_MASKS) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded; + // The number of iterations each thread should do + const size_t segment_size = ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[0], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + } else { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[block_size / sizeof(input_type)]; + for (size_t i = start; i < end; ++i) { + const size_t block_start = block_starts[i]; + const input_type* block_start_ptr = text.data() + block_start; + const uint64_t hash_value = + pasta::copy_le(block_start_ptr) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start, + block_size); + + shard.insert(hash, {i, 0}); + } + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + internal::sharded::SIGMA, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } else { + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + scan_windows_in_block_identity(text, + block_starts[i], + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences + // of every respective block's content. We then fill the + // pointers and offsets with this data and increment counters + // accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + static void + scan_windows_in_block_identity(const std::vector& text, + const size_t block_start, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + const uint64_t HASH_MASK = + internal::sharded::HASH_MASKS[level_data.block_size / + sizeof(input_type)]; + const input_type* block_start_ptr = text.data() + block_start; + for (size_type offset = 0; offset < level_data.block_size; ++offset) { + const uint64_t hash_value = + pasta::copy_le(block_start_ptr + offset) & HASH_MASK; + RabinKarpHash hash(text, + internal::sharded::mix_select(hash_value), + block_start + offset, + level_data.block_size); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData + generate_next_level(const std::vector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const std::vector& text, + std::vector& levels, + int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + + if constexpr (recursion_level > 0) { + auto* bt = new RecursiveDenseBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } + + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.is_internal_rank.reset(); + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + this->leaves_.push_back(text[block_start + b]); + } else { + this->leaves_.push_back(0); + } + } + } + this->amount_of_leaves = leaf_count; + this->compress_leaves(); + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += ceil_div(text_len - last_block_parent_start, block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; i++) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + num_pruned++; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + num_non_pruned++; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveDenseBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } + + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + BitVector& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + RecursiveBlockTreeSharded(const std::vector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->map_unique_chars(text); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } +}; +} // namespace pasta diff --git a/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp new file mode 100644 index 0000000..0ff72cf --- /dev/null +++ b/include/pasta/block_tree/construction/rec_dense_bit_block_tree_sharded.hpp @@ -0,0 +1,1255 @@ + +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/rec_dense_bit_block_tree.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" +#include "pasta/block_tree/utils/sharded_util.hpp" +#include "pasta/block_tree/utils/sync_sharded_map.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +__extension__ typedef unsigned __int128 uint128_t; + +namespace pasta { + +/// @brief A parallel block tree construction algorithm using Rabin-Karp hashes +/// and a sharded hash map. Small blocks are not RK-hashed but rather use the +/// blocks themselves. +/// @tparam size_type The type used for indices etc. (must be a signed integer) +/// in the sharded hash map. +template +class RecursiveDenseBitBlockTreeSharded + : public RecursiveDenseBitBlockTree { + // clang-format off + // ---------------------------------- Type Defs ---------------------------------- + // clang-format on + + using Clock = std::chrono::high_resolution_clock; + using TimePoint = Clock::time_point; + + /// @brief A bit vector + using BitVector = pasta::BitVector; + /// @brief A rank data structure for a bit vector + using Rank = pasta::RankSelect; + + using UseHash = internal::sharded::UseHash; + using LevelData = internal::sharded::LevelData; + using BlockOccurrences = internal::sharded::BlockOccurrences; + using PairOccurrences = internal::sharded::PairOccurrences; + using UpdateBlockOccurrences = + internal::sharded::UpdateBlockOccurrences; + using UpdatePairOccurrences = + internal::sharded::UpdatePairOccurrences; + + /// @brief A sequential hash map used as backing for the sharded hash map. + template + using SeqHashMap = + ankerl::unordered_dense::map>; + + /// @brief A rabin karp hasher preconfigured for the current template + /// parameters + using RabinKarp = + MersenneRabinKarp; + /// @brief A rabin karp hash for the preconfigured rabin karp hasher + using RabinKarpHash = MersenneHash; + + /// @brief A hash map with rabin karp hashes as keys + template update_fn_type, + template typename seq_map_type = SeqHashMap> + using RabinKarpMap = + SyncShardedMap; + + /// @brief A map containing hashed block pairs mapped to their occurrences + using BlockPairMap = RabinKarpMap; + /// @brief A map containing hashed blocks mapped to their occurrences + using BlockMap = RabinKarpMap; + + // clang-format off + // ---------------------------------- End Type Defs ---------------------------------- + // clang-format on + +#ifdef BT_INSTRUMENT +public: + size_t bp_hash_pairs_ns = 0; + size_t bp_scan_pairs_ns = 0; + size_t bp_markings_ns = 0; + size_t bp_bitvec_ns = 0; + + size_t b_hash_blocks_ns = 0; + size_t b_scan_blocks_ns = 0; + size_t b_update_blocks_ns = 0; +#endif + + /// @brief Constructs the block tree. + /// @param text The input text. + /// @param threads The number of threads to use for construction + /// @param queue_size The max number of items in each thread's queue for its + /// hash map + void construct(const pasta::BitVector& text, + const size_t threads, + const size_t queue_size) { +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + const size_type text_len = text.size(); + /// The number of characters a block tree with s top-level blocks and arity + /// of strictly tau would exceed over the text size + int64_t padding; + /// The height of the tree + int64_t tree_height; + /// The size of the largest blocks (i.e. the top level blocks) + int64_t top_block_size; + + this->calculate_padding(padding, text_len, tree_height, top_block_size); + + const bool is_padded = padding > 0; + + std::vector levels; + + // Prepare the top level + levels.emplace_back(0, top_block_size, text_len / top_block_size); + LevelData& top_level = levels.back(); + top_level.block_starts->reserve( + internal::sharded::ceil_div(text_len, top_level.block_size)); + for (size_type i = 0; i < text_len; i += top_level.block_size) { + top_level.block_starts->push_back(i); + } + top_level.block_size = top_block_size; + top_level.num_blocks = top_level.block_starts->size(); + +#ifdef BT_INSTRUMENT + +# ifdef BT_BENCH + const size_t setup_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + + std::cout << " setup=" << setup_ns; +# endif + + size_t pairs_ns = 0; + size_t blocks_ns = 0; + size_t generate_ns = 0; +#endif +#ifdef BT_DBG + std::cerr << "using " << threads << " threads" << std::endl; +#endif + +#ifdef BT_BENCH + std::cout << " queue_capacity=" << queue_size; +#endif + + // Construct the pre-pruned tree level by level + for (size_t level = 0; level < static_cast(tree_height); level++) { +#ifdef BT_DBG + std::cerr << "----------------- level " << level << " -----------------" + << std::endl; +#endif + +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + LevelData& current = levels.back(); + scan_block_pairs(text, + current, + is_padded, + threads, + queue_size); +#ifdef BT_INSTRUMENT + pairs_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + scan_blocks(text, + current, + is_padded, + threads, + queue_size); +#ifdef BT_INSTRUMENT + blocks_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the next level (if we're not at the last level) + if (level < static_cast(tree_height) - 1 && + levels.back().block_size > this->max_leaf_length_ * this->tau_) { + levels.push_back(std::move(generate_next_level(text, current))); +#ifdef BT_INSTRUMENT + generate_ns += std::chrono::duration_cast( + Clock::now() - now) + .count(); +#endif + } else { + break; + } + } +#ifdef BT_INSTRUMENT +# if defined(BT_DBG) + std::cerr << "pairs: " << (pairs_ns / 1'000'000) + << "ms,\n\thash pairs: " << (bp_hash_pairs_ns / 1'000'000) + << "ms,\n\tscan pairs: " << (bp_scan_pairs_ns / 1'000'000) + << "ms,\n\tmarkings: " << (bp_markings_ns / 1'000'000) + << "ms,\n\tbitvec: " << (bp_bitvec_ns / 1'000'000) + << "ms,\nblocks: " << (blocks_ns / 1'000'000) + << "ms,\n\thash blocks: " << (b_hash_blocks_ns / 1'000'000) + << "ms,\n\tscan blocks: " << (b_scan_blocks_ns / 1'000'000) + << "ms,\n\tupdate blocks: " << (b_update_blocks_ns / 1'000'000) + << "ms,\ngenerate_next: " << (generate_ns / 1'000'000) << "ms," + << std::endl; +# elif defined(BT_BENCH) + std::cout << " pairs=" << (pairs_ns / 1'000'000) + << " hash_pairs=" << (bp_hash_pairs_ns / 1'000'000) + << " scan_pairs=" << (bp_scan_pairs_ns / 1'000'000) + << " markings=" << (bp_markings_ns / 1'000'000) + << " bitvec=" << (bp_bitvec_ns / 1'000'000) + << " blocks=" << (blocks_ns / 1'000'000) + << " hash_blocks=" << (b_hash_blocks_ns / 1'000'000) + << " scan_blocks=" << (b_scan_blocks_ns / 1'000'000) + << " update_blocks=" << (b_update_blocks_ns / 1'000'000) + << " generate_next=" << (generate_ns / 1'000'000); + +# endif + now = Clock::now(); +#endif + prune(levels); +#ifdef BT_INSTRUMENT + size_t prune_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +# ifdef BT_DBG + std::cerr << "prune: " << (prune_ns / 1'000'000) << "ms," << std::endl; +# elif defined BT_BENCH + std::cout << " prune=" << (prune_ns / 1'000'000); +# endif +#endif + + make_tree(text, levels, padding, threads, queue_size); +#ifdef BT_INSTRUMENT + size_t make_ns = + std::chrono::duration_cast(Clock::now() - now) + .count(); +# ifdef BT_DBG + std::cout << "make: " << (make_ns / 1'000'000) << "ms" << std::endl; +# elif defined BT_BENCH + std::cout << " make=" << (make_ns / 1'000'000); +# endif +#endif + } + + [[maybe_unused]] static void + print_aggregate(const char* name, + const tlx::Aggregate& agg, + const size_t div = 1) { + printf("%s -> min: %10u, max: %10u, avg: %10.2f, dev: %10.2f, #: %10u\n", + name, + static_cast(agg.min() / div), + static_cast(agg.max() / div), + agg.avg() / static_cast(div), + agg.standard_deviation(0) / static_cast(div), + static_cast(agg.count())); + } + + /// @brief Scan through the blocks pairwise in order to identify which blocks + /// should be replaced with back blocks. + /// + /// @param text The input string. + /// @param level The data for the current level. + /// @param is_padded `true` iff the last block on this level *does not* end at + /// the exact end of the text. + /// @param threads Number of threads to use + /// @param queue_size The size of the queue to use per thread in the sharded + /// hash map. + /// @tparam use_hash Whether to use a Rabin-Karp hasher to hash substrings or + /// use the blocks' contents themselves as hashes. + /// For block sizes greater than 4 bytes, use Rabin-Karp. + /// + template + void scan_block_pairs(const pasta::BitVector& text, + LevelData& level, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + if (level.num_blocks < 4) { + level.is_internal = std::make_unique(level.num_blocks, true); + level.is_internal_rank = std::make_unique(*level.is_internal); + return; + } + + // A map containing hashed block pairs mapped to their indices of the + // pairs' first block respectively + BlockPairMap map(threads, queue_size); + + std::atomic_size_t threads_done = 0; + std::atomic_bool last_done = false; + auto& barrier = map.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, \ + map, \ + text, \ + now, \ + is_padded, \ + threads_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits, \ + threads, \ + std::cout) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level, map, text, is_padded, threads_done, last_done, barrier) +#endif + { + const size_t thread_id = omp_get_thread_num(); + typename BlockPairMap::Shard shard = map.get_shard(thread_id); + const size_t num_threads = omp_get_num_threads(); + const size_t num_block_pairs = level.num_blocks - 1 - is_padded; + const size_t block_size = level.block_size; + const size_t pair_size = 2 * block_size; + const auto& block_starts = *level.block_starts; + + // Hash every window and determine for all block pairs whether + // they have previous occurrences. + const size_t segment_size = std::max( + 1, + internal::sharded::ceil_div(num_block_pairs, num_threads)); + + // Start and end index of the current thread's segment + const auto start = thread_id * segment_size; + const auto end = + std::min(num_block_pairs, (thread_id + 1) * segment_size); + + if constexpr (use_hash == UseHash::RABIN_KARP) { + RabinKarp rk(text, + block_starts[0], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + // If the next block is not adjacent, we cannot hash the pair + // starting at the current block + if (!level.next_is_adjacent(i)) { + continue; + } + rk.restart(block_starts[i]); + // Move the hasher to the current block pair + RabinKarpHash hash = rk.current_hash(); + // Try to find the hash in the map, insert a new entry if it + // doesn't exist, and add the current block to the entry + shard.insert(hash, i); + } + } + + if (const size_t thread_order = + threads_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + // Now, we handle the queue asynchronously + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + { + bp_hash_pairs_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + + if (start < static_cast(num_block_pairs)) { + RabinKarp rk(text, + block_starts[start], + pair_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level.next_is_adjacent(i) | !level.next_is_adjacent(i + 1)) { + continue; + } + if (block_starts[i] != static_cast(rk.init_)) { + rk.restart(block_starts[i]); + } + scan_windows_in_block_pair(rk, + map, + block_size, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } + +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + bp_scan_pairs_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : map.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Pair Map Loads ", map_loads); + print_aggregate("Pair Map Hits ", scan_hits); + print_aggregate("Pair Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Pair Handle Queue (ms) ", finish_idle_ns, 1'000'000); + + BT_ASSERT(map.num_inserts_.load() == map.size()); +# endif +#endif + level.is_internal = std::make_unique(level.num_blocks); + fill_is_internal(*level.is_internal, map); + level.is_internal_rank = std::make_unique(*level.is_internal); + } + + /// @brief Fills the bit vector `is_internal` based on the values in the + /// given map. + /// @param is_internal An unfilled bit vector with a bit for each block on + /// this level. + /// @param map A map, mapping hashed block pairs to their first occurrence's + /// block index. + void fill_is_internal(BitVector& is_internal, BlockPairMap& map) { + const size_type num_blocks = is_internal.size(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); +#endif + // Set up the packed array holding the markings for each block. + // Each mark is a 2-bit number. + // The MSB is 1 iff the block and its successor have a prior + // occurrence. The LSB is 1 iff the block and its predecessor + // have a prior occurrence. + sdsl::int_vector<2> markings(num_blocks, 0); + map.for_each( + [&markings](const RabinKarpHash&, const PairOccurrences& pair_occs) { + for (const size_type occ : pair_occs.occurrences) { + if (pair_occs.first_occ_block < occ) { + markings[occ] = markings[occ] | 0b10; + markings[occ + 1] = markings[occ + 1] | 0b01; + } + } + }); +#ifdef BT_INSTRUMENT + bp_markings_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); +#endif + + // Generate the bit vector indicating which blocks are internal + is_internal[0] = true; + is_internal[num_blocks - 1] = markings[num_blocks - 1] != 0b01; + for (size_type i = 0; i < num_blocks - 1; ++i) { + const bool block_is_internal = markings[i] != 0b11; + is_internal[i] = block_is_internal; + } +#ifdef BT_INSTRUMENT + bp_bitvec_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scan through the windows starting in a block and mark + /// them accordingly if they represent the earliest occurrence of some + /// block hash. + /// + /// The supplied `RabinKarp` hasher must be at the start of the block. + /// @param rk A Rabin-Karp hasher whose state is at the start of the block. + /// @param map The map containing the hashes of block pairs mapped to their + /// block indexes at which they occur. + /// @param num_iterations The number of contiguous windows to hash. + /// @param current_block_index The index of the block being currently + /// hashed. + static inline void + scan_windows_in_block_pair(RabinKarp& rk, + BlockPairMap& map, + const size_t num_iterations, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& agg +#endif + ) { + for (size_t offset = 0; offset < num_iterations; ++offset, rk.next()) { + RabinKarpHash current_hash = rk.current_hash(); + // Find the hash of the current window among the hashed block + // pairs. + auto found = map.find(current_hash); + if (found == map.end()) { +#ifdef BT_INSTRUMENT + agg.add(0); + continue; + } else { + agg.add(100); +#else + continue; +#endif + } + PairOccurrences& occurrences = found->second; + occurrences.update(current_block_index); + } + } + + /// @brief Determine the positions for each block's earliest occurrence if + /// there is any. + /// + /// @param text The input text + /// @param level_data The data for the current level + /// @param is_padded true, iff the last block of the level extends past the + /// end of the text + /// @param threads The number of threads to use during construction. + /// @param queue_size The max number of items in each thread's queues. + /// @tparam use_hash Determines whether to use a rabin karp hash for hashing + /// text windows or to use the block's content as a hash. For any window size + /// greater than 8 bytes, use Rabin-Karp. + template + void scan_blocks(const pasta::BitVector& text, + LevelData& level_data, + const bool is_padded, + const size_t threads, + const size_t queue_size) { + const size_t num_blocks = level_data.num_blocks; + + level_data.pointers = std::make_unique>( + num_blocks, + internal::sharded::NO_EARLIER_OCC); + level_data.offsets = + std::make_unique>(num_blocks, 0); + level_data.counters = + std::make_unique>(num_blocks, 0); + + if (num_blocks <= 2) { + return; + } + + // A map hashing blocks and saving where they occur. + BlockMap links(threads, queue_size); + + // The number of threads finished with hashing blocks + std::atomic_size_t num_done = 0; + // Whether the last thread is done + std::atomic_bool last_done = false; + auto& barrier = links.barrier(); +#ifdef BT_INSTRUMENT + TimePoint now = Clock::now(); + tlx::Aggregate scan_hits; + tlx::Aggregate start_idle_ns; + tlx::Aggregate finish_idle_ns; + tlx::Aggregate total_idle_ns; + tlx::Aggregate handle_queue_ns; + +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, \ + text, \ + links, \ + now, \ + is_padded, \ + num_done, \ + last_done, \ + barrier, \ + start_idle_ns, \ + finish_idle_ns, \ + total_idle_ns, \ + handle_queue_ns, \ + scan_hits) +#else +# pragma omp parallel default(none) num_threads(threads) \ + shared(level_data, text, links, is_padded, num_done, last_done, barrier) +#endif + { + const size_t num_threads = omp_get_num_threads(); + const size_t thread_id = omp_get_thread_num(); + typename BlockMap::Shard shard = links.get_shard(thread_id); + const size_t block_size = + std::min(level_data.block_size, text.size()); + const std::vector& block_starts = *level_data.block_starts; + // Number of total iterations the for loop should do + const size_t num_total_iterations = level_data.num_blocks - is_padded; + // The number of iterations each thread should do + const size_t segment_size = + internal::sharded::ceil_div(num_total_iterations, num_threads); + // The start and end index of the current thread's segment + const size_t start = thread_id * segment_size; + const size_t end = std::min(num_total_iterations, + (thread_id + 1) * segment_size); + + // Hash each block and store their hashes in the map + RabinKarp rk(text, block_starts[0], block_size, internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + rk.restart(block_starts[i]); + RabinKarpHash hash = rk.current_hash(); + shard.insert(hash, {i, 0}); + } + + if (const size_t thread_order = + num_done.fetch_add(1, std::memory_order_acq_rel) + 1; + thread_order == num_threads) { + last_done.store(true, std::memory_order_release); + } + + while (!last_done.load(std::memory_order::acquire)) { + shard.handle_queue_sync(false); + } + barrier.arrive_and_drop(); + shard.handle_queue(); +#pragma omp barrier +#pragma omp single +#ifdef BT_INSTRUMENT + + { + b_hash_blocks_ns += + std::chrono::duration_cast(Clock::now() - + now) + .count(); + now = Clock::now(); + } + + tlx::Aggregate thread_scan_hits; +#else + { + } +#endif + // Hash every window and find the first occurrences for every + // block. + if (start < block_starts.size() - is_padded) { + RabinKarp rk(text, + block_starts[start], + block_size, + internal::sharded::PRIME); + for (size_t i = start; i < end; ++i) { + if (!level_data.next_is_adjacent(i)) { + continue; + } + if (static_cast(rk.init_) != block_starts[i]) { + rk.restart(block_starts[i]); + } + scan_windows_in_block(rk, + links, + level_data, + i +#ifdef BT_INSTRUMENT + , + thread_scan_hits +#endif + ); + } + } +#ifdef BT_INSTRUMENT + auto& start_idle = shard.start_idle_ns(); + auto& finish_idle = shard.finish_idle_ns(); + auto& handle_queue = shard.handle_queue_ns(); + +# pragma omp critical + { + start_idle_ns.add(start_idle.sum()); + finish_idle_ns.add(finish_idle.sum()); + total_idle_ns.add(start_idle.sum() + finish_idle.sum()); + handle_queue_ns.add(handle_queue.sum()); + scan_hits += thread_scan_hits; + }; +#endif + } +#ifdef BT_INSTRUMENT + b_scan_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); + now = Clock::now(); + +# ifdef BT_DBG + tlx::Aggregate map_loads; + + for (size_t load : links.map_loads()) { + map_loads.add(load); + } + + print_aggregate("Block Map Loads ", map_loads); + print_aggregate("Block Map Hits ", scan_hits); + print_aggregate("Block Idle (ms) ", total_idle_ns, 1'000'000); + print_aggregate("Block Handle Queue (ms)", finish_idle_ns, 1'000'000); + + BT_ASSERT(links.num_inserts_.load() == links.size()); +# endif +#endif + + // By this point, the map should contain the first occurrences of + // every respective block's content. We then fill the pointers + // and offsets with this data and increment counters accordingly + links.for_each( + [&level_data](const RabinKarpHash&, const BlockOccurrences& occs) { + auto first_occ = occs.first_occ.load(); + for (const size_type occ : occs.occurrences) { + if (occ == first_occ.block || + (first_occ.offset > 0 && occ == first_occ.block + 1)) { + continue; + } + + (*level_data.pointers)[occ] = first_occ.block; + (*level_data.offsets)[occ] = first_occ.offset; + const bool is_back_block = !(*level_data.is_internal)[occ]; + (*level_data.counters)[first_occ.block] += 1; + (*level_data.counters)[first_occ.block + 1] += + is_back_block && (first_occ.offset > 0); + } + }); + +#ifdef BT_INSTRUMENT + b_update_blocks_ns += + std::chrono::duration_cast(Clock::now() - now) + .count(); +#endif + } + + /// @brief Scans through block-sized windows starting inside one block and + /// tries to find blocks with matching hashes in the map. Such blocks + /// will have their earliest occurrence update. + /// @param rk A Rabin-Karp hasher whose current state is at a block start. + /// @param links A map whose keys are hashed blocks and the values + /// are all block indices of blocks matching the hash in ascending order. + /// @param level_data The data for the current level. + /// @param current_block_index The index of the block which the + /// Rabin-Karp hasher is situated in. + static void scan_windows_in_block(RabinKarp& rk, + BlockMap& links, + LevelData& level_data, + const size_type current_block_index +#ifdef BT_INSTRUMENT + , + tlx::Aggregate& hits +#endif + ) { + for (size_type offset = 0; offset < level_data.block_size; + ++offset, rk.next()) { + RabinKarpHash hash = rk.current_hash(); + // Find all blocks in the multimap that match our hash + auto found = links.find(hash); + if (found == links.end()) { +#ifdef BT_INSTRUMENT + hits.add(0.0); + continue; + } else { + hits.add(100.0); +#else + continue; +#endif + } + BlockOccurrences& occurrences = found->second; + occurrences.update(current_block_index, offset); + } + } + + /// @brief Generate the block size, number of block and block start indices + /// for the next level. + /// + /// This depends on the current level's block size, number of blocks and + /// is_internal bit vector being filled. + /// + /// @param text The input text. + /// @param level The level data of the current level. + /// @return The level data of the next level. + [[nodiscard]] LevelData generate_next_level(const pasta::BitVector& text, + const LevelData& level) const { + const size_t block_size = level.block_size; + const size_t num_blocks = level.num_blocks; + const auto& is_internal = *level.is_internal; + const size_t next_block_size = block_size / this->tau_; + + std::vector new_block_starts; + new_block_starts.reserve(num_blocks * this->tau_); + for (size_t i = 0; i < num_blocks; ++i) { + if (!is_internal[i]) { + continue; + } + + // We generate up to tau new blocks for each internal block, + // excluding blocks that start past the end of the text + const auto parent_block_start = (*level.block_starts)[i]; + for (size_t j = 0, current_block_start = parent_block_start; + j < static_cast(this->tau_) && + current_block_start < text.size(); + ++j, current_block_start += next_block_size) { + new_block_starts.push_back(current_block_start); + } + } + + LevelData next_level(level.level_index + 1, + next_block_size, + new_block_starts.size()); + next_level.block_starts = + std::make_unique>(std::move(new_block_starts)); + return next_level; + } + + /// + /// @brief Takes a vector of levels and fills the block tree fields with + /// them. + /// + /// @param[in] levels A vector containing data for each level, with the + /// first entry corresponding to the topmost level. + /// + void make_tree(const pasta::BitVector& text, + std::vector& levels, + const int64_t padding, + const size_t threads, + const size_t queue_size) { + const bool is_padded = padding > 0; + + // Count the current number of internal blocks per level + std::vector new_num_internal(levels.size(), 0); + for (size_t level = 0; level < levels.size(); level++) { + for (size_t block = 0; block < levels[level].is_internal->size(); + block++) { + if ((*levels[level].is_internal)[block]) { + ++new_num_internal[level]; + } + } + } + + // Create first level + bool found_back_block = levels.size() <= 1 || + levels[0].is_internal->size() > + static_cast(new_num_internal[0]) || + !this->CUT_FIRST_LEVELS; + LevelData& top_level = levels.front(); + if (found_back_block) { + const size_t n = top_level.num_blocks; + const size_t num_internal = new_num_internal[0]; + auto pointers = new sdsl::int_vector<>(n - num_internal, 0); + auto offsets = new sdsl::int_vector<>(n - num_internal, 0); + size_t num_back_blocks = 0; + for (size_t i = 0; i < n; i++) { + // if a back block is found, add its pointer and offset + if (!(*top_level.is_internal)[i]) { + (*pointers)[num_back_blocks] = (*top_level.pointers)[i]; + (*offsets)[num_back_blocks] = (*top_level.offsets)[i]; + num_back_blocks++; + } + } + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = new RecursiveDenseBitBlockTreeSharded( + *top_level.is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + } else { + this->block_tree_types_.push_back(top_level.is_internal.get()); + this->block_tree_types_rs_.push_back(new Rank(*top_level.is_internal)); + if (levels.size() > 1) { + top_level.is_internal.release(); + } + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(top_level.block_size); + } + top_level.pointers.reset(); + top_level.offsets.reset(); + top_level.counters.reset(); + + // Add level data to the tree + for (size_t level_index = 1; level_index < levels.size(); level_index++) { + LevelData& level = levels[level_index]; + LevelData& previous_level = levels[level_index - 1]; + found_back_block |= static_cast(new_num_internal[level_index]) < + levels[level_index].is_internal->size(); + if (!found_back_block && level_index < levels.size() - 1) { + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + continue; + } + + make_tree_level(levels, + new_num_internal, + level_index, + is_padded, + text.size(), + threads, + queue_size); + + // We don't need these anymore + if (level_index < levels.size() - 1) { + level.is_internal.reset(); + } + level.pointers.reset(); + level.offsets.reset(); + level.counters.reset(); + previous_level.block_starts.reset(); + } + + this->leaf_size = levels.back().block_size / this->tau_; + // Construct the leaf string + int64_t leaf_count = 0; + auto& last_is_internal = *levels.back().is_internal; + std::vector& last_block_starts = *levels.back().block_starts; + size_t bit_index = 0; + size_t final_num_internals = 0; + // the rank DS is broken so we count manually + // levels.back().is_internal_rank->rank1(last_is_internal.size()); + for (const bool b : last_is_internal) { + if (b) { + final_num_internals++; + } + } + + this->leaf_bits_ = std::make_unique( + final_num_internals * this->leaf_size * this->tau_, + false); +#ifdef BT_DBG + std::cout << "last size: " << last_is_internal.size() << std::endl; + std::cout << "tau: " << this->tau_ << "\nleaf_size: " << this->leaf_size + << "\nfinal_num_internals: " << final_num_internals + << "\nleaf creation size: " + << final_num_internals * this->leaf_size * this->tau_ / 8 + << std::endl; + size_t padded = 0; + size_t non_padded = 0; +#endif + for (size_t block = 0; block < last_is_internal.size(); block++) { + if (!last_is_internal[block]) { + continue; + } + const size_type block_start = last_block_starts[block]; + // For every leaf on the last level, we have tau leaf blocks + leaf_count += this->tau_; + // Iterate through all characters in this child and + // add them to the leaf string + for (size_t b = 0; b < static_cast(this->leaf_size * this->tau_); + b++) { + if (static_cast(block_start + b) < text.size()) { + (*this->leaf_bits_)[bit_index++] = + static_cast(text[block_start + b]); +#ifdef BT_DBG + non_padded++; +#endif + } else { + (*this->leaf_bits_)[bit_index++] = false; +#ifdef BT_DBG + padded++; +#endif + } + } + } +#ifdef BT_DBG + std::cout << "padded: " << padded << std::endl; + std::cout << "non_padded: " << non_padded << std::endl; +#endif + if constexpr (recursion_level == 0) { + if (levels.size() == 1) { + top_level.is_internal.release(); + } + } + this->amount_of_leaves = leaf_count; + } + + /// @brief Generates a level and adds the relevant data to the block tree. + /// + /// @param levels The vector of levels of the tree. + /// @param level_index The index of the level to generate. This must be + /// strictly greater than 0. + /// @param is_padded Whether there is padding in the last block of the tree + void make_tree_level(std::vector& levels, + const std::vector& new_num_internal, + const size_t level_index, + const bool is_padded, + const size_t text_len, + const size_t threads, + const size_t queue_size) { + LevelData& previous_level = levels[level_index - 1]; + LevelData& level = levels[level_index]; + + size_type new_size = + (new_num_internal[level_index - 1] - is_padded) * this->tau_; + // Determine the number of children the last block generated + if (is_padded) { + const size_type last_block_parent_start = + previous_level.block_starts->back(); + const size_type block_size = level.block_size; + new_size += + internal::sharded::ceil_div(text_len - last_block_parent_start, + block_size); + } + previous_level.block_starts.reset(); + const size_type num_internal = new_num_internal[level_index]; + + // Allocate new vectors for the tree + auto* is_internal = new BitVector(new_size); + auto* pointers = new sdsl::int_vector<>(new_size - num_internal, 0); + auto* offsets = new sdsl::int_vector<>(new_size - num_internal, 0); + + // Number of non-pruned blocks before the current block + size_type num_non_pruned = 0; + // Number of back blocks before the current block + size_type num_back_blocks = 0; + // Number of pruned blocks before the current block + size_type num_pruned = 0; + + // We will reuse the allocated memory of the pointers vector to + // store the number of pruned blocks before the block. The + // invariant is that all values up to i are overwritten while all + // values starting after i will still be valid pointers + // This contains the number of pruned blocks before the block i + std::vector& prefix_pruned_blocks = *level.pointers; + for (size_type i = 0; i < level.num_blocks; ++i) { + const size_type ptr = (*level.pointers)[i]; + prefix_pruned_blocks[i] = num_pruned; + + // If the current block is not pruned, add it to the new tree + if (ptr == internal::sharded::PRUNED) { + ++num_pruned; + continue; + } + + // Add it to the is_internal bit vector + const bool block_is_internal = (*level.is_internal)[i]; + (*is_internal)[num_non_pruned] = block_is_internal; + ++num_non_pruned; + + if (block_is_internal) { + continue; + } + + // If it is a back block, add its pointer and offset + const size_type offset = (*level.offsets)[i]; + + (*pointers)[num_back_blocks] = ptr - prefix_pruned_blocks[ptr]; + (*offsets)[num_back_blocks] = offset; + ++num_back_blocks; + } + + sdsl::util::bit_compress(*pointers); + sdsl::util::bit_compress(*offsets); + if constexpr (recursion_level > 0) { + auto* bt = + new RecursiveDenseBitBlockTreeSharded( + *is_internal, + this->tau_, + this->s_, + this->max_leaf_length_, + threads, + queue_size); + this->block_tree_types_.push_back(bt); + this->block_tree_types_.back()->add_bit_rank_support(threads); + this->block_tree_types_rs_.push_back(bt); + delete is_internal; + } else { + this->block_tree_types_.push_back(is_internal); + this->block_tree_types_rs_.push_back(new Rank(*is_internal)); + } + this->block_tree_pointers_.push_back(pointers); + this->block_tree_offsets_.push_back(offsets); + this->block_size_lvl_.push_back(level.block_size); + } + + /// @brief Prunes the tree of unnecessary nodes. + /// @param levels The levels of the tre represented as a vector of levels. + void prune(std::vector& levels) { + // We need to traverse the block tree in post order, + // handling children from right to left + for (int block_index = levels[0].num_blocks - 1; block_index >= 0; + --block_index) { + prune_block(levels, 0, block_index); + } + } + + /// @brief Prunes a block and its descendants of unnecessary internal nodes. + /// @param levels The WIP levels of the tree. + /// @param level_index The level of the block to prune. + /// @param block_index The index of the block to prune. + /// @return Whether this block is/stays internal after the pruning process + bool prune_block(std::vector& levels, + const size_t level_index, + const size_t block_index) const { + LevelData& level = levels[level_index]; + auto& is_internal = *level.is_internal; + + // If the current block is a back block already, there is nothing + // to prune + if (!is_internal[block_index]) { + return false; + } + + const size_type first_child = + level.is_internal_rank->rank1(block_index) * this->tau_; + + bool has_internal_children = false; + + // On the last level, all blocks just have leaves as children, + // none of which can be pointed to. So only recurse, if we are + // not on the last level. + if (level_index < levels.size() - 1) { + const size_type last_child = + std::min(first_child + this->tau_ - 1, + levels[level_index + 1].is_internal->size() - 1); + // Iterate through children in reverse + for (size_type child = last_child; child >= first_child; --child) { + has_internal_children |= prune_block(levels, level_index + 1, child); + } + } + + // If any of the children is internal, this block stays internal + // as well + if (has_internal_children) { + return true; + } + + const size_type pointer = (*level.pointers)[block_index]; + const size_type offset = (*level.offsets)[block_index]; + const size_type counter = (*level.counters)[block_index]; + // If there is no earlier occurrence or there are blocks pointing + // to this, then this must stay internal + if (pointer == internal::sharded::NO_EARLIER_OCC || counter > 0) { + return true; + } + + // Now we know that there is an earlier occurrence, + // and nothing is pointing here. + // We will make this block here into a back block... + is_internal[block_index] = false; + (*level.counters)[pointer] += 1; + (*level.counters)[pointer + 1] += offset > 0; + + if (level_index == levels.size() - 1) { + return false; + } + + // ...and mark the children as pruned + LevelData& child_level = levels[level_index + 1]; + const size_type last_child = + std::min(first_child + this->tau_ - 1, + child_level.is_internal->size() - 1); + for (size_type child = last_child; child >= first_child; --child) { + const size_type child_pointer = (*child_level.pointers)[child]; + const size_type child_offset = (*child_level.offsets)[child]; +#ifdef BT_DBG + if (!(*child_level.is_internal)[child] && child_pointer < 0) { + std::cout << "non-internal node missing pointer" << std::endl; + std::cout << level_index << ", " << block_index << " / " + << child_level.is_internal->size() << std::endl; + } else if (child_pointer == internal::sharded::PRUNED && + child_pointer < 0) { + std::cout << "pruned node missing pointer" << std::endl; + } + BT_ASSERT(!(*child_level.is_internal)[child] || + child_pointer == internal::sharded::PRUNED); + BT_ASSERT(child_pointer >= 0); +#endif + // Decrement the counter of where the child points + (*child_level.counters)[child_pointer] -= 1; + (*child_level.counters)[child_pointer + 1] -= child_offset > 0; + // Mark the child as pruned + (*child_level.pointers)[child] = internal::sharded::PRUNED; + } + + return false; + } + +public: + RecursiveDenseBitBlockTreeSharded(const pasta::BitVector& text, + const size_t arity, + const size_t root_arity, + const size_t max_leaf_length, + const size_t threads, + const size_t queue_size) { + const auto old = omp_get_max_threads(); + const auto old_dynamic = omp_get_dynamic(); + omp_set_dynamic(0); + omp_set_num_threads(static_cast(threads)); + this->tau_ = arity; + this->s_ = root_arity; + this->max_leaf_length_ = max_leaf_length; + this->num_bits_ = text.size(); + construct(text, threads, queue_size); + omp_set_dynamic(old_dynamic); + omp_set_num_threads(old); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/rec_bit_block_tree.hpp b/include/pasta/block_tree/rec_bit_block_tree.hpp new file mode 100644 index 0000000..775e219 --- /dev/null +++ b/include/pasta/block_tree/rec_bit_block_tree.hpp @@ -0,0 +1,855 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class RecursiveBitBlockTree { +public: + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = + std::conditional_t, + pasta::BitVector>; + using IsInternalRankType = + std::conditional_t, + pasta::RankSelect>; + + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits_; + bool rank_support = false; + /// Recursively compress the bit vectors of the tree + std::vector block_tree_types_; + std::vector block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + ~RecursiveBitBlockTree() { + for (const IsInternalType* b : this->block_tree_types_) { + delete b; + } + // in any other case, block_tree_types_ and block_tree_types_rs_ point to + // the same object (a recursive block tree), so we may only free them once + if constexpr (recursion_level == 0) { + for (const RankSelect* rs : + this->block_tree_types_rs_) { + delete rs; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + [[nodiscard]] size_t size() const { + return num_bits_; + } + + bool operator[](const size_type bit_index) const { + return access(bit_index); + } + + bool access(const size_type bit_index) const { + const int64_t byte_index = bit_index / 8; + const int64_t bit_offset = bit_index % 8; + + int64_t block_size = block_size_lvl_[0]; + int64_t block_index = byte_index / block_size; + int64_t off = byte_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + } + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + off]]; + return ((1 << bit_offset) & byte) != 0; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / (block_size * 8); + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size * 8; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size * 8 - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = (current_block * block_size * 8) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_ones = std::popcount(byte); + if (rank > num_ones) { + rank -= num_ones; + pos += 8; + ++byte_offset; + } else { + for (size_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) > 0; + } + } + } + return pos; + } + + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size * 8 - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size * 8; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (top_block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset * 8; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size * 8 - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size * 8; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset * 8; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset) * 8; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset * 8; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + size_t byte_offset = 0; + while (rank > 0) { + const uint8_t byte = + decompress_map_[compressed_leaves_[current_block * leaf_size + + byte_offset]]; + const uint8_t num_zeros = 8 - std::popcount(byte); + if (rank > num_zeros) { + rank -= num_zeros; + pos += 8; + byte_offset++; + } else { + for (size_t bit = 0; bit < 8 && rank > 0; ++bit) { + pos++; + rank -= ((1 << bit) & byte) == 0; + } + } + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const size_t byte_index = bit_index / 8; + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = byte_index / block_size; + size_t block_offset = byte_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[prefix_leaves * leaf_size + + block]]; + rank += std::popcount(byte); + } + for (size_t block = 0; block < block_offset; block++) { + const uint8_t byte = + decompress_map_[compressed_leaves_[block_index * leaf_size + block]]; + rank += std::popcount(byte); + } + + // Masks to remove bits from the last byte, + // that aren't part of the ran query + static constexpr std::array MASKS = { + 0b0000'0000, + 0b0000'0001, + 0b0000'0011, + 0b0000'0111, + 0b0000'1111, + 0b0001'1111, + 0b0011'1111, + 0b0111'1111, + }; + rank += std::popcount( + decompress_map_[compressed_leaves_[block_index * leaf_size + + block_offset]] & + MASKS[bit_index % 8]); + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + [[nodiscard]] size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + + size_t delta_size = 0; + for (const auto* bt : block_tree_types_) { + if constexpr (types_is_block_tree) { + space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); + } else { + space_usage += bt->size() / 8; + delta_size += bt->size() / 8; + } + } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif + if constexpr (recursion_level == 0) { + for (const auto* rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + delta_size += rs->space_usage(); + } +#ifdef BT_DBG + std::cout << "rs size: " << delta_size << std::endl; +#endif + } + delta_size = 0; + for (const auto iv : block_tree_pointers_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); + ; + } +#ifdef BT_DBG + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; +#endif + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); + } +#ifdef BT_DBG + std::cout << "offs size: " << delta_size << std::endl; +#endif + space_usage += block_size_lvl_.size() * + sizeof(typename decltype(block_size_lvl_)::value_type); + space_usage += block_per_lvl_.size() * + sizeof(typename decltype(block_per_lvl_)::value_type); + + if (rank_support) { + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + } + + // space_usage += leaves_.size() * sizeof(uint8_t); + space_usage += sdsl::size_in_bytes(compressed_leaves_); +#ifdef BT_DBG + std::cout << "leaf size: " << sdsl::size_in_bytes(compressed_leaves_) + << std::endl; +#endif + space_usage += compress_map_.size(); + + return space_usage; + }; + + void + add_bit_rank_support(size_t threads = std::thread::hardware_concurrency()) { + if (rank_support) { + return; + } + rank_support = true; + + if constexpr (recursion_level == 0) { + threads = 1; + } + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + +#pragma omp parallel for default(none) num_threads(threads) + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + } + +protected: + void compress_leaves() { + // Holds a 1 on every char that exists + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + const size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + const uint8_t compressed_byte = + compressed_leaves_[leaf_index * leaf_size + i]; + const uint8_t byte = decompress_map_[compressed_byte]; + result += std::popcount(byte); + } + return result; + } +}; + +template +using BitBlockTree = RecursiveBitBlockTree; + +} // namespace pasta + +/******************************************************************************/ diff --git a/include/pasta/block_tree/rec_block_tree.hpp b/include/pasta/block_tree/rec_block_tree.hpp new file mode 100644 index 0000000..10b5ac1 --- /dev/null +++ b/include/pasta/block_tree/rec_block_tree.hpp @@ -0,0 +1,830 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { +template +class RecursiveBlockTree { +public: + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = + std::conditional_t, + pasta::BitVector>; + using IsInternalRankType = + std::conditional_t, + pasta::RankSelect>; + + /// @brief If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + size_type tau_; + size_type max_leaf_length_; + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + bool rank_support = false; + std::vector block_tree_types_; + std::vector block_tree_types_rs_; + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector block_per_lvl_; + std::vector leaves_; + + std::vector compress_map_; + std::vector decompress_map_; + sdsl::int_vector<> compressed_leaves_; + + ankerl::unordered_dense::map chars_index_; + std::vector chars_; + size_type u_chars_; + std::vector> top_level_c_ranks_; + std::vector>> c_ranks_; + std::vector>> pointer_c_ranks_; + + ~RecursiveBlockTree() { + for (auto& rank : this->block_tree_types_rs_) { + delete rank; + } + if constexpr (!types_is_block_tree) { + for (auto& bv : this->block_tree_types_) { + delete bv; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + int64_t access(size_type index) { + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size_lvl_[0]; + int64_t off = index % block_size_lvl_[0]; + int64_t child; + for (size_type i = 0; static_cast(i) < block_tree_types_.size(); + i++) { + auto& lvl = *block_tree_types_[i]; + auto& lvl_rs = *block_tree_types_rs_[i]; + auto& lvl_ptr = *block_tree_pointers_[i]; + auto& lvl_off = *block_tree_offsets_[i]; + if (lvl[blk_pointer] == 0) { + size_type blk = lvl_rs.rank0(blk_pointer); + off = off + lvl_off[blk]; + blk_pointer = lvl_ptr[blk]; + if (off >= block_size) { + blk_pointer++; + off -= block_size; + } + } + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = lvl_rs.rank1(blk_pointer) * tau_ + child; + } + return decompress_map_[compressed_leaves_[blk_pointer * leaf_size + off]]; + }; + + int64_t select(input_type c, size_type j) { + auto c_index = chars_index_[c]; + auto& top_level = *block_tree_types_[0]; + + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + size_type current_block = (j - 1) / block_size_lvl_[0]; + size_type end_block = c_ranks_[c_index][0].size() - 1; + int64_t block_size = block_size_lvl_[0]; + // find first level block containing the jth occurrence of c with a bin + // search + while (current_block != end_block) { + size_type m = current_block + (end_block - current_block) / 2; + + size_type f = (m == 0) ? 0 : c_ranks_[c_index][0][m - 1]; + if (f < j) { + if (end_block - current_block == 1) { + if (c_ranks_[c_index][0][m] < static_cast(j)) { + current_block = m + 1; + } + break; + } + current_block = m; + } else { + end_block = m - 1; + } + } + + // accumulator + int64_t s = current_block * block_size - 1; + // index that indicates how many c's are still unaccounted for + j -= (current_block == 0) ? 0 : c_ranks_[c_index][0][current_block - 1]; + // we translate unmarked blocks on the top level independently as it differs + // from the other levels + if (!top_level[current_block]) { + int64_t blk = top_level_rs.rank0(current_block); + current_block = top_level_ptr[blk]; + int64_t g = top_level_off[blk]; + int64_t rank_d = (current_block == 0) ? + c_ranks_[c_index][0][0] : + c_ranks_[c_index][0][current_block] - + c_ranks_[c_index][0][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][0][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][0][blk]; + s -= g; + } + } + uint64_t i = 1; + while (i < block_tree_types_.size()) { + auto& current_level = *block_tree_types_[i]; + auto& current_level_rs = *block_tree_types_rs_[i]; + auto& current_level_ptr = *block_tree_pointers_[i]; + auto& current_level_off = *block_tree_offsets_[i]; + auto& prev_level_rs = *block_tree_types_rs_[i - 1]; + current_block = prev_level_rs.rank1(current_block) * tau_; + block_size /= tau_; + int64_t k = current_block; + while ((int64_t)c_ranks_[c_index][i][current_block] < j) { + current_block++; + } + j -= (current_block == k) ? 0 : c_ranks_[c_index][i][current_block - 1]; + s += (current_block - k) * block_size; + if (!current_level[current_block]) { + int64_t blk = current_level_rs.rank0(current_block); + current_block = current_level_ptr[blk]; + int64_t g = current_level_off[blk]; + int64_t rank_d = (current_block % tau_ == 0) ? + c_ranks_[c_index][i][current_block] : + c_ranks_[c_index][i][current_block] - + c_ranks_[c_index][i][current_block - 1]; + rank_d -= pointer_c_ranks_[c_index][i][blk]; + if (rank_d < j) { + j -= rank_d; + s += (block_size - g); + current_block++; + } else { + j += pointer_c_ranks_[c_index][i][blk]; + s -= g; + } + } + i++; + } + + current_block = (*block_tree_types_rs_[i - 1]).rank1(current_block) * tau_; + int64_t l = 0; + while (j > 0) { + if (compressed_leaves_[current_block * leaf_size + l] == compress_map_[c]) + j--; + l++; + } + return s + l; + } + + int64_t rank_base(input_type c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + size_type to = off + top_level_off[blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (to >= block_size) { + int64_t adder = (child == 0) ? + c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + rank += adder; + blk_pointer++; + off = to - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + size_type to = off + ptr_off; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + + if (to >= block_size) { + auto adder = (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + rank += adder; + blk_pointer++; + child = blk_pointer % tau_; + off = to - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + } + + int64_t rank(input_type c, size_type index) { + pasta::BitVector& top_level = *block_tree_types_[0]; + auto& top_level_rs = *block_tree_types_rs_[0]; + auto& top_level_ptr = *block_tree_pointers_[0]; + auto& top_level_off = *block_tree_offsets_[0]; + int64_t c_index = chars_index_[c]; + int64_t block_size = block_size_lvl_[0]; + int64_t blk_pointer = index / block_size; + int64_t off = index % block_size; + int64_t rank = + (blk_pointer == 0) ? 0 : c_ranks_[c_index][0][blk_pointer - 1]; + int64_t child = 0; + if (top_level[blk_pointer]) { + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } else { + size_type blk = top_level_rs.rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][0][blk]; + off = off + top_level_off[blk]; + blk_pointer = top_level_ptr[blk]; + child = blk_pointer; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][0][blk_pointer] : + c_ranks_[c_index][0][blk_pointer] - + c_ranks_[c_index][0][blk_pointer - 1]; + blk_pointer++; + off = off - block_size; + } + block_size = block_size / tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = top_level_rs.rank1(blk_pointer) * tau_ + child; + } + // we first calculate the + uint64_t i = 1; + while (i < block_tree_types_.size()) { + rank += (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + if ((*block_tree_types_[i])[blk_pointer]) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(blk_pointer); + block_size /= tau_; + child = off / block_size; + off = off % block_size; + blk_pointer = rank_blk * tau_ + child; + i++; + } else { + size_type blk = block_tree_types_rs_[i]->rank0(blk_pointer); + rank -= pointer_c_ranks_[c_index][i][blk]; + size_type ptr_off = (*block_tree_offsets_[i])[blk]; + off = off + ptr_off; + blk_pointer = (*block_tree_pointers_[i])[blk]; + child = blk_pointer % tau_; + if (off >= block_size) { + rank += (child == 0) ? c_ranks_[c_index][i][blk_pointer] : + c_ranks_[c_index][i][blk_pointer] - + c_ranks_[c_index][i][blk_pointer - 1]; + blk_pointer++; + child = blk_pointer % tau_; + off = off - block_size; + } + auto remove_prefix = + (child == 0) ? 0 : c_ranks_[c_index][i][blk_pointer - 1]; + rank -= remove_prefix; + } + } + size_type prefix_leaves = blk_pointer - child; + for (int j = 0; j < child * leaf_size; j++) { + if ((compressed_leaves_)[prefix_leaves * leaf_size + j] == + compress_map_[c]) + rank++; + } + for (int j = 0; j <= off; j++) { + if ((compressed_leaves_)[blk_pointer * leaf_size + j] == compress_map_[c]) + rank++; + } + return rank; + }; + + int64_t print_space_usage() { + int64_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + auto delta_size = 0; + if constexpr (recursion_level > 0) { + for (auto bt : block_tree_types_) { + space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); + } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif + } + if constexpr (recursion_level == 0) { + for (auto bv : block_tree_types_) { + space_usage += bv->size() / 8; + delta_size += bv->size() / 8; + } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif + for (auto rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + delta_size += rs->space_usage(); + } +#ifdef BT_DBG + std::cout << "rs size: " << delta_size << std::endl; + delta_size = 0; +#endif + } + + size_t ptr_cnt = 0; + (void)ptr_cnt; + size_t level = 0; + (void)level; + for (const auto iv : block_tree_pointers_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); +#ifdef BT_DBG + ptr_cnt += iv->size(); + std::cout << "level " << level << " ptrs: " << iv->size() + << " block size: " << block_size_lvl_[level] << "\n"; + level++; +#endif + } +#ifdef BT_DBG + std::cout << "ptrs size: " << delta_size << std::endl; + std::cout << "pointer count: " << ptr_cnt << std::endl; + delta_size = 0; +#endif + + for (const auto iv : block_tree_offsets_) { + space_usage += (int64_t)sdsl::size_in_bytes(*iv); + delta_size += (int64_t)sdsl::size_in_bytes(*iv); + } +#ifdef BT_DBG + std::cout << "offs size: " << delta_size << std::endl; + delta_size = 0; +#endif + + if (rank_support) { + for (auto c : chars_) { + int64_t sum = 0; + for (auto lvl : pointer_c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + for (auto lvl : c_ranks_[chars_index_[c]]) { + sum += sdsl::size_in_bytes(lvl); + } + space_usage += sum; + } + } + + for (auto v : block_size_lvl_) { + space_usage += sizeof(v); + } + for (auto v : block_per_lvl_) { + space_usage += sizeof(v); + } + // space_usage += leaves_.size() * sizeof(input_type); + space_usage += sdsl::size_in_bytes(compressed_leaves_); + space_usage += compress_map_.size(); +#ifdef BT_DBG + std::cout << "leaves size: " << sdsl::size_in_bytes(compressed_leaves_) + << std::endl; + delta_size = 0; +#endif + + return space_usage; + }; + + void compress_leaves() { + compress_map_.resize(256, 0); + decompress_map_.resize(256, 0); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compress_map_[this->leaves_[i]] = 1; + } + for (size_t c = 0, cur_val = 0; c < this->compress_map_.size(); ++c) { + size_t tmp = compress_map_[c]; + compress_map_[c] = cur_val; + decompress_map_[cur_val] = c; + cur_val += tmp; + } + + compressed_leaves_.resize(this->leaves_.size()); + for (size_t i = 0; i < this->leaves_.size(); ++i) { + compressed_leaves_[i] = compress_map_[this->leaves_[i]]; + } + sdsl::util::bit_compress(this->compressed_leaves_); + leaves_.resize(0); + leaves_.shrink_to_fit(); + } + + int32_t add_rank_support() { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + int32_t add_rank_support_omp(int32_t threads) { + rank_support = true; + c_ranks_.resize(chars_.size(), std::vector>()); + pointer_c_ranks_.resize(chars_.size(), std::vector>()); + for (uint64_t i = 0; i < c_ranks_.size(); i++) { + c_ranks_[i].resize(block_tree_types_.size(), sdsl::int_vector<0>()); + for (uint64_t j = 0; j < c_ranks_[i].size(); j++) { + c_ranks_[i][j].resize(block_tree_types_[j]->size()); + } + } + for (uint64_t i = 0; i < pointer_c_ranks_.size(); i++) { + pointer_c_ranks_[i].resize(block_tree_pointers_.size(), + sdsl::int_vector<0>()); + for (uint64_t j = 0; j < pointer_c_ranks_[i].size(); j++) { + pointer_c_ranks_[i][j].resize(block_tree_pointers_[j]->size()); + } + } + omp_set_num_threads(threads); + +#pragma omp parallel for default(none) + for (auto c : chars_) { + for (uint64_t i = 0; i < block_tree_types_[0]->size(); i++) { + rank_block(c, 0, i); + } + size_type max = 0; + for (uint64_t i = 1; i < block_tree_types_[0]->size(); i++) { + c_ranks_[chars_index_[c]][0][i] += c_ranks_[chars_index_[c]][0][i - 1]; + if (c_ranks_[chars_index_[c]][0][i] > static_cast(max)) { + max = c_ranks_[chars_index_[c]][0][i]; + } + } + for (uint64_t i = 1; i < block_tree_types_.size(); i++) { + size_type counter = tau_; + size_type acc = 0; + for (uint64_t j = 0; j < block_tree_types_[i]->size(); j++) { + size_type temp = c_ranks_[chars_index_[c]][i][j]; + c_ranks_[chars_index_[c]][i][j] += acc; + acc += temp; + counter--; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (uint64_t i = 0; i < pointer_c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(pointer_c_ranks_[chars_index_[c]][i]); + } + for (uint64_t i = 0; i < c_ranks_[chars_index_[c]].size(); i++) { + sdsl::util::bit_compress(c_ranks_[chars_index_[c]][i]); + } + } + return 0; + } + + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + inline size_type leading_zeros(int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + inline size_type leading_zeros(int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type rank_block(input_type c, size_type i, size_type j) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_block(c, i + 1, rank_blk * tau_ + k); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + for (size_type k = 0; k < tau_; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + size_type rank_g = 0; + rank_c += c_ranks_[chars_index_[c]][i][ptr]; + if (off != 0) { + rank_g = part_rank_block(c, i, ptr, off); + size_type rank_2nd = part_rank_block(c, i, ptr + 1, off); + rank_c -= rank_g; + rank_c += rank_2nd; + } + pointer_c_ranks_[chars_index_[c]][i][rank_0] = rank_g; + } + c_ranks_[chars_index_[c]][i][j] = rank_c; + return rank_c; + } + size_type + part_rank_block(input_type c, size_type i, size_type j, size_type g) { + if (static_cast(j) >= block_tree_types_[i]->size()) { + return 0; + } + size_type rank_c = 0; + if ((*block_tree_types_[i])[j] == 1) { + if (static_cast(i) != block_tree_types_.size() - 1) { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + block_size_lvl_[i + 1] <= g; k++) { + rank_c += c_ranks_[chars_index_[c]][i + 1][rank_blk * tau_ + k]; + k_sum += block_size_lvl_[i + 1]; + } + + if (k_sum != g) { + rank_c += part_rank_block(c, i + 1, rank_blk * tau_ + k, g - k_sum); + } + } else { + size_type rank_blk = block_tree_types_rs_[i]->rank1(j); + size_type k = 0; + size_type k_sum = 0; + for (k = 0; k < tau_ && k_sum + leaf_size <= g; k++) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, leaf_size); + k_sum += leaf_size; + } + + if (k_sum != g) { + rank_c += rank_leaf(c, rank_blk * tau_ + k, g % leaf_size); + } + } + } else { + size_type rank_0 = block_tree_types_rs_[i]->rank0(j); + size_type ptr = (*block_tree_pointers_[i])[rank_0]; + size_type off = (*block_tree_offsets_[i])[rank_0]; + if (g + off >= block_size_lvl_[i]) { + rank_c += c_ranks_[chars_index_[c]][i][ptr] - + pointer_c_ranks_[chars_index_[c]][i][rank_0] + + part_rank_block(c, i, ptr + 1, g + off - block_size_lvl_[i]); + } else { + rank_c += part_rank_block(c, i, ptr, g + off) - + pointer_c_ranks_[chars_index_[c]][i][rank_0]; + } + } + return rank_c; + } + size_type rank_leaf(input_type c, size_type leaf_index, size_type i) { + if (static_cast(leaf_index * leaf_size) >= + compressed_leaves_.size()) { + return 0; + } + // size_type x = leaves_.size() - leaf_index * this->tau_; + // i = std::min(i, x); + size_type result = 0; + for (size_type ind = 0; ind < i; ind++) { + if (compressed_leaves_[leaf_index * leaf_size + ind] == + compress_map_[c]) { + result++; + } + } + return result; + } + + size_type map_unique_chars(const std::vector& text) { + this->u_chars_ = 0; + input_type i = 0; + for (auto a : text) { + if (chars_index_.find(a) == chars_index_.end()) { + chars_index_[a] = i; + i++; + chars_.push_back(a); + } + } + this->u_chars_ = i; + return 0; + }; + size_type + find_next_smallest_index_binary_search(size_type i, + std::vector& pVector) { + int64_t l = 0; + int64_t r = pVector.size(); + while (l < r) { + int64_t m = std::floor((l + r) / 2); + if (i < pVector[m]) { + r = m; + } else { + l = m + 1; + } + } + return r - 1; + }; + int64_t + find_next_smallest_index_linear_scan(size_type i, + std::vector& pVector) { + int64_t b = 0; + while (b < pVector.size() && i >= pVector[b]) { + b++; + } + return b - 1; + }; + size_type find_next_smallest_index_block_tree(size_type index) { + size_type block_size = this->block_size_lvl_[0]; + size_type blk_pointer = index / block_size; + size_type off = index % block_size; + size_type child = 0; + for (size_type i = 0; i < this->block_tree_types_.size(); i++) { + if ((*this->block_tree_types_[i])[blk_pointer] == 0) { + return -1; + } + if (off > 0 && (*this->block_tree_types_[i])[blk_pointer + 1] == 0) { + return -1; + } + size_type rank_blk = this->block_tree_types_rs_[i]->rank1(blk_pointer); + blk_pointer = rank_blk * this->tau_; + block_size /= this->tau_; + child = off / block_size; + off = off % block_size; + blk_pointer += child; + } + return blk_pointer; + }; +}; + +template +using BlockTree = RecursiveBlockTree; +} // namespace pasta +/******************************************************************************/ diff --git a/include/pasta/block_tree/rec_dense_bit_block_tree.hpp b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp new file mode 100644 index 0000000..d1afbeb --- /dev/null +++ b/include/pasta/block_tree/rec_dense_bit_block_tree.hpp @@ -0,0 +1,783 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +template +class RecursiveDenseBitBlockTree { +public: + constexpr static bool types_is_block_tree = recursion_level > 0; + using IsInternalType = std::conditional_t< + types_is_block_tree, + RecursiveDenseBitBlockTree, + pasta::BitVector>; + using IsInternalRankType = std::conditional_t< + types_is_block_tree, + RecursiveDenseBitBlockTree, + pasta::RankSelect>; + + /// If this is true, then the only levels of the tree start to be + /// included starting at the first level that contains a back block + /// + /// For example, if levels 0 to 5 do not contain any back blocks, then the + /// tree will only contain levels 6 and below. + bool CUT_FIRST_LEVELS = true; + + /// The arity of the tree + size_type tau_; + size_type max_leaf_length_; + /// The arity of the tree's root + size_type s_ = 1; + size_type leaf_size = 0; + size_type amount_of_leaves = 0; + size_type num_bits_; + bool rank_support = false; + /// Recursively compress the bit vectors of the tree + std::vector block_tree_types_; + std::vector block_tree_types_rs_; + /// For each level and each back block, contains the index of the + /// block's source + std::vector*> block_tree_pointers_; + std::vector*> block_tree_offsets_; + // std::vector*> block_tree_encoded_; + std::vector block_size_lvl_; + std::vector leaves_; + + std::unique_ptr leaf_bits_; + + /// @brief For each level and each block, contains the number of 1s up to (and + /// including) the block. + std::vector> one_ranks_; + /// @brief For each level and each back block, + /// contains the number of 1s up to (and including) the pointed-to area of + /// the back-block. + std::vector> pointer_prefix_one_counts_; + + ~RecursiveDenseBitBlockTree() { + for (const IsInternalType* b : this->block_tree_types_) { + delete b; + } + // in any other case, block_tree_types_ and block_tree_types_rs_ point to + // the same object (a recursive block tree), so we may only free them once + if constexpr (recursion_level == 0) { + for (const RankSelect* rs : + this->block_tree_types_rs_) { + delete rs; + } + } + for (auto& ptrs : this->block_tree_pointers_) { + delete ptrs; + } + for (auto& offsets : this->block_tree_offsets_) { + delete offsets; + } + } + + [[nodiscard]] size_t height() const { + return block_tree_types_.size(); + } + + [[nodiscard]] size_t size() const { + return num_bits_; + } + + bool operator[](const size_type bit_index) const { + return access(bit_index); + } + + bool access(const size_type bit_index) const { + int64_t block_size = block_size_lvl_[0]; + int64_t block_index = bit_index / block_size; + int64_t off = bit_index % block_size; + for (size_t i = 0; i < height(); i++) { + const auto& is_internal = *block_tree_types_[i]; + const auto& is_internal_rank = *block_tree_types_rs_[i]; + const auto& pointers = *block_tree_pointers_[i]; + const auto& offsets = *block_tree_offsets_[i]; + if (!is_internal[block_index]) { + // If this block is not internal, go to its pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + off = off + offsets[back_block_index]; + block_index = pointers[back_block_index]; + if (off >= block_size) { + ++block_index; + off -= block_size; + } + } + block_size /= tau_; + const int64_t child = off / block_size; + off %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + } + + return (*leaf_bits_)[block_index * leaf_size + off]; + }; + +private: + template + [[nodiscard]] size_t find_initial_block(const size_t rank) const { + const auto& top_one_ranks = one_ranks_[0]; + const size_t block_size = block_size_lvl_[0]; + size_t start = (rank - 1) / block_size; + size_t end = top_one_ranks.size() - 1; + while (start != end) { + const size_t middle = start + (end - start) / 2; + size_t current_rank; + if constexpr (one) { + current_rank = (middle == 0) ? 0 : top_one_ranks[middle - 1]; + } else { + const size_t middle_bits = middle * block_size; + current_rank = + (middle == 0) ? 0 : middle_bits - top_one_ranks[middle - 1]; + } + if (current_rank < rank) { + if (start + 1 == end) { + size_t bits; + if constexpr (one) { + bits = top_one_ranks[middle]; + } else { + bits = (middle + 1) * block_size - top_one_ranks[middle]; + } + // If there is only one block left, it's either the current or the + // next block + if (bits < rank) { + start = middle + 1; + } + break; + } + start = middle; + } else { + end = middle - 1; + } + } + return start; + } + +public: + /// FIXME DOES NOT WORK YET + [[nodiscard("select result discarded")]] size_t select1(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + size_t block_size = block_size_lvl_[0]; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + + size_t pos = current_block * block_size - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_one_ranks[current_block - 1]; + + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + size_t rank_d = + (current_block == 0) ? + top_one_ranks[current_block] : + top_one_ranks[current_block] - top_one_ranks[current_block - 1]; + rank_d -= pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += block_size - offset; + ++current_block; + } else { + rank += pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset; + } + } + + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + const size_t start_block = current_block; + while (one_ranks[current_block] < rank) { + ++current_block; + } + rank -= (current_block == start_block) ? 0 : one_ranks[current_block - 1]; + pos += (current_block - start_block) * block_size; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + size_t rank_d = + (current_block % tau_ == 0) ? + one_ranks[current_block] : + one_ranks[current_block] - one_ranks[current_block - 1]; + rank_d -= pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += (block_size - offset); + ++current_block; + } else { + rank += pointer_ranks[back_block_index]; + pos -= offset; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + for (size_t bit = 0; rank > 0; ++bit, ++pos) { + rank -= (*leaf_bits_)[current_block * leaf_size + bit]; + } + return pos; + } + + [[nodiscard("select result discarded")]] size_t select0(size_t rank) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + const auto& top_one_ranks = one_ranks_[0]; + + const size_t top_block_size = block_size_lvl_[0]; + const auto top_zero_ranks = [&top_one_ranks, + top_block_size](const size_t i) -> size_t { + return (i + 1) * top_block_size - top_one_ranks[i]; + }; + + // Binary Search for the correct top level block containing the correct 1 + size_t current_block = find_initial_block(rank); + const size_t top_block_bits = top_block_size; + + size_t pos = (current_block * top_block_bits) - 1; + // ReSharper disable once CppDFAUnreachableCode + rank -= (current_block == 0) ? 0 : top_zero_ranks(current_block - 1); + // If that block is a back block, we need to move to the back-pointed block + if (!top_is_internal[current_block]) { + const size_t back_block_index = top_is_internal_rank.rank0(current_block); + // const size_t child_block_bits = + // height() == 1 ? leaf_size * 8 : block_size_lvl_[1] * 8; + current_block = top_pointers[back_block_index]; + const size_t offset = top_offsets[back_block_index]; + const size_t prefix_bits = offset; + size_t rank_d = + (current_block == 0) ? + top_zero_ranks(current_block) : + top_zero_ranks(current_block) - top_zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += top_block_size - offset; + ++current_block; + } else { + rank += prefix_bits - pointer_prefix_one_counts_[0][back_block_index]; + pos -= offset; + } + } + + size_t block_size = block_size_lvl_[0]; + size_t level = 1; + while (level < height()) { + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + const auto& prev_is_internal_rank = *block_tree_types_rs_[level - 1]; + const auto& offsets = *block_tree_offsets_[level]; + const auto& pointers = *block_tree_pointers_[level]; + const auto& one_ranks = one_ranks_[level]; + + current_block = prev_is_internal_rank.rank1(current_block) * tau_; + block_size /= tau_; + + const auto zero_ranks = + [&one_ranks, this, block_size](const size_t i) -> size_t { + const size_t rnk = (i % this->tau_ + 1) * block_size - one_ranks[i]; + return rnk; + }; + const size_t start_block = current_block; + while (zero_ranks(current_block) < rank) { + ++current_block; + } + rank -= + (current_block == start_block) ? 0 : zero_ranks(current_block - 1); + pos += (current_block - start_block) * block_size; + if (!is_internal[current_block]) { + size_t back_block_index = is_internal_rank.rank0(current_block); + current_block = pointers[back_block_index]; + const size_t offset = offsets[back_block_index]; + const size_t prefix_bits = offset; + size_t rank_d = + (current_block % tau_ == 0) ? + zero_ranks(current_block) : + zero_ranks(current_block) - zero_ranks(current_block - 1); + rank_d -= prefix_bits - pointer_ranks[back_block_index]; + if (rank > rank_d) { + rank -= rank_d; + pos += block_size - offset; + ++current_block; + } else { + rank += prefix_bits - pointer_ranks[back_block_index]; + pos -= offset; + } + } + ++level; + } + + current_block = + block_tree_types_rs_[level - 1]->rank1(current_block) * tau_; + for (size_t bit = 0; rank > 0; ++bit, ++pos) { + rank -= !(*leaf_bits_)[current_block * leaf_size + bit]; + } + return pos; + } + + /// @brief Counts the number of 1-bits up to (and excluding) an index. + [[nodiscard("rank result discarded")]] size_t + rank1(const size_type bit_index) const { + const auto& top_is_internal = *block_tree_types_[0]; + const auto& top_is_internal_rank = *block_tree_types_rs_[0]; + const auto& top_pointers = *block_tree_pointers_[0]; + const auto& top_offsets = *block_tree_offsets_[0]; + size_t block_size = block_size_lvl_[0]; + size_t block_index = bit_index / block_size; + size_t block_offset = bit_index % block_size; + size_t rank = (block_index == 0) ? 0 : one_ranks_[0][block_index - 1]; + if (!top_is_internal[block_index]) { + // If the top block is a back block, go to it and adjust the offset + const size_t back_block_index = top_is_internal_rank.rank0(block_index); + rank -= pointer_prefix_one_counts_[0][back_block_index]; + block_offset += top_offsets[back_block_index]; + block_index = top_pointers[back_block_index]; + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it + rank += + (block_index == 0) ? + one_ranks_[0][block_index] : + (one_ranks_[0][block_index] - one_ranks_[0][block_index - 1]); + ++block_index; + block_offset -= block_size; + } + } + + // Go down to the next level + block_size /= tau_; + // How many children are we 'skipping over' + size_t child = block_offset / block_size; + block_offset %= block_size; + block_index = top_is_internal_rank.rank1(block_index) * tau_ + child; + + size_t level = 1; + while (level < height()) { + const auto& ranks = one_ranks_[level]; + const auto& pointer_ranks = pointer_prefix_one_counts_[level]; + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + rank += (child == 0) ? 0 : ranks[block_index - 1]; + // If this block is internal, just go to the correct child + if (is_internal[block_index]) { + block_size /= tau_; + child = block_offset / block_size; + block_offset %= block_size; + block_index = is_internal_rank.rank1(block_index) * tau_ + child; + level++; + continue; + } + + // If we have a back block, we need to go to the pointed-to block + const size_t back_block_index = is_internal_rank.rank0(block_index); + rank -= pointer_ranks[back_block_index]; + block_offset += (*block_tree_offsets_[level])[back_block_index]; + block_index = (*block_tree_pointers_[level])[back_block_index]; + child = block_index % tau_; + + if (block_offset >= block_size) { + // If we're exceeding the pointed-to block's offset, + // add the ones inside of it and go to the next block + rank += (child == 0) ? ranks[block_index] : + (ranks[block_index] - ranks[block_index - 1]); + ++block_index; + child = block_index % tau_; + block_offset -= block_size; + } + const size_t remove_prefix = (child == 0) ? 0 : ranks[block_index - 1]; + rank -= remove_prefix; + } + + // Number of leaves that exist before the leaves of the current block + const size_type prefix_leaves = block_index - child; + for (size_t block = 0; block < child * leaf_size; block++) { + rank += (*leaf_bits_)[prefix_leaves * leaf_size + block]; + } + for (size_t block = 0; block < block_offset; block++) { + rank += (*leaf_bits_)[block_index * leaf_size + block]; + } + return rank; + } + + /// @brief Counts the number of 0-bits up to (and excluding) an index. + size_t rank0(const size_type bit_index) const { + return bit_index - rank1(bit_index); + } + + [[nodiscard]] size_t print_space_usage() const { + size_t space_usage = sizeof(tau_) + sizeof(max_leaf_length_) + sizeof(s_) + + sizeof(leaf_size); + + size_t delta_size = 0; + for (const auto* bt : block_tree_types_) { + if constexpr (types_is_block_tree) { + space_usage += bt->print_space_usage(); + delta_size += bt->print_space_usage(); + } else { + space_usage += bt->size() / 8; + delta_size += bt->size() / 8; + } + } +#ifdef BT_DBG + std::cout << "bv size: " << delta_size << std::endl; + delta_size = 0; +#endif + if constexpr (recursion_level == 0) { + for (const auto* rs : block_tree_types_rs_) { + space_usage += rs->space_usage(); + delta_size += rs->space_usage(); + } +#ifdef BT_DBG + std::cout << "rs size: " << delta_size << std::endl; +#endif + } + delta_size = 0; + for (const auto iv : block_tree_pointers_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); + ; + } + size_t ptr_cnt = 0; + (void)ptr_cnt; +#ifdef BT_DBG + std::cout << "ptrs size: " << delta_size << std::endl; + delta_size = 0; + size_t level = 0; +#endif + for (const auto iv : block_tree_offsets_) { + space_usage += sdsl::size_in_bytes(*iv); + delta_size += sdsl::size_in_bytes(*iv); +#ifdef BT_DBG + ptr_cnt += iv->size(); + std::cout << "level " << level << " ptrs: " << iv->size() + << " block size: " << block_size_lvl_[level] << "\n"; + level++; +#endif + } +#ifdef BT_DBG + std::cout << "offs size: " << delta_size << std::endl; + std::cout << "pounter count: " << ptr_cnt << std::endl; +#endif + space_usage += block_size_lvl_.size() * + sizeof(typename decltype(block_size_lvl_)::value_type); + + if (rank_support) { + for (auto& rs : one_ranks_) { + space_usage += sdsl::size_in_bytes(rs); + } + for (auto& rs : pointer_prefix_one_counts_) { + space_usage += sdsl::size_in_bytes(rs); + } + } + + space_usage += leaf_bits_->size() / 8; +#ifdef BT_DBG + std::cout << "leaf string: " << leaf_bits_->size() / 8 << std::endl; +#endif + + return space_usage; + }; + + void + add_bit_rank_support(size_t threads = std::thread::hardware_concurrency()) { + if (rank_support) { + return; + } + rank_support = true; + + // FIXME For the last level where block_tree_types_ is a bitvec, using + // multiple threads doesn't work for some reason + if constexpr (recursion_level == 0) { + threads = 1; + } + + // Resize rank information vectors + one_ranks_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + one_ranks_[level].resize(block_tree_types_[level]->size()); + } + pointer_prefix_one_counts_.resize(height(), sdsl::int_vector<0>()); + for (uint64_t level = 0; level < height(); level++) { + pointer_prefix_one_counts_[level].resize( + block_tree_pointers_[level]->size()); + } + + // FIXME: breaks if parallelism is used + // #pragma omp parallel for default(none) num_threads(threads) + for (size_t block = 0; block < block_tree_types_[0]->size(); block++) { + bit_rank_block(0, block); + } + + for (size_t block = 1; block < block_tree_types_[0]->size(); block++) { + one_ranks_[0][block] += one_ranks_[0][block - 1]; + } + +#pragma omp parallel for default(none) num_threads(threads) + for (size_t level = 1; level < height(); level++) { + size_type counter = tau_; + size_t acc = 0; + for (size_t block = 0; block < one_ranks_[level].size(); block++) { + const size_type ones_in_block = one_ranks_[level][block]; + acc += ones_in_block; + one_ranks_[level][block] = acc; + --counter; + if (counter == 0) { + acc = 0; + counter = tau_; + } + } + } + for (auto& prefix_one_counts : pointer_prefix_one_counts_) { + sdsl::util::bit_compress(prefix_one_counts); + } + for (auto& ranks : one_ranks_) { + sdsl::util::bit_compress(ranks); + } + } + +protected: + /// @brief Calculate the number of leading zeros for a 32-bit integer. + /// This value is capped at 31. + static size_type leading_zeros(const int32_t val) { + return __builtin_clz(static_cast(val) | 1); + } + + /// @brief Calculate the number of leading zeros for a 64-bit integer. + /// This value is capped at 64. + static size_type leading_zeros(const int64_t val) { + return __builtin_clzll(static_cast(val) | 1); + } + + /// + /// @brief Determine the padding and minimum height and the size of the blocks + /// on the top level of a block tree with s top-level blocks and an arity of + /// tau with leaves also of size tau. + /// + /// The height is the number of levels in the tree. + /// The padding is the number of characters that the top-level exceeds the + /// text length. For example, if the result was that the top level consists of + /// s = 5 blocks of size 30 and the text size being 80, then the padding would + /// be (5 * 30) - 80 = 70. + /// + /// @param[out] padding The number of characters in the last block (of the + /// first level of the tree) that are empty. + /// @param[in] text_length The number of characters in the input string. + /// @param[out] height The number of levels in the tree. + /// @param[out] blk_size The size of blocks on the first level of the tree. + /// + void calculate_padding(int64_t& padding, + int64_t text_length, + int64_t& height, + int64_t& blk_size) { + // This is the number of characters occupied by a tree with s*tau^h levels + // and leaves of size tau. At the start, we only have a tree with the first + // level with s leaf blocks which each have size tau. If we insert another + // level, the number of leaf blocks (and therefore the number of occupied + // characters) increases by a factor of tau. + int64_t tmp_padding = this->s_ * this->tau_; + int64_t h = 1; + // Size of the blocks on the current level (starting at the leaf level) + blk_size = tau_; + // While the tree does not cover the entire text, add a level + while (tmp_padding < text_length) { + tmp_padding *= this->tau_; + blk_size *= this->tau_; + h++; + } + // once the tree has enough levels to cover the entire text, we set the + // tree's values + height = h; + // The padding is the number of excess characters that the block tree covers + // over the length of the text. + padding = tmp_padding - text_length; + } + + size_type bit_rank_block(size_type level, size_type block_index) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + if (static_cast(level) < height() - 1) { + // If we are not on the last level recursively call + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_block(level + 1, internal_index * tau_ + k); + } + } else { + // If we are on the last level + for (size_type k = 0; k < tau_; ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + size_type num_ones_parts = 0; + num_ones += one_ranks_[level][ptr]; + if (off > 0) { + num_ones_parts = part_bit_rank_block(level, ptr, off); + const size_type num_ones_2nd_part = + part_bit_rank_block(level, ptr + 1, off); + num_ones -= num_ones_parts; + num_ones += num_ones_2nd_part; + } + pointer_prefix_one_counts_[level][back_block_index] = num_ones_parts; + } + one_ranks_[level][block_index] = num_ones; + return num_ones; + } + + size_type part_bit_rank_block(const size_type level, + const size_type block_index, + const size_type chars_to_process) { + const auto& is_internal = *block_tree_types_[level]; + const auto& is_internal_rank = *block_tree_types_rs_[level]; + if (static_cast(block_index) >= is_internal.size()) { + return 0; + } + + size_type num_ones = 0; + if (is_internal[block_index]) { + const size_type internal_index = is_internal_rank.rank1(block_index); + size_type k = 0; + size_type processed_chars = 0; + if (static_cast(level) < height() - 1) { + const size_type child_size = block_size_lvl_[level + 1]; + // We're not on the last level + // iterate over the children as long as we don't exceed the limit + for (k = 0; + k < tau_ && processed_chars + child_size <= chars_to_process; + ++k) { + num_ones += one_ranks_[level + 1][internal_index * tau_ + k]; + processed_chars += child_size; + } + + // If we still need to process more chars and they end inside the next + // child, rank that part of the next child + if (processed_chars != chars_to_process) { + num_ones += part_bit_rank_block(level + 1, + internal_index * tau_ + k, + chars_to_process - processed_chars); + } + } else { + // We're on the last level + for (k = 0; k < tau_ && processed_chars + leaf_size <= chars_to_process; + ++k) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, leaf_size); + processed_chars += leaf_size; + } + + if (processed_chars != chars_to_process) { + num_ones += bit_rank_leaf(internal_index * tau_ + k, + chars_to_process % leaf_size); + } + } + } else { + const size_type back_block_index = is_internal_rank.rank0(block_index); + const size_type ptr = (*block_tree_pointers_[level])[back_block_index]; + const size_type off = (*block_tree_offsets_[level])[back_block_index]; + + // If we need to process chars beyond this block, we need to + if (chars_to_process + off >= block_size_lvl_[level]) { + // Ones in the entire block this block points to + num_ones += one_ranks_[level][ptr]; + // Ones that overflow into the next block + num_ones += part_bit_rank_block(level, + ptr + 1, + chars_to_process + off - + block_size_lvl_[level]); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } else { + // Number of ones up to the cutoff point + num_ones += part_bit_rank_block(level, ptr, chars_to_process + off); + // Num ones in the pointed-to block *before* the pointed-to area + num_ones -= pointer_prefix_one_counts_[level][back_block_index]; + } + } + return num_ones; + } + + /// + /// @brief Count ones in leaf block. + /// + /// @param leaf_index The index of the leaf block. + /// @param max_char_index The maximum character index (exclusive) to + /// consider. This is used for when this block is at the end of the string. + /// @return The number of ones in this block. + /// + size_type bit_rank_leaf(size_type leaf_index, size_type max_char_index) { + if (static_cast(leaf_index * leaf_size) >= leaf_bits_->size()) { + return 0; + } + + size_type result = 0; + for (size_type i = 0; i < max_char_index; ++i) { + result += (*leaf_bits_)[leaf_index * leaf_size + i]; + } + return result; + } +}; + +template +using DenseBitBlockTree = RecursiveDenseBitBlockTree; + +} // namespace pasta + +/******************************************************************************/ diff --git a/include/pasta/block_tree/utils/MersenneHash.hpp b/include/pasta/block_tree/utils/MersenneHash.hpp index f8fbb14..4ae7c6e 100644 --- a/include/pasta/block_tree/utils/MersenneHash.hpp +++ b/include/pasta/block_tree/utils/MersenneHash.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -21,42 +22,193 @@ #pragma once #include -#include +#include +#include +#include +#include +#include #include +#ifdef BT_INSTRUMENT +# include +#endif + namespace pasta { -template class MersenneHash { +#ifdef BT_INSTRUMENT +static std::atomic_size_t mersenne_hash_comparisons = 0; +static std::atomic_size_t mersenne_hash_equals = 0; +static std::atomic_size_t mersenne_hash_collisions = 0; + +void print_hash_data() { + std::cerr << "comparisons: " << mersenne_hash_comparisons + << ", equals: " << mersenne_hash_equals + << ", collisions: " << mersenne_hash_collisions + << ", percent equals: " + << 100 * mersenne_hash_equals / ((double)mersenne_hash_comparisons) + << ", percent collisions: " + << 100 * mersenne_hash_collisions / + ((double)mersenne_hash_comparisons) + << std::endl; +} +#endif + +template +class MersenneHash { public: - std::vector const &text_; - uint64_t hash_; + __extension__ typedef unsigned __int128 uint128_t; + std::span text_; + uint128_t hash_; uint32_t start_; uint32_t length_; - MersenneHash(std::vector const &text, size_t hash, uint64_t start, - uint64_t length) - : text_(text), hash_(hash), start_(start), length_(length){}; - bool operator==(const MersenneHash &other) const { - // std::cout << start_ << " " << other.start_ << std::endl; - if (length_ != other.length_) + MersenneHash(std::vector const& text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : text_(text), + hash_(hash), + start_(start), + length_(length){}; + + MersenneHash(const std::span text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : text_(text), + hash_(hash), + start_(start), + length_(length){}; + + constexpr MersenneHash() : text_(), hash_(0), start_(0), length_(0){}; + + constexpr MersenneHash(const MersenneHash& other) = default; + constexpr MersenneHash(MersenneHash&& other) = default; + + MersenneHash& operator=(const MersenneHash& other) = default; + MersenneHash& operator=(MersenneHash&& other) = default; + + bool operator==(const MersenneHash& other) const { +#ifdef BT_INSTRUMENT + ++mersenne_hash_comparisons; +#endif + // if (length_ != other.length_) + // return false; + // std::cout << static_cast(hash_) << ", " + // << static_cast(other.hash_) << std::endl; + if (hash_ != other.hash_) return false; - for (uint64_t i = 0; i < length_; i++) { - if (text_[start_ + i] != other.text_[other.start_ + i]) { - return false; + const bool is_same = memcmp(text_.data() + start_, + other.text_.data() + other.start_, + length_) == 0; + +#ifdef BT_INSTRUMENT + if (!is_same) { + // The hash is the same but the substring isn't => collision + ++mersenne_hash_collisions; + } else { + // The substrings are the same + ++mersenne_hash_equals; + } +#endif + return is_same; + }; +}; + +template <> +class MersenneHash { +public: + __extension__ typedef unsigned __int128 uint128_t; + /// @brief The whole string in which the substring lies + const pasta::BitVector* text_; + uint128_t hash_; + /// @brief The bit start-position of the hashed substring + uint64_t start_; + /// @brief The number of bits in the hashed substring + uint64_t length_; + + MersenneHash(const pasta::BitVector& text, + const uint128_t hash, + const uint64_t start, + const uint64_t length) + : text_{&text}, + hash_{hash}, + start_{start}, + length_{length} {}; + + constexpr MersenneHash() : text_(nullptr), hash_(0), start_(0), length_(0){}; + + constexpr MersenneHash(const MersenneHash& other) = default; + constexpr MersenneHash(MersenneHash&& other) = default; + + constexpr MersenneHash& operator=(const MersenneHash& other) = default; + constexpr MersenneHash& operator=(MersenneHash&& other) = default; + + inline bool operator==(const MersenneHash& other) const { +#ifdef BT_INSTRUMENT + ++mersenne_hash_comparisons; +#endif + if (hash_ != other.hash_) { + //std::cout << "hashes unequal" << std::endl; + return false; + } + + size_t pos = 0; + bool is_same = true; + for (size_t remaining = length_; remaining > 64; + pos += 64, remaining -= 64) { + if (slice_at(*text_, start_ + pos) != + slice_at(*other.text_, other.start_ + pos)) { + is_same = false; + break; + } + } + + if (!is_same) { + for (size_t i = pos; i < length_; ++i) { + if ((*text_)[start_ + i] != (*other.text_)[other.start_ + i]) { + is_same = false; + break; + } } } - return true; + +#ifdef BT_INSTRUMENT + if (!is_same) { + // The hash is the same but the substring isn't => collision + ++mersenne_hash_collisions; + } else { + // The substrings are the same + ++mersenne_hash_equals; + } +#endif + return is_same; + }; + + static uint64_t slice_at(const pasta::BitVector& bv, const size_t i) { + const std::span backing = bv.data(); + const uint8_t offset = i % 64; + const size_t data_index = i / 64; + // TODO Check if the right shift actually shifts in zeros + const uint64_t r = + backing[data_index] & (~static_cast(0) << offset); + if (offset > 0) { + const uint64_t l = backing[data_index + 1] & + (~static_cast(0) >> (64 - offset)); + return (l << (64 - offset)) | (r >> offset); + } + return r; } }; } // namespace pasta -namespace std { -template struct hash> { - std::size_t operator()(const pasta::MersenneHash &hS) const { +template +struct std::hash> { + typename pasta::MersenneHash::uint128_t + operator()(const pasta::MersenneHash& hS) const { return hS.hash_; } -}; -} // namespace std +}; // namespace std /******************************************************************************/ diff --git a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp index 4d5f9c4..1d3a5bf 100644 --- a/include/pasta/block_tree/utils/MersenneRabinKarp.hpp +++ b/include/pasta/block_tree/utils/MersenneRabinKarp.hpp @@ -2,6 +2,7 @@ * This file is part of pasta::block_tree * * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -20,66 +21,119 @@ #pragma once -#include +#include "pasta/block_tree/utils/MersenneHash.hpp" namespace pasta { -template class MersenneRabinKarp { - __extension__ typedef unsigned __int128 uint128_t; +__extension__ typedef unsigned __int128 uint128_t; +template +static consteval uint128_t mersenne_prime() { + uint128_t res = 1; + for (size_t i = 0; i < exponent; i++) { + res <<= 1; + } + return res - 1; +} + +/// +/// @brief A Rabin-Karp rolling hasher. +/// +/// @tparam T The type of the characters in the text. +/// @tparam size_type The type to use for indexing etc. +/// @tparam mersenne_exponent If using a mersenne prime 2^p-1, then this should +/// be p. If this is 0, a normal modulus operation will be used. +/// Additionally, if this is != 0, then the prime_ attribute will be ignored +/// and '(1 << mersenne_exponent) - 1' will be used instead. +/// +template +class MersenneRabinKarp { public: - std::vector const &text_; + /// The text being hashed + std::span text_; uint128_t sigma_; + /// The start index of the currently hashed window uint64_t init_; + /// The window size of this hasher uint64_t length_; + /// A large prime used for modulus operations uint128_t prime_; - uint64_t hash_; + /// The current hash value + uint128_t hash_; uint128_t max_sigma_; - MersenneRabinKarp(std::vector const &text, uint64_t sigma, uint64_t init, - uint64_t length, uint128_t prime) - : text_(text), sigma_(sigma), init_(init), length_(length), + /// @brief Construct a new Rabin Karp hasher. + /// @param text The text to hash. (not just the window but the entire text) + /// @param sigma The alphabet size. + /// @param init The start index of the first hashed window in the text. + /// @param length The window size. + /// @param prime A large prime used for modulus operations + /// iff not using mersenne_exponent. + constexpr MersenneRabinKarp(const std::span text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : text_(text), + sigma_(sigma), + init_(init), + length_(length), prime_(prime) { max_sigma_ = 1; uint128_t fp = 0; uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { - fp = mersenneModulo(fp * sigma); + fp = fp * sigma; fp = mersenneModulo(fp + text_[i]); } for (uint64_t i = 0; i < length_ - 1; i++) { sigma_c = mersenneModulo(sigma_c * sigma_); } - hash_ = (uint64_t)(fp); - max_sigma_ = (uint64_t)(sigma_c); + hash_ = fp; + max_sigma_ = sigma_c; }; - void restart(uint64_t index) { + constexpr MersenneRabinKarp(const std::vector& text, + const uint64_t sigma, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : MersenneRabinKarp(std::span(text), sigma, init, length, prime) {} + + /// @brief Moves the hasher to the specified start index in the backing + /// vector. + void restart(const uint64_t index) { if (index + length_ >= text_.size()) { return; } init_ = index; - max_sigma_ = 1; uint128_t fp = 0; - uint128_t sigma_c = 1; for (uint64_t i = init_; i < init_ + length_; i++) { - fp = mersenneModulo(fp * sigma_); + fp = fp * sigma_; fp = mersenneModulo(fp + text_[i]); } - for (uint64_t i = 0; i < length_ - 1; i++) { - sigma_c = mersenneModulo(sigma_c * sigma_); - } - hash_ = (uint64_t)(fp); - max_sigma_ = (uint64_t)(sigma_c); + hash_ = fp; }; - inline uint128_t mersenneModulo(uint128_t k) { - return k % prime_; - // uint128_t i = (k & prime_) + (k >> power_); - // return (i >= prime_) ? i - prime_ : i; + constexpr uint128_t mersenneModulo(uint128_t k) const { + if constexpr (mersenne_exponent == 0) { + return k % prime_; + } else { + constexpr uint128_t MERSENNE = mersenne_prime(); + uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); + i -= (i >= MERSENNE) * MERSENNE; + return i; + } }; - void next() { + /// @brief Retrieves the hash value at the hasher's current position. + /// @return A MersenneHash object representing the current hash value. + constexpr MersenneHash current_hash() const { + return MersenneHash(text_, hash_, init_, length_); + } + + /// @brief Advances the hasher by one character. + constexpr void next() { if (text_.size() <= init_ + length_) { return; } @@ -87,21 +141,144 @@ template class MersenneRabinKarp { uint128_t fp = hash_; T out_char = text_[init_]; T in_char = text_[init_ + length_]; - uint128_t out_char_influence = out_char * max_sigma_; - out_char_influence = mersenneModulo(out_char_influence); - if (out_char_influence < hash_) { - fp -= out_char_influence; + const uint128_t out_char_influence = mersenneModulo(out_char * max_sigma_); + // Conditionally add the prime, of the out_char_influence is too large + if constexpr (mersenne_exponent == 0) { + fp += prime_ * (out_char_influence > hash_) - out_char_influence; } else { - fp = prime_ - (out_char_influence - fp); + fp += mersenne_prime() * (out_char_influence > hash_) - + out_char_influence; } fp *= sigma_; fp += in_char; fp = mersenneModulo(fp); - hash_ = (uint64_t)(fp); + hash_ = fp; init_++; }; }; +/// +/// @brief A Rabin-Karp rolling hasher for bitstrings. +/// +/// @tparam size_type The type to use for indexing etc. +/// @tparam mersenne_exponent If using a mersenne prime 2^p-1, then this should +/// be p. If this is 0, a normal modulus operation will be used. +/// Additionally, if this is != 0, then the prime_ attribute will be ignored +/// and '(1 << mersenne_exponent) - 1' will be used instead. +/// +template +class MersenneRabinKarp { +public: + /// The text being hashed + const pasta::BitVector& text_; + uint64_t init_; + /// The window size of this hasher + uint64_t length_; + /// A large prime used for modulus operations + uint128_t prime_; + /// The current hash value + uint128_t hash_; + uint128_t max_sigma_; + + /// @brief Construct a new Rabin Karp hasher over a bitstring. + /// @param text The bitvector to hash. (not just the window but the entire + /// text) + /// @param init The start bit-index of the first hashed window in the text. + /// @param length The window size in bits. + /// @param prime A large prime used for modulus operations + /// iff not using mersenne_exponent. + constexpr MersenneRabinKarp(const pasta::BitVector& text, + const uint64_t init, + const uint64_t length, + const uint128_t prime) + : text_(text), + init_(init), + length_(length), + prime_(prime) { + max_sigma_ = 1; + uint128_t fp = 0; + uint128_t sigma_c = 1; + for (uint64_t i = init_; i < init_ + length_; i++) { + fp = mersenneModulo(2 * fp + get_bit(i)); + } + for (uint64_t i = 0; i < length_ - 1; i++) { + sigma_c = mersenneModulo(2 * sigma_c); + } + hash_ = fp; + max_sigma_ = sigma_c; + } + + /// @brief Moves the hasher to the specified start index in the backing + /// vector. + constexpr void restart(const uint64_t index) { + if (index + length_ >= text_.size()) { + return; + } + init_ = index; + uint128_t fp = 0; + for (uint64_t i = init_; i < init_ + length_; i++) { + fp = fp * 2; + fp = mersenneModulo(fp + get_bit(i)); + } + hash_ = fp; + }; + + [[nodiscard]] constexpr uint128_t mersenneModulo(const uint128_t k) const { + if constexpr (mersenne_exponent == 0) { + return k % prime_; + } else { + constexpr uint128_t MERSENNE = mersenne_prime(); + uint128_t i = (k & MERSENNE) + (k >> mersenne_exponent); + i -= (i >= MERSENNE) * MERSENNE; + return i; + } + }; + + /// @brief Retrieves the hash value at the hasher's current position. + /// @return A MersenneHash object representing the current hash value. + [[nodiscard]] constexpr MersenneHash current_hash() const { + return {text_, hash_, init_, length_}; + } + + /// @brief Advances the hasher by one character. + constexpr void next() { + if (text_.size() <= init_ + length_) { + return; + } + + uint128_t fp = hash_; + const bool out_char = out_bit(); + const bool in_char = in_bit(); + const uint128_t out_char_influence = out_char * max_sigma_; + // Conditionally add the prime, of the out_char_influence is too large + if constexpr (mersenne_exponent == 0) { + fp += prime_ * (out_char_influence > hash_) - out_char_influence; + } else { + fp += mersenne_prime() * (out_char_influence > hash_) - + out_char_influence; + } + fp *= 2; + fp += in_char; + fp = mersenneModulo(fp); + hash_ = fp; + init_++; + }; + +private: + [[nodiscard]] constexpr bool out_bit() const { + return get_bit(init_); + } + + [[nodiscard]] constexpr bool in_bit() const { + const size_t idx = init_ + length_; + return get_bit(idx); + } + + [[nodiscard]] constexpr bool get_bit(const size_t bit_index) const { + return text_[bit_index]; + } +}; + } // namespace pasta /******************************************************************************/ diff --git a/include/pasta/block_tree/utils/byteread.hpp b/include/pasta/block_tree/utils/byteread.hpp new file mode 100644 index 0000000..52da16e --- /dev/null +++ b/include/pasta/block_tree/utils/byteread.hpp @@ -0,0 +1,57 @@ +#pragma once + +#include "concepts.hpp" + +namespace pasta { + +/// @brief Swaps the bytes in an integer. +template +Int byteswap(Int& i) { + switch (sizeof(Int)) { + case 16: + __bswap_16(i); + break; + case 32: + __bswap_32(i); + break; + case 64: + __bswap_64(i); + break; + default: { + } + } + return i; +} + +/// @brief Copies an integer from a pointer using the architecture's native +/// endianness. +/// @param ptr The pointer to copy from. +template +Int copy_ne(const void* const ptr) { + Int i; + memcpy(&i, ptr, sizeof(Int)); + return i; +} + +/// @brief Copies a little endian integer from a pointer. +/// @param ptr The pointer to copy from. +template +Int copy_le(const void* const ptr) { + Int i = copy_ne(ptr); +#if __BYTE_ORDER == __BIG_ENDIAN + byteswap(i); +#endif + return i; +} + +/// @brief Copies a big endian integer from a pointer. +/// @param ptr The pointer to copy from. +template +Int copy_be(const void* const ptr) { + Int i = copy_ne(ptr); +#if __BYTE_ORDER == __LITTLE_ENDIAN + byteswap(i); +#endif + return i; +} +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/concepts.hpp b/include/pasta/block_tree/utils/concepts.hpp new file mode 100644 index 0000000..a008698 --- /dev/null +++ b/include/pasta/block_tree/utils/concepts.hpp @@ -0,0 +1,55 @@ +#pragma once + +#include +#include +#include + +namespace pasta { + +/// @brief Represents an update function which given a value, +/// updates a value in the map. +/// +/// @tparam Fn The type of the update function. +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +concept UpdateFunction = + requires(const K& k, V& v_lv, typename Fn::InputValue in_v_rv) { + typename Fn::InputValue; + // Updates a pre-existing value in the map. + // Arguments are the key, the value in the map, + // and the input value used to update the value in + // the map + { Fn::update(k, v_lv, std::move(in_v_rv)) } -> std::same_as; + // Initialize a value from an input value + // Arguments are the key, and the value used to + // initialize the value in the map. This returns the + // value to be inserted into the map + { Fn::init(k, std::move(in_v_rv)) } -> std::convertible_to; + }; + +namespace internal { +using Capacity = size_t; +using ThreadCount = size_t; +} // namespace internal + +template +// Should have a constructor that allows the construction with a given capacity +// and thread count, should the queue need it +concept MpscQueue = + std::constructible_from && + requires(Queue q, std::unique_ptr&& e) { + // enqueue should enqueue a value and return true, if the element was + // enqueued (i.e. there was space) + { q.enqueue(std::move(e)) } -> std::convertible_to; // aa + // dequeue dequeue the oldest value and return it + { q.dequeue() } -> std::convertible_to>; + // size should return the current number of elements + { q.size() } -> std::convertible_to; + // capacity should return the maximum number of elements + // this queue can hold + { q.capacity() } -> std::convertible_to; + }; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/debug.hpp b/include/pasta/block_tree/utils/debug.hpp new file mode 100644 index 0000000..fb661d3 --- /dev/null +++ b/include/pasta/block_tree/utils/debug.hpp @@ -0,0 +1,29 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ +#pragma once + +#ifdef BT_DBG +# include +/// @brief An assertion that is only active, if BT_DBG is defined. +# define BT_ASSERT(x) assert(x) +#else +/// @brief An assertion that is only active, if BT_DBG is defined. +# define BT_ASSERT(x) +#endif diff --git a/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp new file mode 100644 index 0000000..15ce0ff --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/jiffy.hpp @@ -0,0 +1,108 @@ +#pragma once + +#include +#include +#include +#include +#include +#include + +namespace pasta { + +namespace jiffy { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wunused-variable" +#pragma GCC diagnostic ignored "-Wreorder" +#pragma GCC diagnostic ignored "-Wunused-parameter" +#pragma GCC diagnostic ignored "-Wparentheses" +#pragma GCC diagnostic ignored "-Wclass-memaccess" +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#pragma GCC diagnostic pop +} // namespace jiffy + +template +class JiffyQueue { + std::unique_ptr> queue_; + std::atomic_size_t size_; + size_t capacity_; + bool space_available_; + std::condition_variable enqueue_cv_; + std::mutex enqueue_mtx_; + std::condition_variable dequeue_cv_; + std::mutex dequeue_mtx_; + +public: + explicit JiffyQueue(size_t capacity, size_t) + : queue_(std::make_unique>(capacity)), + size_(0), + capacity_(capacity), + space_available_(capacity > 0), + enqueue_cv_(), + enqueue_mtx_(), + dequeue_cv_(), + dequeue_mtx_() {} + + JiffyQueue(JiffyQueue&& other) noexcept + : queue_(std::move(other.queue_)), + size_(other.size()), + capacity_(other.capacity_), + space_available_(other.space_available_), + enqueue_cv_(), + enqueue_mtx_(), + dequeue_cv_(), + dequeue_mtx_() {} + + ~JiffyQueue() { + while (size() > 0) { + dequeue().release(); + } + }; + + bool enqueue(std::unique_ptr&& elem) { + std::unique_lock lock(enqueue_mtx_); + const bool can_insert = + enqueue_cv_.wait_for(lock, std::chrono::milliseconds(1), [this] { + return size() < capacity_; + }); + if (!can_insert) { + return false; + } + assert(size() < capacity_); + size_.fetch_add(1); + queue_->enqueue(*elem.release()); + dequeue_cv_.notify_one(); + return true; + } + + std::unique_ptr dequeue() { + std::unique_lock lock(enqueue_mtx_); + const bool can_dequeue = + dequeue_cv_.wait_for(lock, std::chrono::milliseconds(10), [this] { + return size() > 0; + }); + if (!can_dequeue) { + return std::unique_ptr(nullptr); + } + assert(size() > 0); + const auto size = size_.fetch_sub(1) - 1; + T* t = new T; + const bool queue_was_not_empty = queue_->dequeue(*t); + if (!queue_was_not_empty) { + std::cout << "Size is " << size << std::endl; + assert(queue_was_not_empty); + } + enqueue_cv_.notify_one(); + return std::unique_ptr(static_cast(t)); + } + + [[nodiscard]] __attribute_noinline__ size_t size() const { + return size_.load(); + } + + [[nodiscard]] __attribute_noinline__ size_t capacity() const { + return capacity_; + } +}; + +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp new file mode 100644 index 0000000..2068f75 --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/mpscq.hpp @@ -0,0 +1,43 @@ +#pragma once + +#include "../concepts.hpp" + +namespace pasta { + +namespace mpscq { +#pragma GCC diagnostic push +#pragma GCC diagnostic ignored "-Wpedantic" +#include +#pragma GCC diagnostic pop +} // namespace mpscq + +template +class Mpscq { + mpscq::mpscq* queue_; + +public: + Mpscq(size_t capacity, size_t) + : queue_(mpscq::mpscq_create(NULL, capacity)) {} + + ~Mpscq() { + mpscq::mpscq_destroy(queue_); + } + + bool enqueue(std::unique_ptr&& elem) { + return mpscq::mpscq_enqueue(queue_, elem.release()); + } + + std::unique_ptr dequeue() { + return std::unique_ptr(static_cast(mpscq::mpscq_dequeue(queue_))); + } + + size_t size() const { + return mpscq::mpscq_count(queue_); + } + + size_t capacity() const { + return mpscq::mpscq_capacity(queue_); + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp new file mode 100644 index 0000000..bd0ca5c --- /dev/null +++ b/include/pasta/block_tree/utils/mpsc_queue/stupid_queue.hpp @@ -0,0 +1,71 @@ +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// @brief A simple queue that uses a std::list as the underlying data. Not even +/// lock-free. Probably bad, but should work. +/// @tparam T The type of the elements in the queue. +template +class StupidQueue { + std::list queue_; + size_t capacity_; + std::condition_variable enqueue_cv_; + std::mutex enqueue_mtx_; + +public: + explicit StupidQueue(size_t capacity, size_t) + : queue_(), + capacity_(capacity), + enqueue_cv_(), + enqueue_mtx_() {} + + StupidQueue(StupidQueue&& other) noexcept + : queue_(std::move(other.queue_)), + capacity_(other.capacity_), + enqueue_cv_(), + enqueue_mtx_() {} + + bool enqueue(std::unique_ptr&& elem) { + std::unique_lock lock(enqueue_mtx_); + const bool can_insert = + enqueue_cv_.wait_for(lock, std::chrono::milliseconds(10), [this] { + return size() < capacity_; + }); + if (!can_insert) { + return false; + } + assert(size() < capacity_); + queue_.push_back(*elem.release()); + return true; + } + + std::unique_ptr dequeue() { + assert(size() > 0); + T t = queue_.front(); + std::unique_ptr v = std::make_unique(std::move(t)); + { + std::lock_guard lock(enqueue_mtx_); + queue_.pop_front(); + } + enqueue_cv_.notify_one(); + return v; + } + + [[nodiscard]] __attribute_noinline__ size_t size() const { + return queue_.size(); + } + + [[nodiscard]] __attribute_noinline__ size_t capacity() const { + return capacity_; + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/sharded_map.hpp b/include/pasta/block_tree/utils/sharded_map.hpp new file mode 100644 index 0000000..d377eae --- /dev/null +++ b/include/pasta/block_tree/utils/sharded_map.hpp @@ -0,0 +1,263 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ +#pragma once + +#include +#include +#include +#include +#include +#include +#include +#include + +namespace pasta { + +/// @brief A hash map that must be used by multiple threads, each thread having +/// only having write access to a certain segment of the input space. +/// @tparam K The type of the keys in the hash map. +/// @tparam V The type of the values in the hash map. +/// @tparam SeqHashMapType The type of the hash map used internally. +/// This should be compatible with std::unordered_map. +/// @tparam QueueType The type of queue used for communication between threads. +/// @tparam UpdateFn The update function deciding how to insert or update values +/// in the map. +template typename SeqHashMapType = + std::unordered_map, + template typename QueueType = JiffyQueue, + UpdateFunction UpdateFn = Overwrite> + requires MpscQueue>, + std::pair> && + std::movable +class ShardedMap { + /// The sequential backing hash map type + using SeqHashMap = SeqHashMapType; + /// The sequential hash map's hasher + using Hasher = SeqHashMap::hasher; + /// The type used for updates + using InputValue = UpdateFn::InputValue; + /// The task queue used for communication between threads + using Queue = QueueType>; + + /// @brief A value between 0 and 1, determining to which extent + /// each thread's queue should be filled, before the thread is signaled to + /// handle its queued operations. + /// + /// If this value is 0.25, then the thread's value in threshold_met_ + /// is set to true, signaling that the thread should handle its requests in + /// task_queue_ + const double fill_threshold_; + /// @brief The number of threads operating on this map. + const size_t thread_count_; + /// @brief The capacity of each task_queue_. + const size_t queue_capacity_; + /// @brief Contains a hash map for each thread + std::vector map_; + /// @brief Contains a boolean for each thread, that is true, + /// iff the fill_threshold is met. + std::vector threshold_met_; + /// @brief Contains a task queue for each thread, holding insert + /// operations for each thread. + std::vector task_queue_; + + [[nodiscard]] bool threshold_exceeded(const size_t thread_id) const { + return static_cast(task_queue_[thread_id].size()) / + static_cast(task_queue_[thread_id].capacity()) >= + fill_threshold_; + } + +public: + // + /// @brief Creates a new sharded map. + /// + /// @param fill_threshold The fill percentage (between 0 and 1) above which + /// a thread is signaled to handle its own tasks. + /// @param thread_count The exact number of threads working on this map. + /// @param queue_capacity The maximum amount of tasks allowed in each queue. + /// + ShardedMap(double fill_threshold, size_t thread_count, size_t queue_capacity) + : fill_threshold_(fill_threshold), + thread_count_(thread_count), + queue_capacity_(queue_capacity), + map_(), + threshold_met_(), + task_queue_() { + assert(0 <= fill_threshold && fill_threshold <= 1); + threshold_met_.resize(thread_count, false); + map_.reserve(thread_count); + task_queue_.reserve(thread_count); + for (size_t i = 0; i < thread_count; i++) { + map_.emplace_back(); + task_queue_.emplace_back(queue_capacity, thread_count); + } + } + + class Shard { + ShardedMap& sharded_map_; + const size_t thread_id_; + SeqHashMap& map_; + char& threshold_met_; + Queue& task_queue_; + + public: + Shard(ShardedMap& sharded_map, size_t thread_id) + : sharded_map_(sharded_map), + thread_id_(thread_id), + map_(sharded_map_.map_[thread_id]), + threshold_met_(sharded_map_.threshold_met_[thread_id]), + task_queue_(sharded_map_.task_queue_[thread_id]) {} + + /// @brief Inserts or updates a new value in the map, depending on whether + /// @param k The key to insert or update a value for. + /// @param in_value The value with which to insert or update. + inline void insert_or_update_direct(K& k, InputValue&& in_value) { + auto res = map_.find(k); + if (res == map_.end()) { + // If the value does not exist, insert it + K key = k; + V initial = UpdateFn::init(key, std::move(in_value)); + map_.emplace(key, std::move(initial)); + } else { + // Otherwise, update it. + V& val = res->second; + UpdateFn::update(k, val, std::move(in_value)); + } + } + + /// @brief Handles this thread's queue, inserting or updating all values in + /// its queue. + void handle_queue() { + if (task_queue_.size() == 0) { + return; + } + while (task_queue_.size() > 0) { + std::unique_ptr> pair = task_queue_.dequeue(); + assert(pair != nullptr); + insert_or_update_direct(pair->first, std::move(pair->second)); + } + threshold_met_ = false; + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param pair The key-value pair to insert or update. + void insert(std::pair&& pair) { + const size_t hash = Hasher{}(pair.first); + const size_t target_thread_id = hash % sharded_map_.thread_count_; + + // Otherwise enqueue the new value in the target thread + Queue& q = sharded_map_.task_queue_[target_thread_id]; + while (q.size() == q.capacity()) { + handle_queue(); + } + while (!q.enqueue( + std::make_unique>(std::move(pair)))) { + handle_queue(); + }; + + // If the fill threshold is exceeded, mark it as such + sharded_map_.threshold_met_[target_thread_id] = + sharded_map_.threshold_exceeded(target_thread_id); + } + + /// @brief Determines whether this thread should handle its queue. + /// + /// This translates to whether this thread's queue's fill level exceeds the + /// fill threshold. + /// @return `true` iff the fill threshold is exceeded. + [[nodiscard]] bool should_handle_queue() const { + return threshold_met_; + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param key The key of the value to insert. + /// @param value The value to associate with the key. + inline void insert(K& key, InputValue value) { + insert(std::pair(key, value)); + } + }; + + Shard get_shard(const size_t thread_id) { + return Shard(*this, thread_id); + } + + /// @brief Returns the number of key-value pairs in the map. + /// + /// Note, that this method calculates the size for each map separately and + /// is therefore not O(1). + /// @return The number of key-value pairs in the map. + [[nodiscard]] size_t size() const { + size_t size = 0; + for (const SeqHashMap& map : map_) { + size += map.size(); + } + return size; + } + + /// @brief Runs a method for each value in the map. + /// + /// The given function must take const references to a key and a value + /// respectively. + /// @param f The function or lambda to run for each value. + void for_each(std::invocable auto f) { + for (const SeqHashMap& map : map_) { + for (const auto& [k, v] : map) { + f(k, v); + } + } + } + + SeqHashMap::iterator end() { + return map_.back().end(); + } + + SeqHashMap::iterator find(const K& key) { + const size_t hash = Hasher{}(key); + const size_t target_thread_id = hash % thread_count_; + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(key); + if (it == map.end()) { + return end(); + } + return it; + } + + void print_map_loads() { + for (size_t i = 0; i < map_.size(); ++i) { + std::cout << "Map " << i << " load: " << map_[i].size() << std::endl; + } + } + +}; // namespace pasta + +} // namespace pasta \ No newline at end of file diff --git a/include/pasta/block_tree/utils/sharded_util.hpp b/include/pasta/block_tree/utils/sharded_util.hpp new file mode 100644 index 0000000..c494eda --- /dev/null +++ b/include/pasta/block_tree/utils/sharded_util.hpp @@ -0,0 +1,354 @@ +#pragma once + +#include "pasta/bit_vector/bit_vector.hpp" +#include "pasta/block_tree/utils/MersenneHash.hpp" +#include "pasta/block_tree/utils/MersenneRabinKarp.hpp" + +#include +#include +#include +#include +#include + +/// @brief Utilities for the construction algorithms using the sharded hash map +namespace pasta { + +namespace internal::sharded { + +__extension__ typedef unsigned __int128 uint128_t; + +/// @brief A marker for a block that has no earlier occurrence +inline constexpr int64_t NO_EARLIER_OCC = -1; + +/// @brief A marker for a block that has been pruned +inline constexpr int64_t PRUNED = -2; + +/// @brief For some block size (in bytes) i, return the number of trailing +/// zeros in a 64 bit integer when zeroing out characters that are not part +/// of the block. +constexpr static uint64_t MASK_TRAILING_ZEROS[9] = + {64, 56, 48, 40, 32, 24, 16, 8, 0}; + +/// @brief Masks used for the identity hash. These depend on endianness +constexpr static std::array masks() { + if constexpr (std::endian::native == std::endian::big) { + return {0, + static_cast(~0) << MASK_TRAILING_ZEROS[1], + static_cast(~0) << MASK_TRAILING_ZEROS[2], + static_cast(~0) << MASK_TRAILING_ZEROS[3], + static_cast(~0) << MASK_TRAILING_ZEROS[4], + static_cast(~0) << MASK_TRAILING_ZEROS[5], + static_cast(~0) << MASK_TRAILING_ZEROS[6], + static_cast(~0) << MASK_TRAILING_ZEROS[7], + static_cast(~0) << MASK_TRAILING_ZEROS[8]}; + } else { + return {0, + static_cast(~0) >> MASK_TRAILING_ZEROS[1], + static_cast(~0) >> MASK_TRAILING_ZEROS[2], + static_cast(~0) >> MASK_TRAILING_ZEROS[3], + static_cast(~0) >> MASK_TRAILING_ZEROS[4], + static_cast(~0) >> MASK_TRAILING_ZEROS[5], + static_cast(~0) >> MASK_TRAILING_ZEROS[6], + static_cast(~0) >> MASK_TRAILING_ZEROS[7], + static_cast(~0) >> MASK_TRAILING_ZEROS[8]}; + } +} + +/// @brief Masks for identity hashes for a block size i (in bytes) +inline constexpr std::array HASH_MASKS = masks(); + +/// @brief Base of the polynomial used for the Rabin-Karp hasher +inline constexpr size_t SIGMA = 256; + +/// @brief The exponent of the mersenne prime used for the Rabin-Karp hasher +inline constexpr uint8_t PRIME_EXPONENT = 107; + +/// @brief A mersenne prime used for the Rabin-Karp hasher +inline constexpr uint128_t PRIME = pasta::mersenne_prime(); + +/// @brief Determine whether to use a Rabin-Karp hash for hashing text windows +/// or just use the block's content itself as a hash, stored in an integer. +enum class UseHash { + /// @brief Use a Rabin-Karp hash + RABIN_KARP, + /// @brief Use the block's content as a hash + IDENTITY +}; + +/// @brief Contains data about a block tree level under construction +template +struct LevelData { + /// @brief Contains a 1 for each internal block (= block with children) + /// and a 0 for each block that has a back pointer + std::unique_ptr is_internal; + /// @brief Rank data structure for is_internal + std::unique_ptr is_internal_rank; + /// @brief The block from which a back block is copying + std::unique_ptr> pointers; + /// @brief The offset into the block from which the back block is copying + std::unique_ptr> offsets; + /// @brief The number of back blocks pointing to the block + std::unique_ptr> counters; + /// @brief Block start indices + std::unique_ptr> block_starts; + /// @brief The block size on this level + int64_t block_size; + /// @brief The index of the current level. + /// First level is 0, second level is 1 etc. + int64_t level_index; + /// @brief The number of blocks on the current level + int64_t num_blocks; + + LevelData(const int64_t level_index_, + const int64_t block_size_, + const int64_t num_blocks_) + : is_internal(nullptr), + is_internal_rank(nullptr), + pointers(new std::vector()), + offsets(new std::vector()), + counters(new std::vector()), + block_starts(new std::vector()), + block_size(block_size_), + level_index(level_index_), + num_blocks(num_blocks_) {} + + /// @brief Checks whether a block is adjacent in the text + /// to its successor on this level + [[nodiscard]] bool next_is_adjacent(size_t i) const { + return (*block_starts)[i] + block_size == (*block_starts)[i + 1]; + } +}; + +/// @brief Contains data about the occurrences of a hashed block pair +template +struct PairOccurrences { + /// @brief The first block in the text in which the content appears + size_type first_occ_block; + /// @brief A list of block indices in which the content of the hashed block + /// pair appears + /// + /// We're using an std::list here instead of an std::vector, since the + /// reallocation upon insertion lead to issues during parallel access, when + /// another thread tries to access the vector during reallocation. + std::vector occurrences; + + /// @brief Initialize the occurrences of a hashed block pair. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the pair's first block. + inline explicit PairOccurrences(size_type first_occ_block_) + : first_occ_block(first_occ_block_), + occurrences() {} + + PairOccurrences(PairOccurrences&&) noexcept = default; + PairOccurrences& operator=(PairOccurrences&&) = default; + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block_pair(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index is an earlier occurrence, update it + /// @param block_index The block index of an occurrence + void update(size_type block_index) { + first_occ_block = std::min(first_occ_block, block_index); + } +}; + +/// @brief Contains data about the occurrences of a hashed block +template +struct BlockOccurrences { + /// @brief Represents the first occurrence of a block + struct [[gnu::packed]] FirstOccurrence { + /// @brief Block index of the first occurrence of the block's content + int64_t block : 40; + /// @brief The offset into the block at which that first occurrence occurs + int32_t offset : 24; + + inline FirstOccurrence(int64_t first_occ_block_, + int32_t first_occ_offset_) + : block(first_occ_block_), + offset(first_occ_offset_) {} + }; + + static_assert(std::atomic::is_always_lock_free, + "first occurrence must be able to be atomically updated"); + static_assert(sizeof(FirstOccurrence) == 8, + "should be size of computer word"); + + // @brief The block index and offset of the first occurrence of this block's + // content + std::atomic first_occ; + + /// @brief A list of block indices in which the content of the hashed block + /// occurs + std::vector occurrences; + + /// @brief Initialize the occurrences of a hashed block. + /// + /// Note, that this only sets the first occurrence to the given block index, + /// but does not add it to the occurrences list. + /// @param first_occ_block_ The block index of the block's first occurrence. + explicit BlockOccurrences(size_type first_occ_block_) + : first_occ({first_occ_block_, 0}), + occurrences() {} + + BlockOccurrences(const BlockOccurrences& other) + : first_occ(other.first_occ.load()), + occurrences(other.occurrences) {} + + BlockOccurrences(BlockOccurrences&& other) noexcept + : first_occ(other.first_occ.load()), + occurrences(std::move(other.occurrences)) {} + + ~BlockOccurrences() = default; + + BlockOccurrences& operator=(BlockOccurrences&& other) noexcept { + first_occ = other.first_occ.load(); + occurrences = std::move(other.occurrences); + return *this; + } + + /// @brief Add a block index to the occurrences. + /// @param block_index The block index to add to the occurrences. + void add_block(size_type block_index) { + occurrences.push_back(block_index); + } + + /// @brief If the given block index and offset are an earlier occurrence, + /// update them + /// @param block_index The block index of an occurrence + /// @param block_offset The offset of that occurrence + void update(size_type block_index, size_type block_offset) { + FirstOccurrence prev_first_occ = this->first_occ.load(); + FirstOccurrence set(block_index, block_offset); + while (block_index < prev_first_occ.block && + !first_occ.compare_exchange_weak(prev_first_occ, set)) { + } + } +}; + +/// @brief An update function for the sharded hash map that updates the +/// occurrences of a hashed block pair +template +struct UpdatePairOccurrences { + /// @brief The block index to add to the occurrences + using InputValue = size_type; + /// @brief Update the occurrences of a hashed block pair by adding the new + /// block index and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index to add to the occurrences + inline static void update(const MersenneHash&, + PairOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + } + + /// @brief Initialize the occurrences of a hashed block pair + /// @param input_value The block index of the pair's first block + /// @return The initialized occurrences only containing the given block pair + inline static PairOccurrences init(const MersenneHash&, + InputValue&& input_value) { + PairOccurrences occurrences(input_value); + occurrences.add_block_pair(input_value); + occurrences.update(input_value); + return occurrences; + } +}; + +/// @brief An update function for the sharded hash map that updates the +/// occurrences of a hashed block +template +struct UpdateBlockOccurrences { + /// @brief A pair of the block index + /// and offset of the first occurrence of a block + using InputValue = std::pair; + + /// @brief Update the occurrences of a hashed block by adding the new + /// block index and offset and updating the first occurrence if needed + /// @param occurrences A reference to the occurrences in the map + /// @param input_value The new block index and offset to add to the + /// occurrences + inline static void update(const MersenneHash&, + BlockOccurrences& occurrences, + InputValue&& input_value) { + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + } + + /// @brief Initialize the occurrences of a hashed block. + /// @param input_value A pair of the block index and offset of one of the + /// block's occurrences + /// @return The initialized occurrences only containing the given block + inline static BlockOccurrences + init(const MersenneHash&, InputValue&& input_value) { + BlockOccurrences occurrences(input_value.first); + occurrences.add_block(input_value.first); + occurrences.update(input_value.first, input_value.second); + return occurrences; + } +}; + +/// @brief A mixing functions to provide better avalanching to intermediate hash +/// values. +/// +/// https://zimbry.blogspot.com/2011/09/better-bit-mixing-improving-on.html +constexpr uint64_t mix_select(uint64_t key) { + key ^= (key >> 31); + key *= 0x7fb5d329728ea185; + key ^= (key >> 27); + key *= 0x81dadef4bc2dd44d; + key ^= (key >> 33); + return key; +} + +/// @brief Returns the ceiling of x / y for x > 0; +/// +/// https://stackoverflow.com/questions/2745074/fast-ceiling-of-an-integer-division-in-c-c +size_t ceil_div(std::integral auto x, std::integral auto y) { + return 1 + (static_cast(x) - 1) / static_cast(y); +} + +} // namespace internal::sharded + +/// +/// @brief An update function for sharded maps which on update just overwrites +/// the value. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Overwrite { + using InputValue = V; + + inline static void update(K&, V& value, V&& input_value) { + value = input_value; + } + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +/// +/// @brief An update function for sharded maps which upon update does nothing +/// besides inserting the value if it doesn't exist. +/// +/// @tparam K The key type saved in the hash map. +/// @tparam V The value type saved in the hash map. +/// +template +struct Keep { + using InputValue = V; + inline static void update(K&, V&, V&&) {} + + inline static V init(K&, V&& input_value) { + return input_value; + } +}; + +} // namespace pasta diff --git a/include/pasta/block_tree/utils/sync_sharded_map.hpp b/include/pasta/block_tree/utils/sync_sharded_map.hpp new file mode 100644 index 0000000..8628c6f --- /dev/null +++ b/include/pasta/block_tree/utils/sync_sharded_map.hpp @@ -0,0 +1,420 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ +#pragma once + +#include "sharded_util.hpp" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace pasta { + +enum Whereabouts { NOWHERE, IN_MAP, IN_QUEUE }; + +/// @brief A hash map that must be used by multiple threads, each thread having +/// only having write access to a certain segment of the input space. +/// @tparam K The type of the keys in the hash map. +/// @tparam V The type of the values in the hash map. +/// @tparam SeqHashMapType The type of the hash map used internally. +/// This should be compatible with std::unordered_map. +/// @tparam UpdateFn The update function deciding how to insert or update values +/// in the map. +template typename SeqHashMapType = + std::unordered_map, + UpdateFunction UpdateFn = pasta::Overwrite> + requires std::movable +class SyncShardedMap { + /// The sequential backing hash map type + using SeqHashMap = SeqHashMapType; + /// The sequential hash map's hasher + using Hasher = typename SeqHashMap::hasher; + /// The type used for updates + using InputValue = typename UpdateFn::InputValue; + + /// The actual pair of key and value stored in the map + using StoredValue = std::pair; + + using Queue = std::span; + + /// The memory order namespace from the standard library + using mem = std::memory_order; + + /// @brief The number of threads operating on this map. + const size_t thread_count_; + /// @brief Contains a hash map for each thread + std::vector map_; + /// @brief Contains a task queue for each thread, holding insert + /// operations for each thread. + std::vector task_queue_; + /// @brief Contains the number of tasks in each thread's queue. + std::span task_count_; + /// @brief Contains the number of threads currently handling their queues. + /// This is used 1. signal to other threads that they should handle their + /// queue, and 2. to keep track of whether all threads have handled their + /// queues. + std::atomic_size_t threads_handling_queue_; + + const size_t queue_capacity_; + +#ifdef BT_INSTRUMENT + std::atomic_size_t num_cycles_; + std::function FN = [this]() noexcept { + this->num_cycles_.fetch_add(1, mem::acq_rel); + }; +#else + constexpr static std::invocable auto FN = []() noexcept { + }; +#endif + + std::barrier barrier_; + +public: + std::atomic_size_t num_updates_; + std::atomic_size_t num_inserts_; + // + /// @brief Creates a new sharded map. + /// + /// @param thread_count The exact number of threads working on this map. + /// @param queue_capacity The maximum amount of tasks allowed in each queue. + /// @param map_capacity The initial capacity for each thread's local hash map. + /// + SyncShardedMap(size_t thread_count, size_t queue_capacity) + : thread_count_(thread_count), + map_(), + task_queue_(), + task_count_(), + threads_handling_queue_(0), + queue_capacity_(queue_capacity), + barrier_(static_cast(thread_count), FN), + num_updates_(0), + num_inserts_(0) { + map_.reserve(thread_count); + task_queue_.reserve(thread_count); + task_count_ = + std::span(new std::atomic_size_t[thread_count], + thread_count); + for (size_t i = 0; i < thread_count; i++) { + map_.emplace_back(); + task_queue_.emplace_back(new StoredValue[queue_capacity], queue_capacity); + task_count_[i] = 0; + } + } + + ~SyncShardedMap() { + delete[] task_count_.data(); + for (auto& queue : task_queue_) { + delete[] queue.data(); + } + } + + class Shard { + SyncShardedMap& sharded_map_; + const size_t thread_id_; + SeqHashMap& map_; + Queue& task_queue_; + std::atomic_size_t& task_count_; +#if defined BT_INSTRUMENT || defined BT_DBG + tlx::Aggregate start_idle_ns_; + tlx::Aggregate handle_queue_ns_; + tlx::Aggregate finish_idle_ns_; +#endif + + public: + Shard(SyncShardedMap& sharded_map, size_t thread_id) + : sharded_map_(sharded_map), + thread_id_(thread_id), + map_(sharded_map_.map_[thread_id]), + task_queue_(sharded_map_.task_queue_[thread_id]), + task_count_(sharded_map.task_count_[thread_id]) +#if defined BT_INSTRUMENT || defined BT_DBG + , + start_idle_ns_(), + handle_queue_ns_(), + finish_idle_ns_() +#endif + { + } + + /// @brief Inserts or updates a new value in the map, depending on whether + /// @param k The key to insert or update a value for. + /// @param in_value The value with which to insert or update. + inline void insert_or_update_direct(const K& k, InputValue&& in_value) { + auto res = map_.find(k); + if (res == map_.end()) { + // If the value does not exist, insert it + K key = k; + V initial = UpdateFn::init(key, std::move(in_value)); + map_.emplace(key, std::move(initial)); +#ifdef BT_INSTRUMENT + sharded_map_.num_inserts_.fetch_add(1, mem::acq_rel); +#endif + } else { + // Otherwise, update it. + V& val = res->second; + UpdateFn::update(k, val, std::move(in_value)); +#ifdef BT_INSTRUMENT + sharded_map_.num_updates_.fetch_add(1, mem::acq_rel); +#endif + } + } + + void handle_queue_sync(const bool make_others_wait = true) { + if (make_others_wait) { + // If this value is >0 then other threads will also handle their queue + // when trying to insert + sharded_map_.threads_handling_queue_.fetch_add(1, mem::acq_rel); + } +#ifdef BT_INSTRUMENT + auto now = std::chrono::high_resolution_clock::now(); +#endif + sharded_map_.barrier_.arrive_and_wait(); +#ifdef BT_INSTRUMENT + size_t ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + start_idle_ns_.add(ns_count); + + now = std::chrono::high_resolution_clock::now(); +#endif + handle_queue(); +#ifdef BT_INSTRUMENT + ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + handle_queue_ns_.add(ns_count); + + now = std::chrono::high_resolution_clock::now(); +#endif + if (make_others_wait) { + sharded_map_.threads_handling_queue_.fetch_sub(1, mem::acq_rel); + } + sharded_map_.barrier_.arrive_and_wait(); +#ifdef BT_INSTRUMENT + ns_count = std::chrono::duration_cast( + std::chrono::high_resolution_clock::now() - now) + .count(); + finish_idle_ns_.add(ns_count); +#endif + } + + /// @brief Handles this thread's queue, inserting or updating all values in + /// its queue, waiting for other threads to be + /// done with their handle_queue call. + void handle_queue() { + const size_t num_tasks_raw = task_count_.exchange(0, mem::acq_rel); + BT_ASSERT(num_tasks_raw <= sharded_map_.queue_capacity_); + const size_t num_tasks = + std::min(num_tasks_raw, sharded_map_.queue_capacity_); + if (num_tasks == 0) { + return; + } + + // Handle all tasks in the queue + for (size_t i = 0; i < num_tasks; ++i) { + auto& entry = task_queue_[i]; + insert_or_update_direct(entry.first, std::move(entry.second)); + } + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param pair The key-value pair to insert or update. + void insert(StoredValue&& pair) { + if (sharded_map_.threads_handling_queue_.load(mem::acquire) > 0) { + handle_queue_sync(); + } + const size_t hash = Hasher{}(pair.first); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % sharded_map_.thread_count_; + + // Otherwise enqueue the new value in the target thread + std::atomic_size_t& target_task_count = + sharded_map_.task_count_[target_thread_id]; + + size_t task_idx = target_task_count.fetch_add(1, mem::acq_rel); + // If the target queue is full, signal to the other threads, that they + // need to handle their queue and handle this thread's queue + if (task_idx >= sharded_map_.queue_capacity_) { + // Since we incremented that thread's task count, but didn't insert + // anything, we need to decrement it again so that it has the correct + // value + target_task_count.fetch_sub(1, mem::acq_rel); + handle_queue_sync(); + // Since the queue was handled, the task count is now 0 + insert(std::move(pair)); + return; + } + // Insert the value into the queue + sharded_map_.task_queue_[target_thread_id][task_idx] = std::move(pair); + } + + /// @brief Inserts or updates a new value in the map. + /// + /// If the value is inserted into the current thread's map, + /// it is inserted immediately. If not, then it is added to that thread's + /// queue. It will only be inserted into the map, once the thread comes + /// around to handle its queue using the handle_queue method. + /// + /// @param key The key of the value to insert. + /// @param value The value to associate with the key. + inline void insert(K& key, InputValue value) { + insert(StoredValue(key, value)); + } + +#ifdef BT_INSTRUMENT + [[nodiscard]] const tlx::Aggregate& start_idle_ns() const { + return start_idle_ns_; + } + + [[nodiscard]] const tlx::Aggregate& handle_queue_ns() const { + return handle_queue_ns_; + } + + [[nodiscard]] const tlx::Aggregate& finish_idle_ns() const { + return finish_idle_ns_; + } +#endif + }; + +#ifdef BT_INSTRUMENT + [[nodiscard]] size_t num_cycles() const { + return num_cycles_.load(mem::acquire); + } +#endif + + Shard get_shard(const size_t thread_id) { + return Shard(*this, thread_id); + } + + /// @brief Returns the number of key-value pairs in the map. + /// + /// Note, that this method calculates the size for each map separately and + /// is therefore not O(1). + /// @return The number of key-value pairs in the map. + [[nodiscard]] size_t size() const { + size_t size = 0; + for (const SeqHashMap& map : map_) { + size += map.size(); + } + return size; + } + + [[maybe_unused]] Whereabouts where(const K& k) { + const size_t hash = Hasher{}(k); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % thread_count_; + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(k); + if (it != map.end()) { + return IN_MAP; + } + Queue& queue = task_queue_[target_thread_id]; + for (size_t i = 0; i < task_count_[target_thread_id]; ++i) { + if (queue[i].first == k) { + return IN_QUEUE; + } + } + return NOWHERE; + } + + /// @brief Runs a method for each value in the map. + /// + /// The given function must take const references to a key and a value + /// respectively. + /// @param f The function or lambda to run for each value. + void for_each(std::invocable auto f) const { + for (const SeqHashMap& map : map_) { + for (const auto& [k, v] : map) { + f(k, v); + } + } + } + + typename SeqHashMap::iterator end() { + return map_.back().end(); + } + + typename SeqHashMap::iterator find(const K& key) { + const size_t hash = Hasher{}(key); + const size_t target_thread_id = + internal::sharded::mix_select(hash) % thread_count_; + SeqHashMap& map = map_[target_thread_id]; + typename SeqHashMap::iterator it = map.find(key); + if (it == map.end()) { + return end(); + } + return it; + } + + [[maybe_unused]] void print_map_loads() { + for (size_t i = 0; i < map_.size(); ++i) { + std::cout << "Map " << i << " load: " << map_[i].size() << std::endl; + } + } + + [[maybe_unused]] void print_queue_loads() { + auto so = std::osyncstream(std::cout); + + for (size_t i = 0; i < map_.size(); ++i) { + so << "Queue " << i << " load: " << task_count_[i].load(mem::acquire) + << "\n"; + } + so << std::endl; + } + + [[nodiscard]] std::vector map_loads() const { + std::vector loads; + loads.reserve(thread_count_); + for (size_t i = 0; i < thread_count_; ++i) { + loads.push_back(map_[i].size()); + } + return loads; + } + + [[maybe_unused]] void print_ins_upd() const { + std::osyncstream(std::cout) + << "Inserts: " << num_inserts_.load() + << "\nUpdates: " << num_updates_.load() << std::endl; + } + + std::barrier& barrier() { + return barrier_; + } + +}; // namespace pasta + +} // namespace pasta diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index eb76eb9..72fadf0 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -26,16 +26,19 @@ macro(pasta_block_tree_build_test TESTNAME) pasta_block_tree GTest::Main) include_directories(${TESTNAME_REPLACED} PRIVATE ${gtest_SOURCE_DIR}/include) - add_test( - NAME ${TESTNAME_REPLACED} - COMMAND ${TESTNAME_REPLACED} ${ARGN}) + #add_test( + # NAME ${TESTNAME_REPLACED} + # COMMAND ${TESTNAME_REPLACED} ${ARGN}) + gtest_add_tests(TARGET ${TESTNAME_REPLACED}) endmacro(pasta_block_tree_build_test) include(CTest) find_package(GTest REQUIRED) pasta_block_tree_build_test(block_tree/block_tree_fp_test) +pasta_block_tree_build_test(block_tree/block_tree_seq_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_test) pasta_block_tree_build_test(block_tree/block_tree_lpf_parallel_test) +pasta_block_tree_build_test(utils/bit_rabin_karp_test) ################################################################################ diff --git a/tests/block_tree/block_tree_fp_test.cpp b/tests/block_tree/block_tree_fp_test.cpp index a703ab4..6d3dfce 100644 --- a/tests/block_tree/block_tree_fp_test.cpp +++ b/tests/block_tree/block_tree_fp_test.cpp @@ -3,6 +3,7 @@ * * Copyright (C) 2022 Daniel Meyer * Copyright (C) 2023 Florian Kurpicz + * Copyright (C) 2023 Etienne Palanga * * pasta::block_tree is free software: you can redistribute it and/or modify * it under the terms of the GNU General Public License as published by @@ -19,24 +20,19 @@ * ******************************************************************************/ -#include -#include - #include - +#include #include -#include +#include +#include class BlockTreeFPTest : public ::testing::Test { - protected: - std::vector text; - pasta::BlockTreeFP* bt; - - void SetUp() override { + std::unique_ptr> bt; + void SetUp() override { std::random_device rd; std::mt19937 gen(rd()); std::uniform_int_distribution dist(0, 15); @@ -46,15 +42,11 @@ class BlockTreeFPTest : public ::testing::Test { for (size_t i = 0; i < text.size(); ++i) { text[i] = dist(gen); } - - bt = pasta::make_block_tree_fp(text, 2, 1); - bt->add_rank_support(); - } - void TearDown() override { - delete bt; + bt = std::unique_ptr>( + pasta::make_block_tree_fp(text, 2, 8)); + bt->add_rank_support(); } - }; TEST_F(BlockTreeFPTest, access) { @@ -82,8 +74,8 @@ TEST_F(BlockTreeFPTest, select) { } int main(int argc, char** argv) { - ::testing::InitGoogleTest(&argc, argv); - return RUN_ALL_TESTS(); + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); } /******************************************************************************/ diff --git a/tests/block_tree/block_tree_seq_test.cpp b/tests/block_tree/block_tree_seq_test.cpp new file mode 100644 index 0000000..5e40224 --- /dev/null +++ b/tests/block_tree/block_tree_seq_test.cpp @@ -0,0 +1,88 @@ +/******************************************************************************* + * This file is part of pasta::block_tree + * + * Copyright (C) 2022 Daniel Meyer + * Copyright (C) 2023 Florian Kurpicz + * Copyright (C) 2023 Etienne Palanga + * + * pasta::block_tree is free software: you can redistribute it and/or modify + * it under the terms of the GNU General Public License as published by + * the Free Software Foundation, either version 3 of the License, or + * (at your option) any later version. + * + * pasta::block_tree is distributed in the hope that it will be useful, + * but WITHOUT ANY WARRANTY; without even the implied warranty of + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the + * GNU General Public License for more details. + * + * You should have received a copy of the GNU General Public License + * along with pasta::block_tree. If not, see . + * + ******************************************************************************/ + +#include +#include + +#include + +#include +#include + +class BlockTreeSeqTest : public ::testing::Test { + +protected: + std::vector text; + + pasta::BlockTreeFP2 *bt; + + void SetUp() override { + + std::random_device rd; + std::mt19937 gen(1); + std::uniform_int_distribution dist(0, 15); + + size_t const string_length = 100000; + text.resize(string_length); + for (size_t i = 0; i < text.size(); ++i) { + text[i] = dist(gen) + 10; + } + + auto ptr = + std::make_unique>(text, 4, 8, 4); + bt = ptr.release(); + bt->add_rank_support(); + } + + void TearDown() override { delete bt; } +}; + +TEST_F(BlockTreeSeqTest, access) { + for (size_t i = 0; i < text.size(); ++i) { + ASSERT_EQ(bt->access(i), text[i]) << "for index " << i; + } +} + +TEST_F(BlockTreeSeqTest, rank) { + std::array hist = {0}; + + for (size_t i = 0; i < text.size() - 1; ++i) { + ++hist[text[i]]; + ASSERT_EQ(bt->rank(text[i], i), hist[text[i]]); + } +} + +TEST_F(BlockTreeSeqTest, select) { + std::array hist = {0}; + + for (size_t i = 0; i < text.size() - 1; ++i) { + ++hist[text[i]]; + ASSERT_EQ(bt->select(text[i], hist[text[i]]), i); + } +} + +int main(int argc, char **argv) { + ::testing::InitGoogleTest(&argc, argv); + return RUN_ALL_TESTS(); +} + +/******************************************************************************/ diff --git a/tests/utils/bit_rabin_karp_test.cpp b/tests/utils/bit_rabin_karp_test.cpp new file mode 100644 index 0000000..d7a3061 --- /dev/null +++ b/tests/utils/bit_rabin_karp_test.cpp @@ -0,0 +1,107 @@ + +#include "pasta/block_tree/utils/MersenneHash.hpp" + +#include +#include +#include +#include +#include +#include + +class BitRabinKarpTest : public ::testing::Test { +protected: + + static constexpr size_t unique_len = 100000; + pasta::BitVector bv; + + void SetUp() override { + std::random_device rd; + std::mt19937 gen{rd()}; + std::uniform_int_distribution dist(10); + bv.resize(unique_len * 2 + 3); + for (size_t i = 0; i < unique_len; ++i) { + bv[i] = dist(gen) % 2 == 0; + } + // Offset it by some amount to check that even misaligned bit sequences + // correctly match + bv[unique_len] = false; + bv[unique_len + 1] = false; + bv[unique_len + 2] = false; + for (size_t i = 0; i < unique_len; ++i) { + bv[unique_len + 3 + i] = static_cast(bv[i]); + } + } + +public: + bool + compare_ranges(const size_t s1, const size_t s2, const size_t len) const { + for (size_t i = 0; i < len; ++i) { + if (get_bit(s1 + i) != get_bit(s2 + i)) { + return false; + } + } + return true; + } + + [[nodiscard]] bool get_bit(const size_t bit_index) const { + return bv[bit_index]; + } +}; + +TEST_F(BitRabinKarpTest, test_slice) { + for (size_t i = 0; i < bv.size() - 1; i++) { + uint64_t manual_slice = 0; + for (size_t j = 0; j < 64; j++) { + manual_slice |= (static_cast(bv[i + j]) << j); + } + + uint64_t slice = pasta::MersenneHash::slice_at(bv, i); + ASSERT_EQ(manual_slice, slice) + << " bit index " << i + << "\nwith manual slice: " << std::bitset<64>(manual_slice) + << "\nwith hash slice: " << std::bitset<64>(slice); + } +} + +TEST_F(BitRabinKarpTest, test_eq) { + constexpr std::array sizes = {13, 24, 59, 1220}; + + for (const size_t size : sizes) { + pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); + pasta::MersenneRabinKarp rk2(bv, + unique_len + 3, + size, + (1ULL << 61) - 1); + for (size_t i = 0; i < unique_len - size; ++i) { + const auto h1 = rk1.current_hash(); + const auto h2 = rk2.current_hash(); + ASSERT_TRUE(h1 == h2) << "offset: " << i << " for half len " << unique_len + << " and size " << size; + rk1.next(); + rk2.next(); + } + } +} + +TEST_F(BitRabinKarpTest, test_rnd) { + constexpr std::array sizes = {13, 24, 59, 1220}; + + for (const size_t size : sizes) { + pasta::MersenneRabinKarp rk1(bv, 0, size, (1ULL << 61) - 1); + pasta::MersenneRabinKarp rk2(bv, + unique_len, + size, + (1ULL << 61) - 1); + size_t offset = 0; + for (size_t i = 0; i < unique_len - size; ++i) { + const auto h1 = rk1.current_hash(); + const auto h2 = rk2.current_hash(); + ASSERT_EQ((h1 == h2), compare_ranges(offset, unique_len + offset, size)) + << "error at offset " << offset << " for half_len " << unique_len + << " and window size " << size; + rk1.next(); + rk2.next(); + offset++; + } + } +}