Add parallel block tree construction

.github/workflows/ctest.yml

@@ -10,7 +10,6 @@ jobs:
     strategy:
       matrix:
         os: [ubuntu-latest]
-        compiler: [{cpp: g++-10, c: gcc-10}]
 
     runs-on: ${{ matrix.os }}
 
@@ -28,9 +27,6 @@ jobs:
 
     - name: Configure CMake
       run: cmake --preset=release -DPASTA_BLOCK_TREE_BUILD_TESTS=ON
-      env:
-        CC: gcc-10
-        CXX: g++-10
 
     - name: Build
       run: cmake --build ${{github.workspace}}/build/

.gitignore

@@ -1 +1,12 @@
-/build
+/build*
+/cmake-build-*
+.cache
+compile_commands.json
+.idea/*
+perf.data*
+.pdf
+*.txt
+!CMakeLists.txt
+debug
+dev/
+.vscode

.gitmodules

@@ -1,6 +1,37 @@
 [submodule "extlib/libsais"]
 	path = extlib/libsais
 	url = https://github.com/IlyaGrebnov/libsais
+[submodule "extlib/growt"]
+	path = extlib/growt
+	url = https://github.com/TooBiased/growt
+[submodule "extlib/parallel-hashmap"]
+	path = extlib/parallel-hashmap
+	url = https://github.com/greg7mdp/parallel-hashmap
+[submodule "extlib/waitfree-mpsc-queue"]
+	path = extlib/waitfree-mpsc-queue
+	url = https://github.com/dbittman/waitfree-mpsc-queue
+[submodule "extlib/Jiffy"]
+	path = extlib/Jiffy
+	url = https://github.com/DolevAdas/Jiffy
+[submodule "fastwfc"]
+	path = extlib/fastwfc/fast-wait-free-queue
+	url = https://github.com/chaoran/fast-wait-free-queue
+[submodule "extlib/Jiffy-1"]
+	path = extlib/Jiffy-1
+	url = https://github.com/quininer/Jiffy-1
+	branch = fix-atomic
+[submodule "extlib/sdsl-lite"]
+	path = extlib/sdsl-lite
+	url = https://github.com/Skadic/sdsl-lite
+[submodule "extlib/--force"]
+	path = extlib/--force
+	url = https://github.com/rizkg/BBHash
+[submodule "extlib/parlayhash"]
+	path = extlib/parlayhash
+	url = https://github.com/cmuparlay/parlayhash
+[submodule "extlib/unordered_dense"]
+	path = extlib/unordered_dense
+	url = https://github.com/martinus/unordered_dense
 [submodule "extlib/doxygen-awesome-css"]
 	path = extlib/doxygen-awesome-css
 	url = https://github.com/jothepro/doxygen-awesome-css.git

CMakeLists.txt

@@ -24,15 +24,23 @@ cmake_minimum_required(VERSION 3.20 FATAL_ERROR)
 set(CMAKE_CXX_STANDARD 20)
 set(CMAKE_CXX_STANDARD_REQUIRED ON)
 
-include(FetchContent)
-
 project(pasta_block_tree)
 
+# Generate compile_commands.json
+set(CMAKE_EXPORT_COMPILE_COMMANDS ON CACHE INTERNAL "")
+
 ## Build tests
 option(PASTA_BLOCK_TREE_BUILD_TESTS
         "Build blocktree's tests." OFF)
 option(PASTA_BLOCK_TREE_BUILD_EXAMPLES
-  "Build blocktree's benchmarks." OFF)
+        "Build blocktree's benchmarks." OFF)
+option(PASTA_BLOCK_TREE_DEBUG
+        "Print debug information" OFF)
+option(PASTA_BLOCK_TREE_BENCH
+        "Enable outputting benchmark information" OFF)
+
+include(ExternalProject)
+include(FetchContent)
 
 FetchContent_Declare(
   tlx
@@ -49,8 +57,7 @@ FetchContent_Declare(
 FetchContent_MakeAvailable(tlx pasta_bit_vector)
 
 # Optional test
-if(PASTA_BLOCK_TREE_BUILD_TESTS)
-    include(FetchContent)
+if (PASTA_BLOCK_TREE_BUILD_TESTS)
     FetchContent_Declare(
             googletest
             GIT_REPOSITORY https://github.com/google/googletest.git
@@ -59,27 +66,80 @@ if(PASTA_BLOCK_TREE_BUILD_TESTS)
     set(gtest_force_shared_crt ON CACHE BOOL "" FORCE)
     FetchContent_MakeAvailable(googletest)
     enable_testing()
-    add_subdirectory(tests)
     include(GoogleTest)
-endif()
-if(PASTA_BLOCK_TREE_BUILD_EXAMPLES)
-  add_executable(example
-    examples/block_tree_construction.cpp)
-  target_link_libraries(example
-    pasta_block_tree)
-endif()
+    add_subdirectory(tests)
+endif ()
+if (PASTA_BLOCK_TREE_BUILD_EXAMPLES)
+    add_executable(block_tree_construction
+            examples/block_tree_construction.cpp)
+    target_link_libraries(block_tree_construction
+            pasta_block_tree)
+
+    add_executable(build_bt
+            examples/build_bt.cpp)
+    target_link_libraries(build_bt
+            pasta_block_tree)
+    if (PASTA_BLOCK_TREE_DEBUG)
+        target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT)
+        target_compile_definitions(build_bt PRIVATE BT_DBG)
+    endif ()
+
+    if (PASTA_BLOCK_TREE_BENCH)
+        target_compile_definitions(build_bt PRIVATE BT_INSTRUMENT)
+        target_compile_definitions(build_bt PRIVATE BT_BENCH)
+    endif ()
+endif ()
 
 set(LIBSAIS_USE_OPENMP ON CACHE BOOL "Use OpenMP for parallelization of libsais" FORCE)
 add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/libsais)
 
+set(BUILD_DIVSUFSORT64 ON CACHE BOOL "Build libdivsufsort in 64-bits mode")
+add_subdirectory(${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/external/libdivsufsort)
+
+add_library(waitfree-mpsc-queue
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue/mpsc.c)
+target_include_directories(waitfree-mpsc-queue PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/waitfree-mpsc-queue)
+
+add_library(jiffy INTERFACE)
+target_include_directories(jiffy INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy)
+add_library(jiffy1 INTERFACE)
+target_include_directories(jiffy1 INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/Jiffy-1)
+
 add_library(pasta_block_tree INTERFACE)
 target_include_directories(pasta_block_tree INTERFACE
-  ${CMAKE_CURRENT_SOURCE_DIR}/include)
+        ${CMAKE_CURRENT_SOURCE_DIR}/include)
+
+# sdsl
+file(GLOB sdsl_sources ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/lib/*.cpp)
+add_library(sdsl STATIC ${sdsl_sources})
+target_include_directories(sdsl SYSTEM PUBLIC
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/sdsl-lite/include)
+target_include_directories(sdsl SYSTEM PUBLIC
+        ${CMAKE_CURRENT_BINARY_DIR}/extlib/sdsl-lite/external/libdivsufsort/include)
+target_link_libraries(sdsl PUBLIC divsufsort64)
+set_target_properties(sdsl PROPERTIES COMPILE_FLAGS "-w")
 
 target_link_libraries(pasta_block_tree INTERFACE
-  libsais
-  pasta_bit_vector
-  sdsl
-  tlx)
+        libsais
+        pasta_bit_vector
+        tlx
+        waitfree-mpsc-queue
+        sdsl
+        #jiffy
+        jiffy1
+        #        libzstd_static
+)
+
+target_include_directories(pasta_block_tree INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/growt)
+target_include_directories(pasta_block_tree INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parallel-hashmap/parallel_hashmap)
+target_include_directories(pasta_block_tree SYSTEM INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/parlayhash/include)
+target_include_directories(pasta_block_tree SYSTEM INTERFACE
+        ${CMAKE_CURRENT_SOURCE_DIR}/extlib/unordered_dense/include)
 
 ################################################################################

CMakePresets.json

@@ -1,65 +1,124 @@
 {
-    "version": 2,
-    "cmakeMinimumRequired": {
-        "major": 3,
-        "minor": 20,
-        "patch": 0
+  "version": 2,
+  "cmakeMinimumRequired": {
+    "major": 3,
+    "minor": 20,
+    "patch": 0
+  },
+  "configurePresets": [
+    {
+      "name": "default",
+      "displayName": "Default",
+      "description": "Default build options",
+      "hidden": true,
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always",
+        "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3",
+        "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3 -lprofiler",
+        "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -static-libasan -fsanitize=address -fsanitize=leak -fsanitize=undefined"
+      }
     },
-    "configurePresets": [
-        {
-            "name": "default",
-            "displayName": "Default",
-            "description": "Default build options",
-            "hidden": true,
-            "generator": "Ninja",
-            "binaryDir": "${sourceDir}/build",
-            "cacheVariables": {
-                "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always",
-                "CMAKE_CXX_FLAGS_RELEASE": "-DNDEBUG -O3",
-                "CMAKE_CXX_FLAGS_RELWITHDEBINFO": "-DDEBUG -g -O3",
-                "CMAKE_CXX_FLAGS_DEBUG": "-DDEBUG -O0 -g -ggdb -fsanitize=address"
-            }
-        },
-        {
-            "name": "release",
-            "displayName": "Release",
-            "inherits": "default",
-            "binaryDir": "${sourceDir}/build",
-            "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "Release"
-            }
-        },
-        {
-            "name": "relwithdeb",
-            "displayName": "ReleaseWithDebugInfo",
-            "inherits": "default",
-            "binaryDir": "${sourceDir}/build_with_debug_info",
-            "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "RelWithDebInfo"
-            }
-        },
-        {
-            "name": "debug",
-            "displayName": "Debug",
-            "inherits": "default",
-            "binaryDir": "${sourceDir}/debug",
-            "cacheVariables": {
-                "CMAKE_BUILD_TYPE": "Debug"
-            }
-        }
-    ],
-    "buildPresets": [
-        {
-            "name": "release",
-            "configurePreset": "release"
-        },
-        {
-            "name": "relwithdeb",
-            "configurePreset": "relwithdeb"
-        },
-        {
-            "name": "debug",
-            "configurePreset": "debug"
-        }
-    ]
+    {
+      "name": "release",
+      "displayName": "Release",
+      "inherits": "default",
+      "binaryDir": "${sourceDir}/build",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Release"
+      }
+    },
+    {
+      "name": "relwithdeb",
+      "displayName": "ReleaseWithDebugInfo",
+      "inherits": "default",
+      "binaryDir": "${sourceDir}/build_with_debug_info",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "RelWithDebInfo"
+      }
+    },
+    {
+      "name": "debug",
+      "displayName": "Debug",
+      "inherits": "default",
+      "binaryDir": "${sourceDir}/debug",
+      "cacheVariables": {
+        "CMAKE_BUILD_TYPE": "Debug"
+      }
+    },
+    {
+      "name": "ninja-multi",
+      "displayName": "Ninja Multi-Config",
+      "description": "Default build using Ninja Multi-Config generator",
+      "generator": "Ninja Multi-Config",
+      "binaryDir": "${sourceDir}/build_multi",
+      "inherits": "default",
+      "cacheVariables": {
+        "PASTA_BLOCK_TREE_BUILD_TESTS": "ON",
+        "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON"
+      }
+    },
+    {
+      "name": "bench",
+      "displayName": "Behcnmarking Configuration",
+      "description": "Release build with extended debug information",
+      "generator": "Ninja",
+      "binaryDir": "${sourceDir}/build_bench",
+      "inherits": "default",
+      "cacheVariables": {
+        "PASTA_BLOCK_TREE_BUILD_EXAMPLES": "ON",
+        "CMAKE_CXX_FLAGS": "-fopenmp -Wall -Wextra -pedantic -Werror -march=native -fdiagnostics-color=always -g -O3 -lprofiler",
+        "CMAKE_BUILD_TYPE": "Release",
+        "PASTA_BLOCK_TREE_BENCH": "ON"
+      }
+    }
+  ],
+  "buildPresets": [
+    {
+      "name": "release",
+      "configurePreset": "release"
+    },
+    {
+      "name": "relwithdeb",
+      "configurePreset": "relwithdeb"
+    },
+    {
+      "name": "debug",
+      "configurePreset": "debug"
+    },
+    {
+      "name": "release-multi",
+      "configurePreset": "ninja-multi",
+      "configuration": "Release"
+    },
+    {
+      "name": "relwithdeb-multi",
+      "configurePreset": "ninja-multi",
+      "configuration": "RelWithDebInfo"
+    },
+    {
+      "name": "debug-multi",
+      "configurePreset": "ninja-multi",
+      "configuration": "Debug"
+    },
+    {
+      "name": "bench",
+      "configurePreset": "bench"
+    }
+  ],
+  "testPresets": [
+    {
+      "name": "default",
+      "configurePreset": "ninja-multi",
+      "output": {
+        "outputOnFailure": true
+      },
+      "execution": {
+        "noTestsAction": "error",
+        "stopOnFailure": true
+      },
+      "configuration": "RelWithDebInfo"
+    }
+  ]
 }