diff --git a/CMakeLists.txt b/CMakeLists.txt index 1575bd1..5fb55f7 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -4,12 +4,52 @@ include(cmake/platform.cmake) set(USE_ASM ON CACHE BOOL "Use asm implementation for Fr and Fq") set(USE_OPENMP ON CACHE BOOL "Use OpenMP") +set(USE_MARCH_NATIVE OFF CACHE BOOL "Use -march=native for host builds (host-specific; not portable across CPUs)") +set(USE_LTO OFF CACHE BOOL "Enable Link Time Optimization (LTO)") project(rapidsnark LANGUAGES CXX C ASM) set(CMAKE_CXX_STANDARD 11) set(CMAKE_CXX_STANDARD_REQUIRED ON) +# Host-only optimization flags +if (USE_MARCH_NATIVE AND NOT CMAKE_CROSSCOMPILING) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -march=native -mtune=native") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -march=native -mtune=native") + message("Using -march=native -mtune=native") +endif() + +if (USE_LTO) + set(CMAKE_INTERPROCEDURAL_OPTIMIZATION ON) + message("LTO enabled") +endif() + +# Profile-Guided Optimization (host-only, two-phase). PGO=generate builds an +# instrumented binary that writes .gcda profiles to PGO_DIR during a training +# run; PGO=use rebuilds using those profiles. Flags go on both compile and link. +# Only the C++/C glue benefits (the field arithmetic is hand-written NASM). +set(PGO "" CACHE STRING "Profile-guided optimization mode: generate | use | (empty=off)") +set(PGO_DIR "${CMAKE_BINARY_DIR}/pgo-data" CACHE PATH "Directory for PGO .gcda profile data") + +if (PGO STREQUAL "generate") + set(PGO_FLAGS "-fprofile-generate=${PGO_DIR}") + message("PGO: instrumenting (profiles -> ${PGO_DIR})") +elseif (PGO STREQUAL "use") + # -fprofile-correction tolerates the multithreaded prover's racy counters; + # -Wno-missing-profile keeps functions with no profile from erroring. + set(PGO_FLAGS "-fprofile-use=${PGO_DIR} -fprofile-correction -Wno-missing-profile") + message("PGO: using profiles from ${PGO_DIR}") +elseif (NOT PGO STREQUAL "") + message(FATAL_ERROR "PGO must be 'generate', 'use', or empty (got '${PGO}')") +endif() + +if (PGO_FLAGS) + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PGO_FLAGS}") + set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PGO_FLAGS}") + set(CMAKE_EXE_LINKER_FLAGS "${CMAKE_EXE_LINKER_FLAGS} ${PGO_FLAGS}") + set(CMAKE_SHARED_LINKER_FLAGS "${CMAKE_SHARED_LINKER_FLAGS} ${PGO_FLAGS}") +endif() + message("BITS_PER_CHUNK=" ${BITS_PER_CHUNK}) message("USE_ASM=" ${USE_ASM}) message("USE_OPENMP=" ${USE_OPENMP}) diff --git a/Makefile b/Makefile index 90f684f..37e92c8 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ ### -#Build targets +# Build targets host: rm -rf build_prover && mkdir build_prover && cd build_prover && \ cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package && \ @@ -11,6 +11,44 @@ host_noasm: cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_noasm -DUSE_ASM=NO && \ make -j$(nproc) -vvv && make install +host_march: + rm -rf build_prover_march && mkdir build_prover_march && cd build_prover_march && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_march -DUSE_MARCH_NATIVE=ON && \ + make -j$(nproc) -vvv && make install + +host_lto: + rm -rf build_prover_lto && mkdir build_prover_lto && cd build_prover_lto && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_lto -DUSE_LTO=ON && \ + make -j$(nproc) -vvv && make install + +host_march_lto: + rm -rf build_prover_march_lto && mkdir build_prover_march_lto && cd build_prover_march_lto && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_march_lto -DUSE_MARCH_NATIVE=ON -DUSE_LTO=ON && \ + make -j$(nproc) -vvv && make install + +# Profile-Guided Optimization on top of LTO. Two-phase: build an instrumented +# prover, run it on a TRAINING circuit to collect profiles, then rebuild using +# them. The training circuit should resemble your production workload -- a tiny +# circuit produces a profile that can pessimize large-circuit proving. Override: +# make host_pgo PGO_ZKEY=/path/circuit_final.zkey PGO_WTNS=/path/witness.wtns +# Multiple training runs accumulate (.gcda counters sum), improving coverage. +PGO_ZKEY ?= testdata/circuit_final.zkey +PGO_WTNS ?= testdata/witness.wtns +host_pgo: + rm -rf build_prover_pgo && mkdir build_prover_pgo + @echo "=== PGO phase 1/2: instrumented build (LTO on) ===" + cd build_prover_pgo && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_pgo \ + -DUSE_LTO=ON -DPGO=generate && \ + make -j$(nproc) + @echo "=== PGO training run on $(PGO_ZKEY) ===" + ./build_prover_pgo/src/prover $(PGO_ZKEY) $(PGO_WTNS) /tmp/pgo_train_proof.json /tmp/pgo_train_public.json + @echo "=== PGO phase 2/2: rebuild using collected profiles ===" + cd build_prover_pgo && \ + cmake .. -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_pgo \ + -DUSE_LTO=ON -DPGO=use && \ + make -j$(nproc) && make install + host_arm64: rm -rf build_prover_arm64 && mkdir build_prover_arm64 && cd build_prover_arm64 && \ cmake .. -DTARGET_PLATFORM=aarch64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_arm64 && \ @@ -21,6 +59,14 @@ android: cmake .. -DTARGET_PLATFORM=ANDROID -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android -DBUILD_TESTS=OFF -DUSE_OPENMP=OFF && \ make -j$(nproc) -vvv && make install +# LTO variant. On arm64 only the innermost limb multiply is ASM; the rest of the +# field layer plus curve/FFT/MSM glue is C++, so LTO's cross-TU inlining has more +# to work on than on x86. Portable (unlike -march=native). Slower/heavier link. +android_lto: + rm -rf build_prover_android_lto && mkdir build_prover_android_lto && cd build_prover_android_lto && \ + cmake .. -DTARGET_PLATFORM=ANDROID -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android_lto -DBUILD_TESTS=OFF -DUSE_OPENMP=OFF -DUSE_LTO=ON && \ + make -j$(nproc) -vvv && make install + android_openmp: rm -rf build_prover_android_openmp && mkdir build_prover_android_openmp && cd build_prover_android_openmp && \ cmake .. -DTARGET_PLATFORM=ANDROID -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android_openmp -DBUILD_TESTS=OFF -DUSE_OPENMP=ON && \ @@ -31,6 +77,13 @@ android_x86_64: cmake .. -DTARGET_PLATFORM=ANDROID_x86_64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android_x86_64 -DBUILD_TESTS=OFF -DUSE_OPENMP=OFF && \ make -j$(nproc) -vvv && make install +# LTO variant for x86_64 Android (emulator / x86 devices). Same portable LTO +# path as android_lto; useful for benchmarking LTO on the emulator. +android_x86_64_lto: + rm -rf build_prover_android_x86_64_lto && mkdir build_prover_android_x86_64_lto && cd build_prover_android_x86_64_lto && \ + cmake .. -DTARGET_PLATFORM=ANDROID_x86_64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android_x86_64_lto -DBUILD_TESTS=OFF -DUSE_OPENMP=OFF -DUSE_LTO=ON && \ + make -j$(nproc) -vvv && make install + android_openmp_x86_64: rm -rf build_prover_android_openmp_x86_64 && mkdir build_prover_android_openmp_x86_64 && cd build_prover_android_openmp_x86_64 && \ cmake .. -DTARGET_PLATFORM=ANDROID_x86_64 -DCMAKE_BUILD_TYPE=Release -DCMAKE_INSTALL_PREFIX=../package_android_openmp_x86_64 -DBUILD_TESTS=OFF -DUSE_OPENMP=ON && \ @@ -45,6 +98,18 @@ ios: cp ../depends/gmp/package_ios_arm64/lib/libgmp.a src/Release-iphoneos && \ echo "" && echo "iOS Simulator artifacts built in build_prover_ios/src/Release-iphoneos" && echo "" +# LTO variant for iOS. -DUSE_LTO=ON maps to the LLVM_LTO Xcode setting via +# CMAKE_INTERPROCEDURAL_OPTIMIZATION. Same rationale as android_lto (mostly-C++ +# field/curve layer on arm64). Portable; slower/heavier link. +ios_lto: + @if [ ! -d "./depends/gmp/package_ios_arm64" ]; then echo "Looks like gmp lib is not built. Run './build_gmp.sh ios' first." && exit 1; fi + rm -rf build_prover_ios_lto && mkdir build_prover_ios_lto && cd build_prover_ios_lto && \ + cmake .. -GXcode -DTARGET_PLATFORM=IOS -DCMAKE_INSTALL_PREFIX=../package_ios_lto -DUSE_LTO=ON && \ + xcodebuild -destination 'generic/platform=iOS' -scheme rapidsnarkStatic -project rapidsnark.xcodeproj -configuration Release && \ + xcodebuild -destination 'generic/platform=iOS' -scheme rapidsnark -project rapidsnark.xcodeproj -configuration Release CODE_SIGNING_ALLOWED=NO && \ + cp ../depends/gmp/package_ios_arm64/lib/libgmp.a src/Release-iphoneos && \ + echo "" && echo "iOS LTO artifacts built in build_prover_ios_lto/src/Release-iphoneos" && echo "" + ios_simulator: @if [ ! -d "./depends/gmp/package_iphone_simulator" ]; then echo "Looks like gmp lib is not built. Run './build_gmp.sh ios_simulator' first." && exit 1; fi rm -rf build_prover_ios_simulator && mkdir build_prover_ios_simulator && cd build_prover_ios_simulator && \ @@ -68,18 +133,34 @@ macos_x86_64: clean: rm -rf build_prover \ + build_prover_noasm \ + build_prover_lto \ + build_prover_march \ + build_prover_march_lto \ + build_prover_pgo \ build_prover_macos_arm64 \ build_prover_macos_x86_64 \ build_prover_android \ + build_prover_android_lto \ build_prover_android_x86_64 \ + build_prover_android_x86_64_lto \ build_prover_ios \ + build_prover_ios_lto \ build_prover_ios_simulator \ package \ + package_noasm \ + package_lto \ + package_march \ + package_march_lto \ + package_pgo \ package_macos_arm64 \ package_macos_x86_64 \ package_android \ + package_android_lto \ package_android_x86_64 \ + package_android_x86_64_lto \ package_ios \ + package_ios_lto \ package_ios_simulator \ depends/gmp/package \ depends/gmp/package_macos_arm64 \ diff --git a/depends/ffiasm b/depends/ffiasm index aa90166..b9a6261 160000 --- a/depends/ffiasm +++ b/depends/ffiasm @@ -1 +1 @@ -Subproject commit aa90166dc4c5a075b835a398e15cc1e06ac90e95 +Subproject commit b9a62615428e74cc151249d4b556df7126a21391 diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index 720411c..3d381de 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -143,6 +143,10 @@ if(BUILD_TESTS) target_link_libraries(test_public_size rapidsnarkStaticFrFq pthread) add_test(NAME test_public_size COMMAND test_public_size circuit_final.zkey 86 WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}/testdata) + + add_executable(test_msm test_msm.cpp) + target_link_libraries(test_msm rapidsnarkStaticFrFq pthread) + add_test(NAME test_msm COMMAND test_msm) endif() if(OpenMP_CXX_FOUND) @@ -156,6 +160,9 @@ if(OpenMP_CXX_FOUND) target_link_libraries(prover OpenMP::OpenMP_CXX) target_link_libraries(verifier OpenMP::OpenMP_CXX) target_link_libraries(test_public_size OpenMP::OpenMP_CXX) + if(BUILD_TESTS) + target_link_libraries(test_msm OpenMP::OpenMP_CXX) + endif() endif() endif() diff --git a/src/binfile_utils.hpp b/src/binfile_utils.hpp index e2a9d23..1113152 100644 --- a/src/binfile_utils.hpp +++ b/src/binfile_utils.hpp @@ -4,6 +4,7 @@ #include #include #include +#include #include "fileloader.hpp" namespace BinFileUtils { diff --git a/src/groth16.cpp b/src/groth16.cpp index a4acc14..c379859 100644 --- a/src/groth16.cpp +++ b/src/groth16.cpp @@ -1,9 +1,13 @@ #include "random_generator.hpp" #include "logging.hpp" #include "misc.hpp" +#include "msm.hpp" #include #include #include +#include +#include +#include namespace Groth16 { @@ -50,32 +54,85 @@ template std::unique_ptr> Prover::prove(typename Engine::FrElement *wtns) { ThreadPool &threadPool = ThreadPool::defaultPool(); + const uint64_t nThreads = threadPool.getThreadCount(); - LOG_TRACE("Start Multiexp A"); uint32_t sW = sizeof(wtns[0]); typename Engine::G1Point pi_a; - E.g1.multiMulByScalarMSM(pi_a, pointsA, (uint8_t *)wtns, sW, nVars); + typename Engine::G1Point pib1; + typename Engine::G2Point pi_b; + typename Engine::G1Point pi_c; + + uint8_t *scalarsABB2 = (uint8_t *)wtns; + uint8_t *scalarsC = (uint8_t *)((uint64_t)wtns + (nPublic +1)*sW); + const uint64_t nC = nVars - nPublic - 1; + + if (nThreads >= 12) { + // Batched: prepare all four witness MSMs, then run every bucket task + // in one parallel region so no MSM's straggler tail leaves cores idle. + // Each MSM sizes windows/slices for a quarter of the pool. + LOG_TRACE("Start Multiexp A+B1+B2+C (batched)"); + const uint64_t share = std::max(1, nThreads/4); + + MSM msmA(E.g1), msmB1(E.g1), msmC(E.g1); + MSM msmB2(E.g2); + + msmA.prepare(pointsA, scalarsABB2, sW, nVars, share); + msmB1.prepare(pointsB1, scalarsABB2, sW, nVars, share); + msmB2.prepare(pointsB2, scalarsABB2, sW, nVars, share); + msmC.prepare(pointsC, scalarsC, sW, nC, share); + + // One bucket arena serves every MSM: a thread runs one task at a + // time, and a shared stride keeps the per-thread rows disjoint + // across curves. + const uint64_t arenaBytes = std::max( + std::max(msmA.arenaBytesPerThread(), msmB1.arenaBytesPerThread()), + std::max(msmC.arenaBytesPerThread(), msmB2.arenaBytesPerThread())); + + std::unique_ptr arena(arenaBytes ? new uint8_t[nThreads * arenaBytes] : nullptr); + + std::vector> tasks; + + // G2 tasks are the heaviest per point: schedule them first. + msmB2.collectTasks(tasks, arena.get(), arenaBytes); + msmA.collectTasks(tasks, arena.get(), arenaBytes); + msmB1.collectTasks(tasks, arena.get(), arenaBytes); + msmC.collectTasks(tasks, arena.get(), arenaBytes); + + if (!tasks.empty()) { + threadPool.parallelFor(0, tasks.size(), [&] (int begin, int end, int numThread) { + for (int t = begin; t < end; t++) { + tasks[t]((uint64_t)numThread); + } + }); + } + + msmA.finish(pi_a); + msmB1.finish(pib1); + msmB2.finish(pi_b); + msmC.finish(pi_c); + } else { + LOG_TRACE("Start Multiexp A"); + E.g1.multiMulByScalarMSM(pi_a, pointsA, scalarsABB2, sW, nVars); + + LOG_TRACE("Start Multiexp B1"); + E.g1.multiMulByScalarMSM(pib1, pointsB1, scalarsABB2, sW, nVars); + + LOG_TRACE("Start Multiexp B2"); + E.g2.multiMulByScalarMSM(pi_b, pointsB2, scalarsABB2, sW, nVars); + + LOG_TRACE("Start Multiexp C"); + E.g1.multiMulByScalarMSM(pi_c, pointsC, scalarsC, sW, nC); + } + std::ostringstream ss2; ss2 << "pi_a: " << E.g1.toString(pi_a); LOG_DEBUG(ss2); - - LOG_TRACE("Start Multiexp B1"); - typename Engine::G1Point pib1; - E.g1.multiMulByScalarMSM(pib1, pointsB1, (uint8_t *)wtns, sW, nVars); std::ostringstream ss3; ss3 << "pib1: " << E.g1.toString(pib1); LOG_DEBUG(ss3); - - LOG_TRACE("Start Multiexp B2"); - typename Engine::G2Point pi_b; - E.g2.multiMulByScalarMSM(pi_b, pointsB2, (uint8_t *)wtns, sW, nVars); std::ostringstream ss4; ss4 << "pi_b: " << E.g2.toString(pi_b); LOG_DEBUG(ss4); - - LOG_TRACE("Start Multiexp C"); - typename Engine::G1Point pi_c; - E.g1.multiMulByScalarMSM(pi_c, pointsC, (uint8_t *)((uint64_t)wtns + (nPublic +1)*sW), sW, nVars-nPublic-1); std::ostringstream ss5; ss5 << "pi_c: " << E.g1.toString(pi_c); LOG_DEBUG(ss5); @@ -129,68 +186,31 @@ std::unique_ptr> Prover::prove(typename Engine::FrElement }); LOG_TRACE("Initializing fft"); - u_int32_t domainPower = fft->log2(domainSize); - - LOG_TRACE("Start iFFT A"); - fft->ifft(a, domainSize); - LOG_TRACE("a After ifft:"); - LOG_DEBUG(E.fr.toString(a[0]).c_str()); - LOG_DEBUG(E.fr.toString(a[1]).c_str()); - LOG_TRACE("Start Shift A"); - - threadPool.parallelFor(0, domainSize, [&] (int64_t begin, int64_t end, uint64_t idThread) { - for (u_int64_t i=begin; iroot(domainPower+1, i)); - } - }); - LOG_TRACE("a After shift:"); - LOG_DEBUG(E.fr.toString(a[0]).c_str()); - LOG_DEBUG(E.fr.toString(a[1]).c_str()); - LOG_TRACE("Start FFT A"); - fft->fft(a, domainSize); - LOG_TRACE("a After fft:"); - LOG_DEBUG(E.fr.toString(a[0]).c_str()); - LOG_DEBUG(E.fr.toString(a[1]).c_str()); - LOG_TRACE("Start iFFT B"); - fft->ifft(b, domainSize); - LOG_TRACE("b After ifft:"); - LOG_DEBUG(E.fr.toString(b[0]).c_str()); - LOG_DEBUG(E.fr.toString(b[1]).c_str()); - LOG_TRACE("Start Shift B"); - threadPool.parallelFor(0, domainSize, [&] (int64_t begin, int64_t end, uint64_t idThread) { - for (u_int64_t i=begin; iroot(domainPower+1, i)); - } - }); - LOG_TRACE("b After shift:"); - LOG_DEBUG(E.fr.toString(b[0]).c_str()); - LOG_DEBUG(E.fr.toString(b[1]).c_str()); - LOG_TRACE("Start FFT B"); - fft->fft(b, domainSize); - LOG_TRACE("b After fft:"); - LOG_DEBUG(E.fr.toString(b[0]).c_str()); - LOG_DEBUG(E.fr.toString(b[1]).c_str()); - - LOG_TRACE("Start iFFT C"); - fft->ifft(c, domainSize); - LOG_TRACE("c After ifft:"); - LOG_DEBUG(E.fr.toString(c[0]).c_str()); - LOG_DEBUG(E.fr.toString(c[1]).c_str()); - LOG_TRACE("Start Shift C"); - threadPool.parallelFor(0, domainSize, [&] (int64_t begin, int64_t end, uint64_t idThread) { - for (u_int64_t i=begin; iroot(domainPower+1, i)); - } - }); - LOG_TRACE("c After shift:"); - LOG_DEBUG(E.fr.toString(c[0]).c_str()); - LOG_DEBUG(E.fr.toString(c[1]).c_str()); - LOG_TRACE("Start FFT C"); - fft->fft(c, domainSize); - LOG_TRACE("c After fft:"); - LOG_DEBUG(E.fr.toString(c[0]).c_str()); - LOG_DEBUG(E.fr.toString(c[1]).c_str()); + // Permutation-free coset pipeline: DIF-iFFT leaves the coefficients in + // bit-reversed order, the fused pointwise pass applies the ω_2n coset + // shift (with 1/n folded in) through the bit-reversed table, and the + // DIT-FFT consumes bit-reversed input, returning the coset evaluations + // in natural order — same result as ifft+shift+fft, minus six + // bit-reversal permutation passes and three scaling passes. + typename Engine::FrElement *abc[3] = { a, b, c }; + + for (int poly = 0; poly < 3; poly++) { + typename Engine::FrElement *v = abc[poly]; + + LOG_TRACE("Start iFFT (DIF)"); + fft->ifftDIFNatToRev(v, domainSize); + + LOG_TRACE("Start coset shift"); + threadPool.parallelFor(0, domainSize, [&] (int64_t begin, int64_t end, uint64_t idThread) { + for (u_int64_t i=begin; ifftDITRevToNat(v, domainSize); + } LOG_TRACE("Start ABC"); threadPool.parallelFor(0, domainSize, [&] (int64_t begin, int64_t end, uint64_t idThread) { diff --git a/src/groth16.hpp b/src/groth16.hpp index c113491..334daf7 100644 --- a/src/groth16.hpp +++ b/src/groth16.hpp @@ -69,6 +69,7 @@ namespace Groth16 { typename Engine::G1PointAffine *pointsH; FFT *fft; + typename Engine::FrElement *cosetBR; public: Prover( Engine &_E, @@ -104,12 +105,47 @@ namespace Groth16 { pointsB2(_pointsB2), pointsC(_pointsC), pointsH(_pointsH) - { - fft = new FFT(domainSize*2); + { + // The transforms only ever run at domainSize; the finer omega_2n + // needed for the coset shift is derived directly instead of + // paying for a roots table twice the transform size. + fft = new FFT(domainSize); + + // Coset shift ω_2n^BR(i) with the iFFT's 1/n folded in, indexed + // in bit-reversed order for the permutation-free h pipeline. + // Built once; reused by every proof of this prover. + cosetBR = new typename Engine::FrElement[domainSize]; + + u_int32_t domainPow = fft->log2(domainSize); + + typename Engine::FrElement w2n; + fft->higherRootOfUnity(w2n, 1); + + // sequential powers of omega_2n by chunked scan, then permute + // into bit-reversed order with 1/n folded in + std::unique_ptr seq( + new typename Engine::FrElement[domainSize]); + + ThreadPool::defaultPool().parallelFor(0, domainSize, [&] (int begin, int end, int numThread) { + if (begin >= end) return; + + u_int64_t k = begin; + E.fr.exp(seq[k], w2n, (uint8_t *)&k, sizeof(k)); + for (k = begin+1; k < (u_int64_t)end; k++) { + E.fr.mul(seq[k], seq[k-1], w2n); + } + }); + + ThreadPool::defaultPool().parallelFor(0, domainSize, [&] (int begin, int end, int numThread) { + for (int i=begin; inInv(domainPow), seq[BR(i, domainPow)]); + } + }); } ~Prover() { delete fft; + delete [] cosetBR; } std::unique_ptr> prove(typename Engine::FrElement *wtns); diff --git a/src/prover.cpp b/src/prover.cpp index 8221cd6..97e05aa 100644 --- a/src/prover.cpp +++ b/src/prover.cpp @@ -106,6 +106,22 @@ class Groth16Prover : zkey(zkey_buffer, zkey_size, "zkey", 1), zkeyHeader(ZKeyUtils::loadHeader(&zkey)) + { + init(); + } + + // File-path constructor: the BinFile owns its FileLoader (mmap) for the + // prover's lifetime, so the zkey sections stay mapped across every prove(). + explicit Groth16Prover(const std::string &zkey_file_path) + + : zkey(zkey_file_path, "zkey", 1), + zkeyHeader(ZKeyUtils::loadHeader(&zkey)) + { + init(); + } + +private: + void init() { if (!PrimeIsValid(zkeyHeader->rPrime)) { throw std::invalid_argument("zkey curve not supported"); @@ -130,6 +146,7 @@ class Groth16Prover ); } +public: void prove(const void *wtns_buffer, unsigned long long wtns_size, std::string &stringProof, @@ -261,22 +278,37 @@ groth16_prover_create_zkey_file( char *error_msg, unsigned long long error_msg_maxsize) { - BinFileUtils::FileLoader fileLoader; - try { - fileLoader.load(zkey_file_path); + if (prover_object == NULL) { + throw std::invalid_argument("Null prover object"); + } + + if (zkey_file_path == NULL) { + throw std::invalid_argument("Null zkey file path"); + } + + // The Groth16Prover keeps the zkey mmap'd for its whole lifetime + // (BinFile owns the FileLoader), so section pointers stay valid across + // every prove() call. + Groth16Prover *prover = new Groth16Prover(std::string(zkey_file_path)); + + *prover_object = prover; } catch (std::exception& e) { CopyError(error_msg, error_msg_maxsize, e); return PROVER_ERROR; + + } catch (std::exception *e) { + CopyError(error_msg, error_msg_maxsize, *e); + delete e; + return PROVER_ERROR; + + } catch (...) { + CopyErrorFmt(error_msg, error_msg_maxsize, "unknown error"); + return PROVER_ERROR; } - return groth16_prover_create( - prover_object, - fileLoader.dataBuffer(), - fileLoader.dataSize(), - error_msg, - error_msg_maxsize); + return PROVER_OK; } int diff --git a/src/test_msm.cpp b/src/test_msm.cpp new file mode 100644 index 0000000..aa8f468 --- /dev/null +++ b/src/test_msm.cpp @@ -0,0 +1,458 @@ +// Correctness and benchmark harness for the ffiasm MSM used by the prover. +// +// Correctness: compares multiMulByScalarMSM against the older, independent +// ParallelMultiexp implementation (multiMulByScalar) on scalar distributions +// that mimic circom witnesses (mostly 0/1 and small values) plus boundary +// cases around the 64-bit partition threshold. +// +// Usage: test_msm [bench] + +#include +#include +#include +#include +#include +#include +#include + +#include "alt_bn128.hpp" +#include "fft.hpp" + +using namespace AltBn128; + +namespace { + +std::mt19937_64 rng(0xC0FFEEULL); + +typedef AltBn128::FrElement Scalar; // RawFr::Element, 4 little-endian 64-bit limbs + +void setZero(Scalar &s) { s.v[0] = s.v[1] = s.v[2] = s.v[3] = 0; } + +void setU64(Scalar &s, uint64_t v) { setZero(s); s.v[0] = v; } + +void setRand254(Scalar &s) +{ + s.v[0] = rng(); + s.v[1] = rng(); + s.v[2] = rng(); + s.v[3] = rng() & 0x1fffffffffffffffULL; // < 2^253 < r +} + +void setFromDecimal(Scalar &s, const char *dec) +{ + Fr.fromString(s, dec); + Fr.fromMontgomery(s, s); +} + +// Distinct affine points: pool[i] = (i+1)*G, tiled over the output array. +template +void fillBases(CurveT &C, PointAffineT *bases, int n, int poolSize) +{ + std::vector pool(poolSize); + + C.copy(pool[0], C.oneAffine()); + for (int i = 1; i < poolSize; i++) { + typename CurveT::Point t; + C.add(t, pool[i-1], C.oneAffine()); + C.copy(pool[i], t); + } + for (int i = 0; i < n; i++) { + bases[i] = pool[i % poolSize]; + } +} + +enum Dist { DIST_RAND254, DIST_WITNESS, DIST_IDEN3, DIST_BINARY, DIST_U64, DIST_ZEROS, DIST_ONES }; + +const char *distName(Dist d) +{ + switch (d) { + case DIST_RAND254: return "rand254"; + case DIST_WITNESS: return "witness"; + case DIST_IDEN3: return "iden3"; + case DIST_BINARY: return "binary"; + case DIST_U64: return "u64"; + case DIST_ZEROS: return "zeros"; + case DIST_ONES: return "ones"; + } + return "?"; +} + +void fillScalars(Scalar *scalars, int n, Dist d) +{ + for (int i = 0; i < n; i++) { + switch (d) { + case DIST_RAND254: + setRand254(scalars[i]); + break; + case DIST_WITNESS: { + // bit-decomposition-heavy circuit: mostly 0/1, a tail of full-width + uint64_t p = rng() % 100; + if (p < 30) setZero(scalars[i]); + else if (p < 60) setU64(scalars[i], 1); + else if (p < 75) setU64(scalars[i], rng() & 0xffff); + else if (p < 85) setU64(scalars[i], rng()); + else setRand254(scalars[i]); + break; + } + case DIST_IDEN3: { + // measured on the credentialAtomicQuery* witnesses: hash-heavy, + // so most wires are uniform field elements + uint64_t p = rng() % 100; + if (p < 15) setZero(scalars[i]); + else if (p < 20) setU64(scalars[i], 1); + else if (p < 21) setU64(scalars[i], rng()); + else setRand254(scalars[i]); + break; + } + case DIST_BINARY: + setU64(scalars[i], rng() & 1); + break; + case DIST_U64: + setU64(scalars[i], rng()); + break; + case DIST_ZEROS: + setZero(scalars[i]); + break; + case DIST_ONES: + setU64(scalars[i], 1); + break; + } + } +} + +int failures = 0; + +void checkG1(const char *name, G1PointAffine *bases, Scalar *scalars, int n) +{ + G1Point res, ref; + + G1.multiMulByScalarMSM(res, bases, (uint8_t *)scalars, sizeof(Scalar), n); + G1.multiMulByScalar(ref, bases, (uint8_t *)scalars, sizeof(Scalar), n); + + if (!G1.eq(res, ref)) { + printf("FAIL G1 %-16s n=%d\n", name, n); + failures++; + } else { + printf("ok G1 %-16s n=%d\n", name, n); + } +} + +void checkG2(const char *name, G2PointAffine *bases, Scalar *scalars, int n) +{ + G2Point res, ref; + + G2.multiMulByScalarMSM(res, bases, (uint8_t *)scalars, sizeof(Scalar), n); + G2.multiMulByScalar(ref, bases, (uint8_t *)scalars, sizeof(Scalar), n); + + if (!G2.eq(res, ref)) { + printf("FAIL G2 %-16s n=%d\n", name, n); + failures++; + } else { + printf("ok G2 %-16s n=%d\n", name, n); + } +} + +void correctness() +{ + const int n = 8192; + std::vector bases(n); + std::vector scalars(n); + + fillBases(G1, bases.data(), n, 1024); + + const Dist dists[] = { DIST_RAND254, DIST_WITNESS, DIST_IDEN3, DIST_BINARY, + DIST_U64, DIST_ZEROS, DIST_ONES }; + + for (Dist d : dists) { + fillScalars(scalars.data(), n, d); + checkG1(distName(d), bases.data(), scalars.data(), n); + } + + // Boundary values around the small/big partition threshold and the + // top of the field, several of each so no class has a single element. + { + const char *rMinus1 = "21888242871839275222246405745257275088548364400416034343698204186575808495616"; + const char *rMinus2 = "21888242871839275222246405745257275088548364400416034343698204186575808495615"; + int i = 0; + + setU64(scalars[i++], 0); + setU64(scalars[i++], 1); + setU64(scalars[i++], 2); + setU64(scalars[i++], 3); + setU64(scalars[i++], 0x7fffffffffffffffULL); // 2^63-1 + setU64(scalars[i++], 0x8000000000000000ULL); // 2^63 + setU64(scalars[i++], 0xffffffffffffffffULL); // 2^64-1 (max small) + setZero(scalars[i]); scalars[i].v[1] = 1; i++; // 2^64 (min big) + setZero(scalars[i]); scalars[i].v[1] = 1; scalars[i].v[0] = 1; i++; // 2^64+1 + setZero(scalars[i]); scalars[i].v[3] = 0x2000000000000000ULL; i++; // 2^253 + setFromDecimal(scalars[i++], rMinus1); + setFromDecimal(scalars[i++], rMinus2); + setU64(scalars[i++], 0xffffffffffffffffULL); + setU64(scalars[i++], 1); + setU64(scalars[i++], 0); + setU64(scalars[i++], 0xffff); + + checkG1("boundary", bases.data(), scalars.data(), i); + + // very small n + checkG1("n2", bases.data(), scalars.data(), 2); + checkG1("n3", bases.data(), scalars.data(), 3); + } + + // ~10% infinity bases (real zkeys contain them in pointsA/B) + { + std::vector basesInf(bases); + + for (int i = 0; i < n; i += 10) { + G1.copy(basesInf[i], G1.zeroAffine()); + } + fillScalars(scalars.data(), n, DIST_IDEN3); + checkG1("infinity", basesInf.data(), scalars.data(), n); + fillScalars(scalars.data(), n, DIST_RAND254); + checkG1("infinity254", basesInf.data(), scalars.data(), n); + } + + // tiny base pool: many equal points land in the same bucket, forcing + // the doubling/cancellation paths of the batch-affine accumulator + { + std::vector basesDup(n); + + fillBases(G1, basesDup.data(), n, 4); + fillScalars(scalars.data(), n, DIST_RAND254); + checkG1("dup4-rand", basesDup.data(), scalars.data(), n); + fillScalars(scalars.data(), n, DIST_U64); + checkG1("dup4-u64", basesDup.data(), scalars.data(), n); + } + + // n=1 against plain scalar mul + { + G1Point res, ref; + setRand254(scalars[0]); + G1.multiMulByScalarMSM(res, bases.data(), (uint8_t *)scalars.data(), sizeof(Scalar), 1); + G1.mulByScalar(ref, bases[0], (uint8_t *)scalars.data(), sizeof(Scalar)); + if (!G1.eq(res, ref)) { printf("FAIL G1 n1\n"); failures++; } + else printf("ok G1 n1\n"); + } + + // n=0 + { + G1Point res; + G1.multiMulByScalarMSM(res, bases.data(), (uint8_t *)scalars.data(), sizeof(Scalar), 0); + if (!G1.isZero(res)) { printf("FAIL G1 n0\n"); failures++; } + else printf("ok G1 n0\n"); + } + + // G2 + { + const int n2 = 4096; + std::vector bases2(n2); + std::vector scalars2(n2); + + fillBases(G2, bases2.data(), n2, 512); + + for (Dist d : dists) { + fillScalars(scalars2.data(), n2, d); + checkG2(distName(d), bases2.data(), scalars2.data(), n2); + } + } +} + +// The permutation-free DIF -> bitrev-indexed coset shift -> DIT pipeline +// must produce exactly the ifft -> shift -> fft coset evaluations the +// prover previously computed. +void fftEquivalence() +{ + const int n = 1024; + + FFT F(n*2); + const u_int32_t pow = F.log2(n); + + std::vector x(n), y(n); + + for (int i = 0; i < n; i++) { + setRand254(x[i]); + y[i] = x[i]; + } + + // classic path + F.ifft(x.data(), n); + for (int i = 0; i < n; i++) { + Fr.mul(x[i], x[i], F.root(pow+1, i)); + } + F.fft(x.data(), n); + + // permutation-free path (1/n deferred into the pointwise pass) + F.ifftDIFNatToRev(y.data(), n); + for (int i = 0; i < n; i++) { + Fr.mul(y[i], y[i], F.nInv(pow)); + Fr.mul(y[i], y[i], F.root(pow+1, BR(i, pow))); + } + F.fftDITRevToNat(y.data(), n); + + for (int i = 0; i < n; i++) { + if (!Fr.eq(x[i], y[i])) { + printf("FAIL fft pipeline equivalence at %d\n", i); + failures++; + return; + } + } + printf("ok fft pipeline equivalence n=%d\n", n); +} + +// Simulates the prover's witness phase: A, B1 (G1, shared scalars), +// B2 (G2, shared scalars), C (G1, scalar suffix) batched into one task +// region, checked against independent single-MSM runs. +void batchCorrectness() +{ + const int n = 8192; + const int nPublic = 100; + + std::vector basesA(n), basesB1(n), basesC(n - nPublic - 1); + std::vector basesB2(n); + std::vector scalars(n); + + fillBases(G1, basesA.data(), n, 512); + fillBases(G1, basesB1.data(), n, 300); + fillBases(G1, basesC.data(), n - nPublic - 1, 700); + fillBases(G2, basesB2.data(), n, 256); + fillScalars(scalars.data(), n, DIST_IDEN3); + + ThreadPool &pool = ThreadPool::defaultPool(); + const uint64_t nThreads = pool.getThreadCount(); + const uint64_t share = nThreads >= 4 ? nThreads/4 : 1; + + MSM, RawFq> msmA(G1), msmB1(G1), msmC(G1); + MSM>, F2Field> msmB2(G2); + + msmA.prepare(basesA.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n, share); + msmB1.prepare(basesB1.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n, share); + msmB2.prepare(basesB2.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n, share); + msmC.prepare(basesC.data(), (uint8_t *)(scalars.data() + nPublic + 1), sizeof(Scalar), n - nPublic - 1, share); + + const uint64_t g1Bytes = std::max(msmA.arenaBytesPerThread(), + std::max(msmB1.arenaBytesPerThread(), msmC.arenaBytesPerThread())); + const uint64_t g2Bytes = msmB2.arenaBytesPerThread(); + + std::vector g1Arena(nThreads * g1Bytes); + std::vector g2Arena(nThreads * g2Bytes); + + std::vector> tasks; + msmB2.collectTasks(tasks, g2Arena.data(), g2Bytes); + msmA.collectTasks(tasks, g1Arena.data(), g1Bytes); + msmB1.collectTasks(tasks, g1Arena.data(), g1Bytes); + msmC.collectTasks(tasks, g1Arena.data(), g1Bytes); + + pool.parallelFor(0, tasks.size(), [&] (int begin, int end, int numThread) { + for (int t = begin; t < end; t++) { + tasks[t]((uint64_t)numThread); + } + }); + + G1Point rA, rB1, rC, refA, refB1, refC; + G2Point rB2, refB2; + + msmA.finish(rA); + msmB1.finish(rB1); + msmB2.finish(rB2); + msmC.finish(rC); + + G1.multiMulByScalarMSM(refA, basesA.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n); + G1.multiMulByScalarMSM(refB1, basesB1.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n); + G2.multiMulByScalarMSM(refB2, basesB2.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n); + G1.multiMulByScalarMSM(refC, basesC.data(), (uint8_t *)(scalars.data() + nPublic + 1), sizeof(Scalar), n - nPublic - 1); + + bool ok = G1.eq(rA, refA) && G1.eq(rB1, refB1) && G2.eq(rB2, refB2) && G1.eq(rC, refC); + + if (!ok) { + printf("FAIL batch A/B1/B2/C\n"); + failures++; + } else { + printf("ok batch A/B1/B2/C\n"); + } +} + +double medianMs(std::vector &v) +{ + std::sort(v.begin(), v.end()); + return v[v.size()/2]; +} + +void bench() +{ + const int n = 1 << 20; + const int reps = 5; + + std::vector bases(n); + std::vector scalars(n); + + fillBases(G1, bases.data(), n, 4096); + + printf("\nG1 MSM, n=%d (median of %d)\n", n, reps); + + const Dist dists[] = { DIST_RAND254, DIST_WITNESS, DIST_IDEN3, DIST_BINARY, DIST_U64 }; + + for (Dist d : dists) { + fillScalars(scalars.data(), n, d); + + G1Point res; + G1.multiMulByScalarMSM(res, bases.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n); // warmup + + std::vector times; + for (int r = 0; r < reps; r++) { + auto t0 = std::chrono::steady_clock::now(); + G1.multiMulByScalarMSM(res, bases.data(), (uint8_t *)scalars.data(), sizeof(Scalar), n); + auto t1 = std::chrono::steady_clock::now(); + times.push_back(std::chrono::duration(t1 - t0).count()); + } + printf(" %-8s %8.2f ms\n", distName(d), medianMs(times)); + } + + // G2, smaller n + { + const int n2 = 1 << 17; + std::vector bases2(n2); + std::vector scalars2(n2); + + fillBases(G2, bases2.data(), n2, 4096); + + printf("\nG2 MSM, n=%d (median of %d)\n", n2, reps); + + const Dist dists2[] = { DIST_RAND254, DIST_WITNESS, DIST_IDEN3 }; + + for (Dist d : dists2) { + fillScalars(scalars2.data(), n2, d); + + G2Point res; + G2.multiMulByScalarMSM(res, bases2.data(), (uint8_t *)scalars2.data(), sizeof(Scalar), n2); + + std::vector times; + for (int r = 0; r < reps; r++) { + auto t0 = std::chrono::steady_clock::now(); + G2.multiMulByScalarMSM(res, bases2.data(), (uint8_t *)scalars2.data(), sizeof(Scalar), n2); + auto t1 = std::chrono::steady_clock::now(); + times.push_back(std::chrono::duration(t1 - t0).count()); + } + printf(" %-8s %8.2f ms\n", distName(d), medianMs(times)); + } + } +} + +} // namespace + +int main(int argc, char **argv) +{ + correctness(); + batchCorrectness(); + fftEquivalence(); + + if (failures) { + printf("\n%d FAILURES\n", failures); + return 1; + } + printf("\nall correctness tests passed\n"); + + if (argc > 1 && strcmp(argv[1], "bench") == 0) { + bench(); + } + return 0; +}