diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..6132915 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,10 @@ +# Enable hardware AES (ARMv8 Cryptography Extensions) in the `aes` crate on +# aarch64. Without this cfg, `aes` v0.8 falls back to its software +# (fixsliced) backend on ARM — roughly 60x slower per block on Apple +# Silicon. x86_64 AES-NI is auto-detected and needs no flag. +# +# This config covers builds *of this workspace* (tests, benches, examples). +# Downstream crates need the same cfg in their own build configuration to +# get hardware AES on ARM; see the README. +[target.'cfg(target_arch = "aarch64")'] +rustflags = ["--cfg", "aes_armv8"] diff --git a/docs/benchmarks/2026-06-13-pr3-results.md b/docs/benchmarks/2026-06-13-pr3-results.md new file mode 100644 index 0000000..3d4b1bd --- /dev/null +++ b/docs/benchmarks/2026-06-13-pr3-results.md @@ -0,0 +1,42 @@ +# Benchmark results — ORE v2 PR 3 (efficient unary encoding + hardware AES) + +Apple M1 Max, rustc 1.87.0, `cargo bench --bench oreaes128`, criterion +medians. Branches: `main` (= PR 1 vectors branch, identical code), +PR 2 (`feat/ore-v2-core-refactor`), PR 3 (`feat/ore-v2-efficient-encoding`). + +## Hardware AES (`--cfg aes_armv8`), the new reference configuration + +| Benchmark | main | PR 2 | PR 3 | PR 3 vs main | +|---|---:|---:|---:|---:| +| encrypt-8 (u64) | 39.4 µs | 32.9 µs (−15.2%) | 25.1 µs | **−36.3%** | +| encrypt-left-8 | 21.6 µs | 21.4 µs (−0.1%) | 17.8 µs | **−17.8%** | +| compare-8 | 230 ns | ~noise | 234 ns | ~noise | + +## The headline: default build on Apple Silicon + +The `aes` crate v0.8 needs `--cfg aes_armv8` to use ARMv8 hardware AES; +without it (the previous default) every AES block ran in software, ~60× +slower per block. End to end for `u64` encrypt: + +| Configuration | encrypt-8 | vs old default | +|---|---:|---:| +| main, soft AES (old default) | 381.1 µs | 1.0× | +| main, hardware AES | 39.4 µs | 9.7× | +| PR 3, hardware AES (new default) | **25.1 µs** | **15.2×** | + +## Software AES control (PR 3 vs main, both soft) + +Isolates the algorithmic changes from the AES backend: encrypt-8 −8.4%, +encrypt-left-8 −25.0%, compare/serialize/deserialize unchanged. The right +path improves less in relative terms here because software AES dominates +its cost; under hardware AES the same changes are worth −36%. + +## Where the remaining time goes (hardware AES) + +encrypt-left-8 at 17.8 µs ≈ 2.2 µs per block, almost entirely Knuth-shuffle +PRP construction (rejection-sampled PRNG, ~1.3k draws with unpredictable +branches per block). This is now the dominant cost of the Left path and +~70% of the full encrypt path, and is wire-format-frozen for this scheme. +The Bit6 scheme (PR 5) shrinks the shuffle domain 4×; the open question +about a constant-time small-domain PRP for new schemes (plan, open q. 1) +is now the highest-leverage remaining item. diff --git a/docs/plans/2026-06-12-ore-v2-architecture.md b/docs/plans/2026-06-12-ore-v2-architecture.md index 3b52318..3dbb859 100644 --- a/docs/plans/2026-06-12-ore-v2-architecture.md +++ b/docs/plans/2026-06-12-ore-v2-architecture.md @@ -174,7 +174,12 @@ number; AES remains the floor.) | Hash-LSB mask | strided bit gather from AES output blocks | shifts + `vpmovmskb` on gathered bytes | shifts + narrowing | | Compare: first-differing-block scan | 16-byte tag equality across blocks | `vpcmpeqb` + movemask, branch-free fold | `cmeq` + fold | -AES itself already uses AES-NI / ARMv8-CE via the `aes` crate — no work needed there. +AES itself uses AES-NI automatically on x86_64. **On aarch64 the `aes` crate (v0.8) +requires `--cfg aes_armv8` in RUSTFLAGS to use the ARMv8 Cryptography Extensions; +without it the software backend runs ~60× slower per block** (measured on M1 Max). +The workspace now sets this in `.cargo/config.toml` (PR 3) and the README documents +it for downstream builds — by far the largest single performance lever in this +program for ARM users. **Mechanism:** @@ -313,6 +318,14 @@ rules out both. Whichever wins, **internal crypto review happens before PR 6 is written, not after.** The fixed-N schemes never use the accumulator, so review risk doesn't block goals 1–3. +> **Benchmark gate result (2026-06-13, Apple M1 Max, hardware AES):** key expansion +> costs ~172 ns ≈ 160 batched block encryptions (no key-schedule instruction on +> aarch64), making Candidate A's per-block overhead **~84%** at Bit6 width (~35% at +> Bit8) — roughly 6× over the threshold. The CMAC/XE-style control (one extra +> encryption per block) measured ~0%. **The decision rule selects Candidate B (CMAC +> with cached prefix state).** Spike code preserved at `/tmp/ore-keyexp-spike` +> (re-runnable; numbers recorded here are the durable record). + Cost shape is preserved in all candidates: the prefix is absorbed once per block, and the `DOMAIN` RO keys per block remain a single batched `encrypt_all` under one cipher instance, so right-encryption throughput stays comparable to the packed scheme. diff --git a/packages/ore-rs/README.md b/packages/ore-rs/README.md index d7851c7..db4a1e5 100644 --- a/packages/ore-rs/README.md +++ b/packages/ore-rs/README.md @@ -47,18 +47,24 @@ Example benchmark results below: ![Benchmark](https://user-images.githubusercontent.com/12306/145158987-9846bd94-24c7-4163-b655-1cb3ad686dd9.png) -## ARMv8 and M1 Support +## ARMv8 and Apple Silicon Support -ARMv8 and M1 Macs work out of the box but will default to AES in software which is around 4x slower than AES-NI (at least on the test machine using an Intel i7 8700K). +ARMv8 and Apple Silicon Macs work out of the box but will default to AES in +software, which is dramatically slower than the hardware backend (~60x per +AES block on an M1 Max). To use the ARMv8 Cryptography Extensions, enable +the `aes` crate's cfg flag — stable Rust is fine (1.61+): -To take advantage of hardware AES using NEON Intrinsics on ARM, you need to use Rust nightly. - -``` -asdf install rust nightly -asdf local rust nightly -cargo +nightly bench +```toml +# .cargo/config.toml in your project +[target.'cfg(target_arch = "aarch64")'] +rustflags = ["--cfg", "aes_armv8"] ``` +This repository's own workspace sets this already (tests, benches and +examples get hardware AES); the flag is per-final-binary, so downstream +projects need it in their own build configuration. x86_64 AES-NI is +auto-detected and needs no flag. + ## Security Warning This package is a pre-1.0 release and has not yet had significant scrutiny (although ORE generally has been quite well studied). diff --git a/packages/ore-rs/src/primitives.rs b/packages/ore-rs/src/primitives.rs index d304618..b771829 100644 --- a/packages/ore-rs/src/primitives.rs +++ b/packages/ore-rs/src/primitives.rs @@ -10,6 +10,36 @@ pub type PrfKey = GenericArray; pub type HashKey = GenericArray; pub const NONCE_SIZE: usize = 16; +/// Pack one bit per element of `src` into `out`, LSB-first, eight elements per +/// byte: the bit derived from `src[j]` lands in `out[j / 8]` at position +/// `j % 8` — the bit order used by `RightBlock32::set_bit`. `bit` extracts the +/// 0/1 value of each element; `merge` folds each packed byte into the existing +/// `out` byte (assign for a fresh buffer, XOR to overlay). This is the single +/// home of the right-ciphertext bit-packing convention, shared by +/// [`Hash::hash_all_into`] and [`Prp::indicator_mask_xor`]. +/// +/// Panics unless `out.len() * 8 == src.len()`, which also guarantees +/// `chunks_exact(8)` consumes `src` with no dropped remainder. +pub(crate) fn pack_bits_lsb_first( + out: &mut [u8], + src: &[T], + bit: impl Fn(&T) -> u8, + merge: impl Fn(&mut u8, u8), +) { + assert_eq!( + out.len() * 8, + src.len(), + "pack_bits_lsb_first: out.len()*8 must equal src.len()" + ); + for (slot, chunk) in out.iter_mut().zip(src.chunks_exact(8)) { + let mut byte = 0u8; + for (i, elem) in chunk.iter().enumerate() { + byte |= (bit(elem) & 1) << i; + } + merge(slot, byte); + } +} + pub trait Prf { fn new(key: &PrfKey) -> Self; fn encrypt_all(&self, data: &mut [AesBlock]); @@ -18,7 +48,10 @@ pub trait Prf { pub trait Hash { fn new(key: &HashKey) -> Self; fn hash(&self, data: &[u8]) -> u8; - fn hash_all(&self, input: &mut [AesBlock]) -> Vec; + /// Hash every block in `input` (in place, trashing it) and pack the + /// 1-bit outputs LSB-first into `out`: bit `j` of `out` is the hash of + /// `input[j]`. `out.len() * 8` must equal `input.len()`. + fn hash_all_into(&self, input: &mut [AesBlock], out: &mut [u8]); } #[derive(Debug, Error)] @@ -29,5 +62,19 @@ pub type PrpResult = Result; pub trait Prp: Sized { fn new(key: &[u8]) -> PrpResult; fn permute(&self, data: T) -> PrpResult; + /// Inverse of [`Self::permute`]. The encrypt path uses the bulk + /// [`Self::indicator_mask_xor`] instead; this remains the per-value + /// reference (used by the mask equivalence tests). + #[allow(dead_code)] fn invert(&self, data: T) -> PrpResult; + + /// XOR the indicator mask for `data` into `out`: bit `j` of the mask is + /// `1` iff `invert(j) > data`. Bit order matches `RightBlock32::set_bit` + /// (LSB-first within each byte). `out.len() * 8` must equal the + /// permutation domain. + /// + /// This is the bulk form of the per-`j` `invert`-and-compare loop the + /// right-ciphertext encoder needs; implementations walk their inverse + /// table linearly instead of doing `DOMAIN` indexed lookups. + fn indicator_mask_xor(&self, data: T, out: &mut [u8]); } diff --git a/packages/ore-rs/src/primitives/hash.rs b/packages/ore-rs/src/primitives/hash.rs index ae7c9d5..d78b506 100644 --- a/packages/ore-rs/src/primitives/hash.rs +++ b/packages/ore-rs/src/primitives/hash.rs @@ -31,17 +31,16 @@ impl Hash for Aes128Z2Hash { output[0] & 1u8 } - // TODO: this mutates - see how much a copy effects performance (clone_from_slice) - fn hash_all(&self, data: &mut [AesBlock]) -> Vec { + fn hash_all_into(&self, data: &mut [AesBlock], out: &mut [u8]) { self.cipher.encrypt_blocks(data); - - let mut vec = Vec::with_capacity(data.len()); - for &mut block in data { - // Output is Z2 (1-bit) - vec.push(block[0] & 1u8); - } - - vec + // Pack the Z2 (1-bit) outputs LSB-first, eight blocks per byte (assign + // into a fresh block); see `pack_bits_lsb_first` for the convention. + crate::primitives::pack_bits_lsb_first( + out, + data, + |block| block[0], + |slot, byte| *slot = byte, + ); } } diff --git a/packages/ore-rs/src/primitives/prp.rs b/packages/ore-rs/src/primitives/prp.rs index 222b8a4..3881ee3 100644 --- a/packages/ore-rs/src/primitives/prp.rs +++ b/packages/ore-rs/src/primitives/prp.rs @@ -37,7 +37,12 @@ impl Prp for KnuthShufflePRP { perm.permutation[i] = i as u8; } - (0..=255usize).rev().for_each(|i| { + // Iterations stop at i = 1: the i = 0 step always degenerates to + // `swap(0, 0)` after drawing rejection-sampled bytes until one is + // zero (expected 256 draws — a full PRNG regeneration), and the RNG + // is dropped right after this loop, so skipping it consumes no + // observable state and yields a byte-identical permutation. + (1..=255usize).rev().for_each(|i| { let j = rng.gen_range(i as u8); perm.permutation.swap(i, j as usize); }); @@ -75,6 +80,20 @@ impl Prp for KnuthShufflePRP { None => Err(PrpError), } } + + fn indicator_mask_xor(&self, data: u8, out: &mut [u8]) { + // `invert(j)` is `self.permutation[j]` (see `invert` above), so the + // mask is one linear pass over the table: a bytewise `> data` compare, + // XORed over the hash bits. Branch-free with a fixed trip count — the + // scalar form of a SIMD compare-and-movemask (v2 plan §3). Bit order + // is shared with `hash_all_into` via `pack_bits_lsb_first`. + crate::primitives::pack_bits_lsb_first( + out, + &self.permutation, + |&p| u8::from(p > data), + |slot, byte| *slot ^= byte, + ); + } } #[cfg(test)] @@ -87,6 +106,29 @@ mod tests { Prp::new(&key) } + quickcheck! { + /// The bulk indicator mask must agree with the naive per-bit + /// reference: bit j = (invert(j) > x). This is the regression guard + /// for `indicator_mask_xor` (and, later, its SIMD overrides). + fn indicator_mask_matches_reference(key: Vec, x: u8) -> quickcheck::TestResult { + if key.len() < 16 { + return quickcheck::TestResult::discard(); + } + let prp: KnuthShufflePRP = Prp::new(&key[0..16]).unwrap(); + + let mut mask = [0u8; 32]; + prp.indicator_mask_xor(x, &mut mask); + + let mut reference = [0u8; 32]; + for j in 0..=255u8 { + let indicator = u8::from(prp.invert(j).unwrap() > x); + reference[(j / 8) as usize] |= indicator << (j % 8); + } + + quickcheck::TestResult::from_bool(mask == reference) + } + } + #[test] fn test_invert() -> Result<(), PrpError> { let prp = init_prp()?; diff --git a/packages/ore-rs/src/primitives/prp/prng.rs b/packages/ore-rs/src/primitives/prp/prng.rs index 31ce504..b16f8b8 100644 --- a/packages/ore-rs/src/primitives/prp/prng.rs +++ b/packages/ore-rs/src/primitives/prp/prng.rs @@ -5,8 +5,8 @@ use zeroize::{Zeroize, ZeroizeOnDrop}; pub struct Aes128Prng { cipher: Aes128, data: [GenericArray; 16], - ptr: (usize, usize), // ptr to block and byte within block - ctr: u32, // increments with each new encryption + ptr: usize, // flat index into the 256 buffered bytes (block = ptr >> 4) + ctr: u32, // increments with each new encryption } impl Zeroize for Aes128Prng { @@ -15,8 +15,7 @@ impl Zeroize for Aes128Prng { d.as_mut_slice().zeroize(); } // Also clear the keystream position/counter state (ZA-0001). - self.ptr.0.zeroize(); - self.ptr.1.zeroize(); + self.ptr.zeroize(); self.ctr.zeroize(); } } @@ -46,8 +45,9 @@ const _: fn() = || { }; /* - * To aid in performance this PRNG can only generate 256 random numbers - * before it panics. Should _only_ be used inside the PRP. + * Buffers 256 AES-CTR keystream bytes at a time and regenerates a fresh batch + * when the buffer is exhausted (see `next_byte`). Should _only_ be used inside + * the PRP. */ impl Aes128Prng { pub fn init(key: &[u8]) -> Self { @@ -57,7 +57,7 @@ impl Aes128Prng { cipher, data: Default::default(), ctr: 0, - ptr: (0, 0), + ptr: 0, }; prng.generate(); prng @@ -65,11 +65,21 @@ impl Aes128Prng { /* * Generates the next byte of the random number sequence. + * + * NOTE: when the buffer is exhausted, byte 0 of the regenerated buffer + * is skipped. This preserves the byte stream of the original + * tuple-pointer implementation (which regenerated *before* the + * post-read increment), and the byte stream is load-bearing: it + * determines the Knuth-shuffle PRP and therefore the ciphertext bytes. */ pub fn next_byte(&mut self) -> u8 { - debug_assert!(self.ptr.0 < 16 && self.ptr.1 < 16); - let value: u8 = self.data[self.ptr.0][self.ptr.1]; - self.inc_ptr(); + let value: u8 = self.data[self.ptr >> 4][self.ptr & 15]; + if self.ptr == 255 { + self.generate(); // resets ptr to 0 + self.ptr = 1; // historical skip of byte 0 (see NOTE above) + } else { + self.ptr += 1; + } value } @@ -86,7 +96,7 @@ impl Aes128Prng { } fn generate(&mut self) { - self.ptr = (0, 0); + self.ptr = 0; for i in 0..16 { // Counter self.data[i][0..4].copy_from_slice(&self.ctr.to_be_bytes()); @@ -94,19 +104,6 @@ impl Aes128Prng { } self.cipher.encrypt_blocks(&mut self.data); } - - #[inline] - fn inc_ptr(&mut self) { - if self.ptr == (15, 15) { - self.generate(); - } - if self.ptr.1 < 15 { - self.ptr.1 += 1; - } else { - self.ptr.1 = 0; - self.ptr.0 += 1; - } - } } #[cfg(test)] @@ -129,7 +126,7 @@ mod tests { for _i in 3..=255 { prg.next_byte(); } - assert_eq!((15, 15), prg.ptr); + assert_eq!(255, prg.ptr); } #[test] @@ -163,7 +160,7 @@ mod tests { prng.data.iter().all(|b| b.iter().all(|&x| x == 0)), "keystream not cleared" ); - assert_eq!(prng.ptr, (0, 0), "position not cleared"); + assert_eq!(prng.ptr, 0, "position not cleared"); assert_eq!(prng.ctr, 0, "counter not cleared"); } diff --git a/packages/ore-rs/src/scheme/bit2.rs b/packages/ore-rs/src/scheme/bit2.rs index 36c8594..6a607fb 100644 --- a/packages/ore-rs/src/scheme/bit2.rs +++ b/packages/ore-rs/src/scheme/bit2.rs @@ -46,10 +46,6 @@ pub type OreAes128ChaCha20 = OreAes128; type EncryptLeftResult = Result, N>, OreError>; type EncryptResult = Result, N>, OreError>; -fn cmp(a: u8, b: u8) -> u8 { - u8::from(a > b) -} - /// Derive the per-block PRP seeds for `x` under `prf2`: seed `n` is /// `PRF₂(x[0..n] ‖ 0…)`. The seeds are **key-equivalent material** — anyone /// holding seed `n` can rebuild that block's permutation and invert `xt[n]` @@ -80,22 +76,22 @@ fn derive_prp_seeds(prf2: &Aes128Prf, x: &PlainText) -> SeedB /// Build the right-ciphertext bitvector for one block: for every candidate /// value `j`, bit `j` is `(π⁻¹(j) > x) ⊕ h[j]`. /// -/// This is the naive per-bit form, retained in this refactor for risk -/// staging; the permutation-direct bulk form replaces it in the next PR -/// (v2 plan §2) without changing the output bytes. +/// Bulk form (v2 plan §2): the hash bits are packed straight into the +/// block's bitvector, then the PRP XORs its indicator mask over the top in +/// one linear pass of its inverse table — no per-bit `invert` lookups, no +/// heap allocation. `ro_blocks` holds the PRF₁-encrypted RO keys and is +/// trashed by the hash pass. fn encode_right_block( block: &mut W::RightBlock, prp: &W::Prp, x: u8, - hashes: &[u8], -) -> Result<(), OreError> { - debug_assert_eq!(hashes.len(), W::DOMAIN); - for (j, h) in hashes.iter().enumerate() { - let jstar = prp.invert(j as u8)?; - let indicator = cmp(jstar, x); - block.set_bit(j, indicator ^ h); - } - Ok(()) + hasher: &Aes128Z2Hash, + ro_blocks: &mut [AesBlock], +) { + debug_assert_eq!(ro_blocks.len(), W::DOMAIN); + let out = block.as_mut_bytes(); + hasher.hash_all_into(ro_blocks, out); + prp.indicator_mask_xor(x, out); } impl OreAes128 { @@ -203,8 +199,7 @@ impl OreCipher for OreAes128 { work.copy_from(&template); self.prf1.encrypt_all(work.as_mut_slice()); - let hashes = hasher.hash_all(work.as_mut_slice()); - encode_right_block::(&mut right.data[n], &prp, x[n], &hashes)?; + encode_right_block::(&mut right.data[n], &prp, x[n], &hasher, &mut work); } self.prf1.encrypt_all(&mut left.f); diff --git a/packages/ore-rs/src/scheme/bit2/block_types.rs b/packages/ore-rs/src/scheme/bit2/block_types.rs index 5f5d9b7..8502541 100644 --- a/packages/ore-rs/src/scheme/bit2/block_types.rs +++ b/packages/ore-rs/src/scheme/bit2/block_types.rs @@ -28,6 +28,13 @@ impl RightBlock32 { self.data[byte_index] |= v; } + /// The raw bitvector bytes, LSB-first within each byte — the bit order + /// used by [`Self::set_bit`] / [`Self::get_bit`]. + #[inline] + pub(crate) fn bytes_mut(&mut self) -> &mut [u8] { + &mut self.data + } + /// Read bit `bit` (in `0..256`); returns `0` or `1`. Panics in debug if /// `bit >= 256`. #[inline] diff --git a/packages/ore-rs/src/scheme/width.rs b/packages/ore-rs/src/scheme/width.rs index 40a9dc6..ee85dc4 100644 --- a/packages/ore-rs/src/scheme/width.rs +++ b/packages/ore-rs/src/scheme/width.rs @@ -50,21 +50,22 @@ impl AesBlockBuf for [AesBlock; 256] { /// Per-block bitvector operations on a Right ciphertext block, one bit per /// value in the block domain. pub trait RightBitVec { - /// Set bit `bit` to `value` (`0` or `1`). - fn set_bit(&mut self, bit: usize, value: u8); /// Read bit `bit`. (The width-generic comparator lands with the Bit6 /// scheme; the legacy comparator calls the inherent method.) #[allow(dead_code)] fn get_bit(&self, bit: usize) -> u8; + /// The raw bitvector bytes, LSB-first within each byte (bit `j` lives in + /// byte `j / 8` at position `j % 8`), for bulk mask construction. + fn as_mut_bytes(&mut self) -> &mut [u8]; } impl RightBitVec for RightBlock32 { - fn set_bit(&mut self, bit: usize, value: u8) { - RightBlock32::set_bit(self, bit, value) - } fn get_bit(&self, bit: usize) -> u8 { RightBlock32::get_bit(self, bit) } + fn as_mut_bytes(&mut self) -> &mut [u8] { + self.bytes_mut() + } } /// The number of plaintext bits consumed per ORE block, and the types that