diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 1734617..659d9f4 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -19,4 +19,70 @@ new version of the snapshot with: Prop tests essentially fuzz the parser and make sure that it doesn't panic. Failed test cases are saved and version controlled to avoid regressions. +## Benchmarks + +Performance benchmarks use [Criterion](https://bheisler.github.io/criterion.rs/). They live in +`benches/parser_benchmarks.rs` with Norg fixtures in `benches/inputs/`. + +### Running benchmarks + +```bash +# All benchmarks +cargo bench + +# Specific group +cargo bench -- parse_tree +cargo bench -- stage_4 +``` + +HTML reports are generated at `target/criterion/report/index.html`. + +### Benchmark groups + +| Group | What it measures | +|-------|-----------------| +| `parse_tree` | Full pipeline (`parse_tree()`) | +| `parse_flat` | Stages 1+2+3 (`parse()`) | +| `stage_1` | Lexer only | +| `stage_2` | Block parser (tokens → blocks) | +| `stage_3` | Inline parser (blocks → flat AST) | +| `stage_4` | Tree builder (flat AST → tree) | + +### Input fixtures + +| File | Size | Purpose | +|------|------|---------| +| `benches/inputs/small.norg` | ~30 lines | All construct types, no nesting | +| `benches/inputs/medium.norg` | ~1100 lines | Real blog post, mixed content | +| `benches/inputs/large.norg` | ~180 lines | Deep nesting stress test | + +### Comparing across changes + +Criterion supports comparing benchmark results against a saved baseline: + +```bash +# 1. Save baseline before making changes +cargo bench -- --save-baseline before + +# 2. Make your optimization changes + +# 3. Compare against the baseline +cargo bench -- --baseline before +``` + +Criterion automatically reports whether each benchmark improved, regressed, or stayed the same, +with confidence intervals and percentage deltas. + +To reset the baseline for a new comparison cycle: + +```bash +cargo bench -- --save-baseline main +``` + +### Bench profile + +`Cargo.toml` includes a `[profile.bench]` section with `lto = true` and `codegen-units = 1` for +more consistent and representative timings. This only affects the benchmark target — debug builds +and regular tests remain fast to compile. + diff --git a/Cargo.lock b/Cargo.lock index f5ee7fd..3dad65d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -11,7 +11,25 @@ dependencies = [ "cfg-if", "once_cell", "version_check", - "zerocopy", + "zerocopy 0.7.34", +] + +[[package]] +name = "aho-corasick" +version = "1.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" +dependencies = [ + "memchr", +] + +[[package]] +name = "alloca" +version = "0.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", ] [[package]] @@ -20,6 +38,18 @@ version = "0.2.18" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c6cb57a04249c6480766f7f7cef5467412af1490f8d1e243141daddada3264f" +[[package]] +name = "anes" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" + +[[package]] +name = "anstyle" +version = "1.0.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "940b3a0ca603d1eade50a4846a2afffd5ef57a9feac2c0e2ec2e14f9ead76000" + [[package]] name = "autocfg" version = "1.3.0" @@ -47,6 +77,18 @@ version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf4b9d6a944f767f8e5e0db018570623c85f3d925ac718db4e06d0187adb21c1" +[[package]] +name = "bumpalo" +version = "3.20.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "72f5acc6cb2ba439de613abc23857ec3d78374d8ed5ac84e9d11336e87da8649" + +[[package]] +name = "cast" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" + [[package]] name = "cc" version = "1.0.97" @@ -69,6 +111,58 @@ dependencies = [ "stacker", ] +[[package]] +name = "ciborium" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42e69ffd6f0917f5c029256a24d0161db17cea3997d185db0d35926308770f0e" +dependencies = [ + "ciborium-io", + "ciborium-ll", + "serde", +] + +[[package]] +name = "ciborium-io" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05afea1e0a06c9be33d539b876f1ce3692f4afea2cb41f740e7743225ed1c757" + +[[package]] +name = "ciborium-ll" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57663b653d948a338bfb3eeba9bb2fd5fcfaecb9e199e87e1eda4d9e8b240fd9" +dependencies = [ + "ciborium-io", + "half", +] + +[[package]] +name = "clap" +version = "4.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1ddb117e43bbf7dacf0a4190fef4d345b9bad68dfc649cb349e7d17d28428e51" +dependencies = [ + "clap_builder", +] + +[[package]] +name = "clap_builder" +version = "4.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "714a53001bf66416adb0e2ef5ac857140e7dc3a0c48fb28b2f10762fc4b5069f" +dependencies = [ + "anstyle", + "clap_lex", +] + +[[package]] +name = "clap_lex" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c8d4a3bb8b1e0c1050499d1815f5ab16d04f0959b233085fb31653fbfc9d98f9" + [[package]] name = "console" version = "0.15.8" @@ -81,6 +175,72 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "criterion" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "950046b2aa2492f9a536f5f4f9a3de7b9e2476e575e05bd6c333371add4d98f3" +dependencies = [ + "alloca", + "anes", + "cast", + "ciborium", + "clap", + "criterion-plot", + "itertools", + "num-traits", + "oorandom", + "page_size", + "plotters", + "rayon", + "regex", + "serde", + "serde_json", + "tinytemplate", + "walkdir", +] + +[[package]] +name = "criterion-plot" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d8d80a2f4f5b554395e47b5d8305bc3d27813bacb73493eb1001e8f76dae29ea" +dependencies = [ + "cast", + "itertools", +] + +[[package]] +name = "crossbeam-deque" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9dd111b7b7f7d55b72c0a6ae361660ee5853c9af73f70c3c2ef6858b950e2e51" +dependencies = [ + "crossbeam-epoch", + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-epoch" +version = "0.9.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" + +[[package]] +name = "crunchy" +version = "0.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" + [[package]] name = "either" version = "1.11.0" @@ -115,6 +275,30 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "getrandom" version = "0.2.15" @@ -126,6 +310,17 @@ dependencies = [ "wasi", ] +[[package]] +name = "half" +version = "2.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" +dependencies = [ + "cfg-if", + "crunchy", + "zerocopy 0.8.27", +] + [[package]] name = "hashbrown" version = "0.14.5" @@ -158,6 +353,23 @@ dependencies = [ "either", ] +[[package]] +name = "itoa" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f42a60cbdf9a97f5d2305f08a87dc4e09308d1276d28c869c684d7777685682" + +[[package]] +name = "js-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "03d04c30968dffe80775bd4d7fb676131cd04a1fb46d2686dbffbaec2d9dfd31" +dependencies = [ + "cfg-if", + "futures-util", + "wasm-bindgen", +] + [[package]] name = "lazy_static" version = "1.4.0" @@ -188,6 +400,12 @@ version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78b3ae25bc7c8c38cec158d1f2757ee79e9b3740fbc7ccf0e59e4b08d793fa89" +[[package]] +name = "memchr" +version = "2.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88904434abc2901f197fe8cc55f0445e7ded921dba5911dad2e2b39b48e663c4" + [[package]] name = "num-traits" version = "0.2.19" @@ -204,6 +422,56 @@ version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3fdb12b2476b595f9358c5161aa467c2438859caa136dec86c26fdd2efe17b92" +[[package]] +name = "oorandom" +version = "11.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" + +[[package]] +name = "page_size" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + +[[package]] +name = "plotters" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5aeb6f403d7a4911efb1e33402027fc44f29b5bf6def3effcc22d7bb75f2b747" +dependencies = [ + "num-traits", + "plotters-backend", + "plotters-svg", + "wasm-bindgen", + "web-sys", +] + +[[package]] +name = "plotters-backend" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df42e13c12958a16b3f7f4386b9ab1f3e7933914ecea48da7139435263a4172a" + +[[package]] +name = "plotters-svg" +version = "0.3.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51bae2ac328883f7acdfea3d66a7c35751187f870bc81f94563733a154d7a670" +dependencies = [ + "plotters-backend", +] + [[package]] name = "ppv-lite86" version = "0.2.17" @@ -302,6 +570,49 @@ dependencies = [ "rand_core", ] +[[package]] +name = "rayon" +version = "1.12.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fb39b166781f92d482534ef4b4b1b2568f42613b53e5b6c160e24cfbfa30926d" +dependencies = [ + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" +dependencies = [ + "crossbeam-deque", + "crossbeam-utils", +] + +[[package]] +name = "regex" +version = "1.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4219d74c6b67a3654a9fbebc4b419e22126d13d2f3c4a07ee0cb61ff79a79619" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "38caf58cc5ef2fed281f89292ef23f6365465ed9a41b7a7754eb4e26496c92df" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + [[package]] name = "regex-syntax" version = "0.8.4" @@ -313,6 +624,7 @@ name = "rust-norg" version = "0.1.0" dependencies = [ "chumsky", + "criterion", "insta", "itertools", "proptest", @@ -334,6 +646,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "rustversion" +version = "1.0.22" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" + [[package]] name = "rusty-fork" version = "0.3.0" @@ -346,6 +664,21 @@ dependencies = [ "wait-timeout", ] +[[package]] +name = "ryu" +version = "1.0.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" + +[[package]] +name = "same-file" +version = "1.0.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93fc1dc3aaa9bfed95e02e6eadabb4baf7e3078b0bd1b4d7b6b0b68378900502" +dependencies = [ + "winapi-util", +] + [[package]] name = "serde" version = "1.0.203" @@ -366,12 +699,30 @@ dependencies = [ "syn", ] +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "itoa", + "memchr", + "ryu", + "serde", +] + [[package]] name = "similar" version = "2.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "fa42c91313f1d05da9b26f267f931cf178d4aba455b4c4622dd7355eb80c6640" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "smawk" version = "0.3.2" @@ -425,6 +776,16 @@ dependencies = [ "unicode-width", ] +[[package]] +name = "tinytemplate" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "be4d6b5f19ff7664e8c98d03e2139cb510db9b0a60b55f8e8709b689d939b6bc" +dependencies = [ + "serde", + "serde_json", +] + [[package]] name = "unarray" version = "0.1.4" @@ -470,12 +831,77 @@ dependencies = [ "libc", ] +[[package]] +name = "walkdir" +version = "2.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "29790946404f91d9c5d06f9874efddea1dc06c5efe94541a7d6863108e3a5e4b" +dependencies = [ + "same-file", + "winapi-util", +] + [[package]] name = "wasi" version = "0.11.0+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +[[package]] +name = "wasm-bindgen" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ddb3f79143bced6de84270411622a2699cee572fc0875aeaf1e7867cf9fca1a" +dependencies = [ + "cfg-if", + "once_cell", + "rustversion", + "wasm-bindgen-macro", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-macro" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4e21a184b13fb19e157296e2c46056aec9092264fab83e4ba59e68c61b323c3d" +dependencies = [ + "quote", + "wasm-bindgen-macro-support", +] + +[[package]] +name = "wasm-bindgen-macro-support" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fecefd9c35bd935a20fc3fc344b5f29138961e4f47fb03297d88f2587afb5ebd" +dependencies = [ + "bumpalo", + "proc-macro2", + "quote", + "syn", + "wasm-bindgen-shared", +] + +[[package]] +name = "wasm-bindgen-shared" +version = "0.2.125" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23939e44bb9a5d7576fa2b563dc2e136628f1224e88a8deed09e04858b77871f" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "web-sys" +version = "0.3.102" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a6430a72df5eb332242960fe84b3002a241163998241eb596d4f739b9757061d" +dependencies = [ + "js-sys", + "wasm-bindgen", +] + [[package]] name = "winapi" version = "0.3.9" @@ -492,6 +918,15 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" +[[package]] +name = "winapi-util" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" +dependencies = [ + "windows-sys", +] + [[package]] name = "winapi-x86_64-pc-windows-gnu" version = "0.4.0" @@ -577,7 +1012,16 @@ version = "0.7.34" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ae87e3fcd617500e5d106f0380cf7b77f3c6092aae37191433159dda23cfb087" dependencies = [ - "zerocopy-derive", + "zerocopy-derive 0.7.34", +] + +[[package]] +name = "zerocopy" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0894878a5fa3edfd6da3f88c4805f4c8558e2b996227a3d864f47fe11e38282c" +dependencies = [ + "zerocopy-derive 0.8.27", ] [[package]] @@ -590,3 +1034,14 @@ dependencies = [ "quote", "syn", ] + +[[package]] +name = "zerocopy-derive" +version = "0.8.27" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "88d2b8d9c68ad2b9e4340d7832716a4d21a22a1154777ad56ea55c51a9cf3831" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] diff --git a/Cargo.toml b/Cargo.toml index 1afe57f..fed93f6 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -11,5 +11,14 @@ textwrap = "0.16.1" unicode_categories = "0.1.1" [dev-dependencies] +criterion = { version = "0.8.2", features = ["html_reports"] } insta = { version = "1.39.0", features = ["yaml"] } proptest = "1.4.0" + +[profile.bench] +lto = true +codegen-units = 1 + +[[bench]] +name = "parser_benchmarks" +harness = false diff --git a/benches/inputs/large.norg b/benches/inputs/large.norg new file mode 100644 index 0000000..7067523 --- /dev/null +++ b/benches/inputs/large.norg @@ -0,0 +1,205 @@ +* Introduction + This document is a synthetic stress test for the rust-norg parser. + It contains deep heading nesting, many paragraphs with inline markup, + nested lists, quotes, definitions, and code blocks. + The *bold text*, /italic text/, and `inline verbatim` are used heavily here. + We also test {https://example.com}[links] and [anchor] references throughout. +** Deep Nesting Section + Paragraph with *multiple* /inline/ `modifiers` and {https://rust-lang.org}[Rust link]. + This tests the stage_3 inline parser with lots of attached modifiers. +*** Level Three Heading + The /quick/ *brown* fox `jumps` over the *lazy* /dog/. Every word has markup. + {https://example.org}[Another link] and [yet-another] anchor definition. +**** Level Four Heading + - First unordered list item with *bold* text + - Second unordered list item with /italic/ text + -- Nested item with `verbatim code` + -- Another nested item with {https://docs.rs}[a link] + --- Deeply nested with *all* /the/ `markup` + - Third outer item +***** Level Five Heading + ~ First ordered list item + ~ Second ordered list item + ~~ Nested ordered with *bold* + ~~ Nested ordered with /italic/ + ~ Third ordered item +****** Level Six Heading + > Quote block with *bold* and /italic/ markup + > Second quote line with `verbatim` inline + >> Nested quote with {https://example.com}[a link] + >> More nested quote content + > Back to first level quote + This paragraph is under the deepest heading level. + It has *several* /inline/ `modifiers` to stress the parser. +***** Back to Level Five + More content at level five. + $ Definition term one + Definition content with *bold* and /italic/ markup. + $$ Ranged definition + Content inside ranged definition with `code` and {https://example.com}[links]. + $$ + ^ Footnote title + Footnote content here. + ^^ Ranged footnote + Content with *markup* inside. + ^^ + : Table cell + Table content with /italic/ and `code`. + :: Ranged table + More table content. + :: +**** Back to Level Four + @code rust + fn factorial(n: u64) -> u64 { + match n { + 0 => 1, + _ => n * factorial(n - 1), + } + } + + fn main() { + for i in 0..10 { + println!("{}! = {}", i, factorial(i)); + } + } + @end + + Another paragraph after the code block. + + |example + This is a *ranged* tag with /inline/ markup. + It spans multiple lines. + |end + + =macro.test param1 param2 + Content inside a macro tag with *bold* text. + =end +*** Back to Level Three + More paragraphs to increase document size. + Each paragraph has *some* /inline/ `markup` to stress stage_3. + The parser needs to handle {https://links.example.com}[many links] efficiently. + + +html.class important + > *Important*: This carryover tag applies a CSS class to the next element. + + +html.loading lazy + .image /assets/large-image.png A large synthetic image + + ___ + + A paragraph after a horizontal rule. + Still testing *bold*, /italic/, and `verbatim` inline parsing. +** Second Major Section + This section contains more diverse constructs. + + *** Subsection A + - List item one + - List item two + - List item three + -- Nested A + -- Nested B + --- Deep nested + - List item four + - List item five + + > Quote one + > Quote two + >> Nested quote + > Quote three + + *** Subsection B + ~ Ordered one + ~ Ordered two + ~~ Nested ordered A + ~~ Nested ordered B + ~ Ordered three + ~~~~ Very nested ordered + ~ Ordered four + + *** Subsection C + $ Term A + Definition of term A. + $ Term B + Definition of term B with *bold*. + $$ Ranged Term C + Content of ranged term C. + $$ + +*** Third Level Again + Paragraph with extensive inline markup: *bold* /italic/ `verbatim` and + a {https://norgolith.amartin.beer}[link to norgolith docs]. + Also testing syntax here. + + @code python + def fibonacci(n): + if n <= 1: + return n + return fibonacci(n - 1) + fibonacci(n - 2) + + for i in range(20): + print(f"F({i}) = {fibonacci(i)}") + @end + + |tag.name param + Some content inside a ranged standard tag. + |end + + .infirm-tag param1 param2 + +** Fourth Major Section + Carrying on with more content to reach target size. + + *** More Nesting + * First bold text in a heading paragraph. + /Italic text here too./ + + - Alpha + - Beta + - Gamma + -- Delta + -- Epsilon + --- Zeta + --- Eta + ---- Theta + + > Philosophical quote + >> Attributed to someone *important* + >>> Deep nested quote + + $ Alpha definition + With some definition content. + + : Cell one + Cell content. + + *** Even More Content + More paragraphs with *bold*, /italic/, `code`, and {https://example.com}[URL] markers. + The quick brown fox jumps over the lazy dog. + /The five boxing wizards/ jump quickly. + *How vexingly* quick daft zebras jump. + `Pack my box with` five dozen liquor jugs. + + @code javascript + const express = require('express'); + const app = express(); + + app.get('/', (req, res) => { + res.send('Hello World!'); + }); + + app.listen(3000, () => { + console.log('Server running on port 3000'); + }); + @end + +* Conclusion + This is the end of the large benchmark fixture. + It contains diverse Norg constructs to stress all four parser stages. + *Stage 1* tokenizes the raw input. + /Stage 2/ organizes tokens into blocks. + `Stage 3` resolves inline markup and attached modifiers. + {https://github.com/nvim-neorg/rust-norg}[Stage 4] builds the tree structure. + + ___ + + Final paragraph with /no/ special markup at all. \ No newline at end of file diff --git a/benches/inputs/medium.norg b/benches/inputs/medium.norg new file mode 100644 index 0000000..c0e7771 --- /dev/null +++ b/benches/inputs/medium.norg @@ -0,0 +1,1108 @@ +@document.meta +title: Norg on the Web +description: How to escape markdown and adopt norg for your blog +authors: [ + amartin +] +categories: [ + neorg +] +created: 2025-07-15T08:02:56-04:00 +updated: 2025-07-24T16:51:07-0400 +draft: false +layout: post +truncate: 159 +version: 1.1.1 +@end + +It's been three years since I made a {https://amartin.codeberg.page/posts/using-neorg-in-hugo/}[post] about a workaround for using Norg in Hugo - /who would've +thought/ I'd end up developing my own solution to the problem a few years later. + +While the solution no longer involves Hugo, I feel like it's much better now /and it has nothing to +do with me being the maintainer of the solution, I promise!/ + ++html.class note +> *Note*: If you want to skip the long introduction where I explain what led me to develop Norgolith +and its early development alongside some technical explanations, and just want the sauce, {** Freeing from Markdown (Migration-to-Norgolith)}[click here]. + +** Introduction + Before launching into a conference talk about why my tool is awesome and why you should use + it—like some salesman whose financial stability depends on you buying their product—let me + explain how this all started and why I didn't just adopt some alternative that wouldn't involve + reinventing the wheel /which I absolutely hate doing/. + + Beyond reasons such as reinventing the wheel /which fascinates us as open-source developers/, I had + reasons to start hating using markdown with all my soul (which I will later explain in a rant + post), and thus began the search for alternatives to use it everywhere. + + And so I found a solution, which although not (and still isn't) definitive and complete, was and + remains /promising/: {https://github.com/nvim-neorg/norg-specs}[norg]. As I explained in the aforementioned post, I found a way to use it so I + wouldn't have to write markdown myself, since the {https://github.com/nvim-neorg/neorg}[Neovim plugin] has a markdown exporter for + convenience. Simple—but it got the job done. + +*** The breaking point + Further down the road, I also noticed problems I had to solve myself between the conversion to + Markdown and my static site generator at the time, Hugo. Some of those problems - if not all - + required manual intervention to be solved, and each one was more tedious than the last. + Especially when I wanted nothing to do with editing after exporting my norg documents to + markdown. This frustrated me because it was impractical, and I had my blog on hiatus for just + over a year. That's when during that hiatus I realized I needed to start creating a better + solution for my problem. + +** Hello, Norgolith + +html.class warning + > *Warning*: please forgive me if you are using a mobile device to read this section, as several + of its sections contain technical flowcharts that may require zooming on small screens. There's + not much I can do about it. :( + + +html.class border-none! mt-0! +*** A tough beginning + I'll be honest, I started thinking about creating Norgolith during a period when I was still + burned out (thanks to those damn Agile development methodologies implemented in startups). I’m + a very methodical and by-the-book kind of guy when it comes to working on projects. I need to + know all the details in advance before I can get started: including the name (which I’m really + bad at, /lmao/) and, most importantly, which language I’ll use. At first I only had one thing + clear - I wanted it to be a self-contained binary, with options like Zig and (/unfortunately/) + Rust. + + In the end, I don't know if it's just another one of those ironic twists, but for a long time + Rust was my most hated language (as any good C developer would) and yet it seemed like my best + option for developing my project thanks to its vast and mature ecosystem for everything - I + wouldn't have to waste time reinventing the wheel everywhere. + + For a while I hesitated about whether to use Rust. I don't know why, but during burnout my most + hated language became my most loved one (excluding Elixir, which is the love of my life). I + understood web server theory but not how the hell to implement it (aka make my own web server), + so it was a tough start. Yet again, I knew the theory behind something but not how to actually + implement it, /how the turntables/. + +*** Lightbulb moments + Norgolith started as a simple HTTP server with {https://hyper.rs}[hyper] that spat out a hello world in the web + browser, while my mind raced a thousand times thinking about how to implement everything. Little + by little I broke down all the work into small functionality PRs, and the main one was simple to + establish in terms of priorities: parsing norg files. + + At the time, a {https://github.com/nvim-neorg/rust-norg}[Rust parser for norg] was in active development (though already in a "usable" + state), which greatly lightened my workload in my mission to get rid of markdown. All the work + on Norgolith excited me tremendously and rekindled my love for writing code, but what I + initially enjoyed most was starting to write the converter using the AST generated by the parser + library. + + The process of interpreting and converting the format is really fun to work on, since—unlike + parsers—I don’t have to worry about anything except interpreting an already-built AST and + breaking it down to generate strings. And that’s how the converter started: + + @code rust + use rust_norg::{parse, ParagraphSegmentToken, NorgASTFlat}; + + trait NorgToHtml { + fn to_html(&self) -> String; + } + + fn paragraph_to_string(segment: &[ParagraphSegmentToken]) -> String { + let mut paragraph: String = "".to_owned(); + segment.iter().for_each(|node| match node { + ParagraphSegmentToken::Text(s) => paragraph.push_str(s), + ParagraphSegmentToken::Whitespace => paragraph.push(' '), + ParagraphSegmentToken::Escape(c) => paragraph.push_str(&c.to_string()), + ParagraphSegmentToken::Special(c) => paragraph.push_str(&c.to_string()), + }); + paragraph + } + + impl NorgToHtml for NorgASTFlat { + fn to_html(&self) -> String { + match self { + NorgASTFlat::Heading {level, title, ..} => { + let mut heading_tag: String = "<".to_owned(); + match level { + 1 => { + heading_tag.push_str("h1>") + }, + _ => todo!() + } + let heading_title = paragraph_to_string(title); + heading_tag.push_str(&heading_title); + heading_tag.push_str(""); + heading_tag + }, + NorgASTFlat::Paragraph(s) => { + let mut paragraph_tag: String = "

".to_owned(); + let paragraph = paragraph_to_string(s); + paragraph_tag.push_str(¶graph); + paragraph_tag.push_str("

"); + paragraph_tag + } + _ => todo!(), + } + } + } + + fn norg_to_html(ast: &NorgASTFlat) -> String { + ast.to_html() + } + + fn main() -> Result<()> { + let norg_code = "* Hello, World\n This is a paragraph lol\n"; + println!("Norg code:\n{}\nProduced HTML:", norg_code); + let ast = parse(norg_code).unwrap(); + for node in ast { + let html = norg_to_html(&node); + println!("{}", html); + } + Ok(()) + } + @end + + Extremely simple, and yet it was the most fun I'd had in months! Of course, that wasn't enough + and it later evolved into something much bigger and more capable, but we'll save that for the + next section. Let's return to the other essential piece, the HTTP server. + + The development server is the heart of Norgolith, and it's easily the most complex piece of code + within Norgolith (with just over 1100 lines of code written {https://github.com/NTBBloodbath/norgolith/blob/7fac3a2fda9261a6d6e4968ade51248024292616/src/cmd/dev.rs}[at the time of writing this]). And + this being the fundamental core of Norgolith, its development was full of mental gaps and pauses. + After all, putting that theory was complicated even after examining the code of several SSGs + like {https://github.com/getzola/zola}[Zola] and {https://github.com/cobalt-org/cobalt.rs}[Cobalt] to try understanding the implementation process. But eventually I slowly + made my way through the implementation! Though it ended up being even much larger than the SSGs + used as reference, /woops/. + +*** HTTP Server Architecture + Since flowcharts are better than words for covering technical aspects for us nerds, I'll use + them to explain how the server works in a way that mirrors the end-user experience. + +**** Server Initialization Sequence + Initializing the dev server is a precision operation: first we hunt down `norgolith.toml` to + identify whether we are in a Norgolith site directory (no guesswork, no defaults), then we load + templates and scan content in parallel. The best part? A reload broadcast channel that links + the WebSockets server, file watcher, and HTTP handlers before any ports open. Only when all + systems are armed do we launch server and optionally fire the browser - no half-ready states, + no dangling processes. + + @embed html +
+ 
+         flowchart TD
+         A[Start Dev Server] --> B[Locate Config]
+         B --> C[Setup Server State]
+         C --> D[Load Templates]
+         C --> E[Scan Content]
+         C --> F[Create Reload Channel]
+         D --> G[Start WebSockets Server]
+         E --> H[Start HTTP Server]
+         F --> I[Start File Watcher]
+         H --> J{Open Browser?}
+
+ Everything runs asynchronously in + isolated threads, so every single piece of the process is as fast as possible +
+ @end + +**** Request Handling Flow + Every request hits a *deterministic* path router - no fuzzy matching, /no black magic involved/. + Assets get served directly from memory-mapped files (no disk reads after startup), while + content paths resolve to Norg files with extreme precision: `/about` becomes `/about.norg` or + `/about/index.norg` under the hood. The kicker? We inject LiveReload scripts during template + rendering - not through string replacement - so your HTML stays untouched. + + @embed html +
+ 
+         flowchart LR
+         A[Incoming Request] --> B{Path Type?}
+         B -->|/livereload.js| C[Inject LiveReload Script]
+         B -->|/assets/*| D[Find Asset]
+         D --> E{Exists?}
+         E -->|Yes| F[Return Asset]
+         E -->|No| G[Error 404]
+         B -->|/categories| H[Render Category Index]
+         B -->|/categories/*| I[Render Category Page]
+         B -->|Other| J[Resolve Norg Path]
+         J --> K{Found?}
+         K -->|Yes| L[Process Norg + Template]
+         K -->|No| G
+         L --> M[Inject LiveReload]
+         M --> N[Render HTML]
+
+ Everything renders from in-memory, and stays + in-memory until either the artifact no longer exists or the server is closed +
+ @end + +**** Hot-reload pipeline + Hot reloading is /physics-grade engineering/: file changes get *debounced* to avoid editor + temp-file noise (fuck off, Vim swap/undo files), then routed by type. Template updates trigger + full Tera reloads without restarting the server by using Tokio's {https://docs.rs/tokio/latest/tokio/sync/struct.RwLock.html}[RwLock] atomic swaps. Content + changes rescan content itself and metadata in the background while broadcasting reload commands through a + non-blocking WebSockets channel. The result? Updates reflect in under ~300ms without clobbering + your browser state. + + @embed html +
+ 
+         flowchart TB
+         A[File Change] --> B[Debounce Events]
+         B --> C{Change Type?}
+         C -->|Template| D[Reload Tera]
+         C -->|Asset| E[Broadcast Reload]
+         C -->|Content| F[Rescan Content/Metadata]
+         D --> G[Broadcast Reload]
+         F --> G
+         G --> H[WebSockets Server]
+         H --> I[Push Reload Command]
+         I --> J[Browser Updates]
+
+ As everything happens inside broadcast + channels, if something fucks up Norgolith can recover and continue doing its job +
+ @end + +**** WebSockets Lifecycle + WebSockets operate on a strict protocol: new connections get handshaked immediately, then join + a broadcast group. The magic's in the channel design - file watchers push reload signals + without knowing connection details, while the WebSockets server handles multiplexing. When we + say “live reload”, we mean it: no polling, no timeouts, just pure server-pushed updates the + instant files change. + + @embed html +
+ 
+         flowchart LR
+         A[New Connection] --> B[Send Hello]
+         B --> C[Add to Broadcast Group]
+         C --> D[Listen Loop]
+         D --> E{Message?}
+         E -->|Reload Signal| F[Push Reload]
+         E -->|Close| G[Remove Connection]
+         H[File Change] --> I[Broadcast Channel]
+         I --> F
+         F --> J[All Clients]
+
+ We are bundling livereload.js for live + reloading functionalities, no external CDN scripts are injected +
+ @end + +**** Shutdown Sequence + Even shutdowns are designed for zero residue: `SIGTERM` triggers graceful termination where HTTP + drains active connections, WebSockets send close frames, and the file watcher releases inotify + handles. This isn't just quitting, it's an orchestrated exit ensuring your next `lith dev` starts + in milliseconds /or even less/ without residual cache or memory leftovers. + + @embed html +
+ 
+         flowchart BT
+         A[Ctrl-C] --> B[Signal All Tasks]
+         B --> C[HTTP: Drain Connections]
+         C --> D[WebSockets: Close Channels]
+         D --> E[Watcher: Release Handles]
+         E --> F[Exit Cleanly]
+
+ Right now, Ctrl-C will return an exit code + different than 0 due to the nature of SIGTERM.
This is something that will be addressed in + the future +
+ @end + +*** Norg Under the Hood + Going back to the implementation of norg processing, one thing must be kept clear: *it's + extremely fast and without intermediaries*. No intermediate code, no external tools, no bullshit. + Pure and hard understanding of *norg as a first-class citizen*, we don't know what markdown or + pandoc is here, *only* norg. The implementation is a little complex, but we can illustrate it with + flowcharts: + + @embed html +
+ 
+        flowchart TD
+        A[Converter] --> B[HTML]
+        A-->C[Meta]
+
+ Internal structure of the converter module +
+ @end + + The beginning is simple, right? The converter module splits into two modules: one that converts + the document to HTML and another that converts the document's metadata into a TOML structure. + Each of these modules has a single task, and they're separated to avoid unnecessary complexity + in HTML transpilation. However, the internal use of each module in Norgolith becomes more + extensive, let's break it down again from the very beginning so everything makes sense at first + sight! + +**** Overall Conversion Flow + The core conversion process starts by parsing the raw Norg document into an Abstract Syntax + Tree (AST) through the `rust-norg` library - no intermediaries, no bullshit. This AST gets + processed recursively: Headings build our Table of Contents (ToC), paragraphs get decomposed + into segments, and specialized elements like verbatim blocks transform directly into HTML + primitives. The entire pipeline outputs pure HTML and structured TOML metadata in a single pass - no wasted cycles. + + @embed html +
+ 
+         flowchart LR
+         A(Start Conversion)-->B[Parse Norg Document]
+         B-->C(Convert AST to HTML)
+         C-->D[Process Carryover Tags]
+         C-->E[Handle Headings]
+         C-->F[Process Paragraphs]
+         C-->G[Handle Lists/Quotes]
+         C-->H[Process Verbatim Blocks]
+         C-->I[Handle Links/Anchors]
+         E-->J[Generate ToC TOML]
+         J-->K[Return HTML & ToC]
+
+ Conversion flow in a nutshell +
+ @end + + It doesn't look so simple anymore, right? Let's break it down into many more pieces to make it + easier to process - I also struggled at first while designing it. + +**** Content Processing Pipeline + The entire conversion pipeline is a straight-shot rocket: Norg files gets parsed to ASTs, then + fork into dual processing streams in parallel. One stream handles pure HTML transpilation, the + other extract and /validates/ metadata against schemas, if any schema is declared into the site + configuration file. These streams converge in Tera templates where validated metadata meets + rendered HTML. No JSON intermediaries anywhere, just pure, efficient transformation. + + @embed html +
+ 
+         flowchart LR
+         A[Norg File] --> B[Parser]
+         B --> C[AST]
+         C --> D[HTML Converter]
+         C --> E[Metadata Extractor]
+         D --> F[HTML Fragments]
+         E --> G[Raw Metadata]
+         G --> H[Schema Validator]
+         H --> I[Validated HTML]
+         F --> J[Tera Templates]
+         I --> J
+         J --> K[Final HTML]
+
+ Simplified processing pipeline +
+ @end + +**** Metadata Conversion + Metadata isn't some afterthought - it's first-class Norg as well. We extract raw document + metadata and convert it directly to TOML through deterministic type mapping: booleans stay + booleans, dates become ISO strings, and nested structures preserve their hierarchy. The Table + of Contents gets injected as TOML arrays for seamless integration with the code generation + later - no external parsers, no fragile regex. + + @embed html +
+ 
+         flowchart TD
+         A[Start Metadata Conversion] --> B[Extract Meta Block]
+         B --> C[Parse Metadata]
+         C --> D{Value Type?}
+         D -->|Bool| E[TOML Boolean]
+         D -->|String| F[TOML Value]
+         D -->|Number| G[TOML Integer/Float]
+         D -->|Array| H[Recurse Array]
+         D -->|Object| I[Recurse Object]
+         J[Merge ToC if Exists] --> K[Return TOML Value]
+
+ If ToC doesn't exist then we just directly + return the TOML Value after finishing the parsing process +
+ @end + +**** Opt-in Schema Validation + This is where Norgolith gets /dead serious/ about content integrity. We don't just extract + metadata - we /validate/ it against strict /yet/ flexible customizable schemas, if the user wants + to. The process starts by resolving hierarchical schema rules (global → directory-specific), + then applies four validation layers: required fields (no half-assed frontmatter), type checks + (dates better be dates!), pattern matching and conditional logic (if `draft=false` then require + `updated_at`). Any violation gets reported with surgical precision, even on the development + server - no fuzzy errors anywhere, nor need to waste your precious time debugging something not + working as expected just to find out a damn frontmatter was broken with no feedback in the + console. + + @embed html +
+ 
+         flowchart TD
+         A[Content File] --> B[Extract Metadata]
+         B --> C[Resolve Schema Path]
+         C --> D[Load Schema Hierarchy]
+         D --> E[Merge Schemas]
+         E --> F{Validation Rules}
+         F -->|Required Fields| G[Check Presence]
+         F -->|Field Types| H[Type Check]
+         F -->|Constraints| I[Validate Patterns/Lengths]
+         F -->|Conditional Rules| J[Apply If-Then Logic]
+         K[Collect Errors] --> L{Errors?}
+         L -->|No| M[Approve Metadata]
+         L -->|Yes| N[Report Violations]
+         N -->|Production| O[Error]
+         N -->|Development| P[Warning]
+
+ + Metadata validation process, simplified. If the development server is being used then a + warning is returned instead of crashing out the server, in order to keep an unobtrusive + workflow + +
+ @end + +**** Heading Conversion + Headings do triple duty: they become semantic HTML headings (`

`-`

`), generate anchor IDs by + slugification, and populate our Table of Contents. The secret sauce: we sanitize IDs in-place + during conversion by stripping link syntax - no separate cleanup pass. Each heading's content + gets processed recursively, maintaining strict context isolation. + + @embed html +
+ 
+         flowchart TD
+         A[Process Heading] --> B[Sanitize Links in Title]
+         B --> C[Generate Heading ID]
+         C --> D[Build Heading Tag]
+         D --> E[Apply Carryover Attributes]
+         E --> F[Add to ToC]
+         F --> G[Process Heading Content]
+         G --> H[Return Heading HTML]
+
+ Everything happens in the same pass to + guarantee maximum efficiency +
+ @end + +**** Paragraph Conversion + Paragraphs are where Norg's magic shines - as in any other /sane/ markup language, they're not + dumb text blocks but rich element containers. Each segment gets processed differently: text + gets sanitized, modifiers become proper HTML tags (`*bold*` → ``), and links transform + into actual anchors. The kicker? Carryover tags apply *context-aware* HTML attributes + mid-conversion - no post-processing needed! + + @embed html +
+ 
+         flowchart TD
+         A[Start Paragraph Conversion] --> B[Process Segment]
+         B --> C{Segment Type?}
+         C -->|Token| D[Append Escaped Text]
+         C -->|Attached Modifier| E[Wrap in HTML Tag]
+         C -->|Inline Verbatim| F[Wrap in code Tag]
+         C -->|Link| G[Build Anchor Tag]
+         C -->|Anchor Definition| H[Build Anchor Definition]
+         G --> I[Handle Local Paths]
+         G --> J[Handle External URLs]
+         G --> K[Handle Headings]
+         H --> L[Resolve Link Target]
+         M[Apply Weak Carryover] --> N[Append to Paragraph]
+         N --> O[Return HTML Paragraph]
+
+ If no carryover tags are present then we + directly return the HTML paragraph, and everything happens in the same pass again. +
+ @end + +**** Carryover Tag Processing + We have mentioned them several times already, but if you ain't familiar with Norg you must be + wondering /“Ok, but what the actual fuck are those?”/, and let me explain it now. Carryover + tags are Norg's secret weapon, and you can find a full explanation in the {https://github.com/nvim-neorg/norg-specs/blob/main/1.0-specification.norg}[specs]. In Norgolith, + we use them for context-aware HTML attributes. When we hit a carryover tag during conversion, + we immediately store its parameters as HTML attributes. These get injected into the very next + element processed (as we use only the /weak/ variant of carryover tags) - whether it's a heading, + paragraph, or verbatim block. The kick? They self-destruct after being applied. /Poof/, no + residual state, no side effects. + + @embed html +
+ 
+         flowchart TD
+         A[Encounter Carryover] --> B{Tag Type?}
+         B -->|Attribute| C[Store HTML Attributes]
+         B -->|Macro| D[Error: Unsupported]
+         C --> E[Apply to Next Element]
+         E --> F[Clear After Use]
+
+ + Currently, macro (strong) carryover tags crash Norgolith, as they are not yet + implemented.
If I recall correctly, they are not supported in the parser. + +
+ @end + +**** List/Quote Handling + Lists and quotes aren't primitive strings nor they depend on indentation - they're recursive + structures. We handle them with nesting-aware conversion: lists morph into proper `
    `/`
      ` + hierarchies with depth tracking, while quotes become `
      ` wrappers. Carryover + attributes inject directly into container tags during traversal. + + @embed html +
      + 
      +         flowchart TD
+         A[Process Nestable] --> B{Type?}
+         B -->|Ordered/Unordered List| C[Generate List Tags]
+         B -->|Quote| D[Generate Blockquote Tag]
+         C --> E[Process List Items]
+         E --> F[Apply Carryover]
+         D --> F
+         F --> G[Process Children]
+         G --> H[Return List HTML]
+
      + + In Norg, blockquotes can have nested blockquotes too. And thus, we must preserve it as well. + +
      + @end + +**** Template Rendering Process + This is where Norgolith ends the conversion process /before the production build process, of + course/. We don't just spit out HTML fragments - we /nuclear-fuse/ them with Templates. The + {https://keats.github.io/tera/}[Tera engine] takes four inputs for our HTML templates: raw HTML from conversion, validated + metadata, global site config, and the complete post collection (with their whole data as well!). + These get injected into layout templates where they become proper web pages. No front\-end + frameworks, no abstractions bullshit - just pure server-rendered HTML at build time. + + @embed html +
      + 
      +         flowchart LR
+         A[HTML Fragment] --> B[Tera Engine]
+         C[Validated Metadata] --> B
+         D[Site Config] --> B
+         E[Post Collection] --> B
+         B --> F[Layout Template]
+         F --> G[Final HTML]
+
      + + The final HTML produced is only written in disk for production builds, otherwise it stays + in-memory during the development process + +
      + @end + +**** Build Orchestration + The production build process is a kill chain: content files get validated against opt-in + schemas first if they do exist in the site config (fail fast, fail loud). Only clean metadata + proceeds to HTML conversion. Meanwhile, assets get minified in parallel - no wasteful + sequential processing. Everything converges in the public directory as deploy-ready files. This + is continuous integration for content - no manual steps, no fragile scripts, no synchronous + execution. + + @embed html +
      + 
      +         flowchart TB
+         A[Content Files] --> B[Schema Validation]
+         B --> C{Valid?}
+         C -->|Yes| D[Convert to HTML]
+         C -->|No| E[Fail Build]
+         D --> F[Apply Templates]
+         G[Assets] --> H[Minify/Copy]
+         F --> I[Public Directory]
+         H --> I
+         I --> J[Deploy]
+
      + + The schema validation step happens only when the user declared schemas into his site + configuration, otherwise it is skipped. Everything is asynchronous, so there are no slowdowns! + +
      + @end + + --- + + Norgolith's conversion pipeline is a masterclass in efficient content processing: raw Norg gets + parsed to ASTs and split into dual HTML/metadata async streams. HTML conversion handles + everything from carryover tags to recursive lists with perfect precision, while metadata + undergoes ruthless schema validation. These streams reunite in Tera templates where they become + final HTML, with assets minified in parallel. The result? Blazing-fast builds where every + heading, paragraph, and metadata field transforms directly into web-ready output - no external + transpilers, no bloated JavaScript frameworks, just pure norg-to-HTML conversion. + +*** Writing Brings Joy Again + Finally, the cherry on top. We've already seen the architecture of how things work, and how I + finished implementing everything. But there's one essential thing left: throughout my life using + static site generators, I've always run into one problem - *insane defaults*. Let's be honest, + most SSGs require modifying the configuration /sometimes/, especially if we want to enable or + disable something after creating the site. + + +html.loading lazy + .image /assets/posts/norg-on-the-web/01.png Norgolith init site demo + + +html.class note + > *Note*: While Norgolith is interactive by default, you can pass the `--no-prompt` flag to the `init` + command and still get sane defaults for your site configuration. All you will have to do when + pushing to production is changing the `rootUrl` option if you haven't already and that's it! + + +html.class mt-3 text-base-alt dark:brightness-120 + ___ + + With Norgolith, I took the trouble to create interactivity and better defaults. While somewhat + opinionated in certain areas (e.g. RSS enabled by default), these are options I typically see + enabled on blogs, which is the primary use case for SSGs. *The configuration is dead simple* - + most options are dynamically generated through prompts when creating the site, and you can + easily extend the configuration if you wish. But why do this? Simple, *better defaults lead to + better initial results*. If I create a new site, and only have to sit down and worry about + creating my site's content instead of spending time reading documentation about my SSG to set + things up exactly how I want them /or even have sane settings/, my brain will feel comfortable. + + +html.class important + > *Important*: The only exception to this rule is the highlighter configuration. By default, + Norgolith will have ootb support for both `PrismJS` and `Highlight.js` through CDN sources in the + default `base.html` template. However, it is disabled by default as some people would like to keep + it disabled or bundle the highlighting libraries with other methods. However, it is as easy as + changing the enable value to `true` in order to enable the support. + + And let's talk about creating content. Everything is pretty straightforward! You enter a + command, get a prompt with a few sane defaults so you just have to press `Enter` if you agree with + them, and call it a day. The content's title case will be automatically transformed into the + {https://daringfireball.net/2008/05/title_case}[proper case] for you, so that even your title is properly capitalized! + + +html.loading lazy + .image /assets/posts/norg-on-the-web/02.png Norgolith create content demo + + You might notice that I didn't enter the file extension in the command as well, and Norgolith + will /automagically/ infer it for us! I know, it's only five more characters, but these five + characters are completely redundant. Why do you need to enter the file extension when it's + obvious that you're using norg, in your norg-first SSG? + + Some other niceties is that we can directly create JavaScript and CSS files with the `--kind + ` option, and open the created files with your preferred system editor by passing the + `--open` flag. You create, edit then save and leave - simple and straightforward workflow. + +** Freeing from Markdown (Migration to Norgolith) + Whether you've read this far (or got here using the note at the very beginning of the post), + thank you. Now let's get to what really matters, the heart of this post and what you probably + came here for: *how to free yourself from the torturous yoke of living under markdown*, or /at least/ + achieve it for your writing needs. + +*** First Steps + Let's start with the basics: install (or just download) Norgolith, familiarize ourselves with + the CLI, and make this our home. It is a simple process, I promise! + + +html.class warning + > *Warning*: + Since most things explained during this section are already explained very well in the + documentation site, I will only do a brief explanation of some things. Please {https://norgolith.amartin.beer}[check out] the + documentation site if you want to know more. Additionally, this guide covers the latest + development version so you may find some commands are named differently (e.g. `serve` is now `dev`). + +**** Obtaining Norgolith + Norgolith can be downloaded /and optionally installed/ in multiple ways depending on your + environment and preferences. Below are the recommended methods: + + +html.class important + > *Important*: The Norgolith binary is called `lith` for ergonomic reasons. + +***** Binary Installation + For stable releases, statically compiled binaries with `musl` are provided in the {https://github.com/NTBBloodbath/norgolith/releases}[releases] + section. There are binaries for Linux and macOS (`x86_64` and `arm64`). Windows is *currently + unsupported* by Norgolith. + + The release tarballs contains the following files: + - `lith` + - `LICENSE` + - `README.md` + + You can also install Norgolith using `cargo`, however this /might/ add an additional step + which is compiling the source code. Prefer this way /only/ if you want to use a development + version. + + *Prerequisites*: + - Rust toolchain (`rustup` installed). + - Rust version `>= 1.77`. + @code bash + # Install from the repository + cargo install --git https://github.com/NTBBloodbath/norgolith --locked + + # Verify the installation + lith --version + @end + +***** Nix Flake + You can add Norgolith to your NixOS configuration or local project with flakes. + + *Adding Norgolith to your NixOS configuration*: + @code nix + { + inputs = { + nixpkgs.url = "github:NixOS/nixpkgs/nixos-unstable"; + + norgolith.url = "github:NTBBloodbath/norgolith"; + }; + outputs = + { nixpkgs, norgolith, ... }: + { + nixosConfigurations.mysystem = nixpkgs.lib.nixosSystem { + modules = [ + { + # add norgolith as a package + environment.systemPackages = [ + norgolith.packages.x86_64-linux.default + ]; + } + ]; + }; + }; + } + @end + + *Adding Norgolith to your local project flake*: + @code nix + # This is just a reference flake from my own blog + { + description = "A very basic flake"; + + inputs = { + nixpkgs.url = "github:nixos/nixpkgs?ref=nixos-unstable&shallow=1"; + flake-utils.url = "github:numtide/flake-utils?shallow=1"; + norgolith.url = "github:NTBBloodbath/norgolith?shallow=1"; + }; + + outputs = { + nixpkgs, + flake-utils, + norgolith, + ... + }: + flake-utils.lib.eachDefaultSystem (system: let + pkgs = import nixpkgs {inherit system;}; + in + { + # nix develop + devShells.default = pkgs.mkShell { + buildInputs = with pkgs; [ + norgolith.packages.${system}.default + ]; + }; + }); + } + @end + +***** Arch Linux (AUR) + Use Arch User Repository (AUR) helper to install `norgolith-git`. + + @code bash + paru -S norgolith-git + @end + + This will compile Norgolith from source. If you do not want to compile it, refer to the {***** Binary Installation}[Binary Installation] section. + +**** Getting Familiar with Norgolith + Now that we have Norgolith up and running on our system, we need to get comfortable using it + and learn how it works. + + +html.class important + > *Important*: this section does *NOT* cover everything in Norgolith, topics such as theming and + content schemas are not mentioned here, for these things please refer to the {https://norgolith.amartin.beer/docs}[documentation site]. + +***** CLI commands + Norgolith keeps familiarity with other SSGs. For a full reference of the available commands + and their flags/options, please refer to the {https://norgolith.amartin.beer/docs/commands/}[commands reference]. + + Norgolith has a simple CLI with just a handful of commands. The ones you’ll use most in your + daily workflow, though, are these: + - `lith new`: Create a new asset in the site. + - `lith serve` (`lith dev` on the latest development version): Build a site for development. + - `lith build`: Build a site for production. + +***** Site structure + Once you create a new site using `lith init `, you will find it has an internal + structure: + + *Site structure for latest stable version*: + @code bash + foo + ├── .build # Dev server artifacts + ├── assets # Static assets (CSS, JS, images) + ├── content # Norg documents go here + ├── templates # Tera templates for HTML generation + ├── theme # Active theme files + └── norgolith.toml # Site configuration + @end + + *Site structure for latest development version*: + @code bash + foo + ├── assets # Static assets (CSS, JS, images) + ├── content # Norg documents go here + ├── templates # Tera templates for HTML generation + ├── theme # Active theme files + └── norgolith.toml # Site configuration + @end + + The key files and directories of your site structure are the following: + - `norgolith.toml`: Contains site-wide settings such as `title`, `author` and content schemas. + - `content/`: Store your Norg files here. Each file will be converted to HTML. + - `templates/`: Customize the HTML output using Tera templates. + + +html.class note + > *Note*: there is also the `public/` directory, which will be generated only after running + the `lith build` command to build your site for production. + +***** Configuration + Norgolith uses the TOML format for configurations, please refer to the {https://norgolith.amartin.beer/docs/configuration/}[configuration reference] + for a full list of every possible configuration option and their valid values. + + *The following is the default configuration file contents*: + @code toml + rootUrl = 'http://localhost:3030' + language = 'en-US' + title = 'foo' + author = 'JohnDoe' + + # Code blocks highlighting + [highlighter] + enable = false + # engine = 'prism' # Can be 'prism' or 'hljs'. Defaults to 'prism' + + # RSS feed + [rss] + enable = true + description = 'Latest posts' + ttl = 60 + image = '/assets/favicon.png' + @end + + +html.class note + > *Note*: the commands configuration has been temporarily removed from the configuration file + in the current development version, they will be re-added in the future. + +*** Getting Familiar with Tera + Tera is a powerful templating engine for Rust (inspired by Jinja2/Django) that Norgolith uses to + generate HTML for your static site. Here's a concise guide to mastering Tera templates in your + `templates/` directory: + + +html.class note + > *Note*: the Norgolith documentation site also has a {https://norgolith.amartin.beer/docs/templating/}[Templating Guide] section which pretty much + covers all of this and includes real-world examples. + +**** Template Basics + - *Directory Structure*: Place all templates in `templates/` (e.g. `templates/base.html`). + - *Variables*: Render content with `{{ variable }}`: + @code jinja2 +

      {{ metadata.title }}

      + @end + + *Norgolith provides several built-in variables for use in your templates*: + - `metadata`: Frontmatter data from the Norg file (includes an additional `permalink` field). + - `config`: Global site configuration from `norgolith.toml`. + - `content`: The rendered HTML content. + - `posts`: All the posts metadata as an array. + +**** Template Inheritance + Create reusable layouts using *inheritance*: + + - *Base Template* (`base.html`): + @code jinja2 + + + + {% block title %}{% endblock %} + + + {% block content %}{% endblock %} + + + @end + - *Child Template* (`page.html`): + @code jinja2 + {% extends "base.html" %} + + {% block title %}My Page{% endblock %} + + {% block content %} +
      {{ content | safe }}
      + {% endblock %} + @end + + See {https://keats.github.io/tera/docs/#inheritance}[Inheritance] + +**** Control Structures + - *Conditionals*: + @code jinja2 + {% if metadata.draft %} + This post is currently a draft, it might change in the future. + {% endif %} + @end + + - *Loops*: + @code jinja2 + {% for post in posts %} +

      {{ post.title }}

      + {% endfor %} + @end + + See {https://keats.github.io/tera/docs/#control-structures}[Control structures]. + +**** Filters & Functions + Modify data directly in templates: + + @code jinja2 + {{ "hello" | capitalize }} → "Hello" + {{ posts | length }} → Number of posts + {{ now() | date(format="%Y-%m-%d") }} → Current date + @end + + See {https://keats.github.io/tera/docs/#built-in-filters}[Full list of built-in filters]. + +*** Migrating Old Posts + I'm going to be honest with this one, the migration process for your posts is kind of *painful*, + and I'm still going through it with my own posts. + + Right now, as I write this post, there’s no such thing as a definitive tool for converting + files to norg. However, there is one tool that—while not perfect—lightens the load of + manually transcribing our posts by a huge margin: {https://github.com/pysan3/minorg}[minorg]. + + Depending on how complex your posts are, it might make a few mistakes or imperfections, but + overall it does a great job and I'm incredibly thankful to `@pysan3` for creating this tool. If + you spot any bugs, please report them in the repo along with clear steps to reproduce them so we + can keep making it better! + + The following snippet is for converting markdown documents to norg using Nix shells. Special + thanks to `@Ladas552` on GitHub for providing me this snippet to convert my old blog posts (and + the Norgolith wiki on GitHub to a documentation site): + @code bash + # Create a Nix shell with Nim, Nimble and Pandoc + nix shell nixpkgs#nim-2_0 nixpkgs#nimble nixpkgs#pandoc + + # Follow the minorg README instructions + + # Finally, convert your documents + pandoc -f 'markdown' -t json \ + /home/JohnDoe/old_blog/posts/example.md | ./minorg generate -o './posts/example.norg' + @end + + Make sure to read the {https://github.com/pysan3/minorg#known-issues}[Known Issues] section in the README to know what will need to be fixed + after converting your files! + +*** Deploying Your Norgolith Site + Deploying your site built with Norgolith is really straightforward, although the final process + will depend on your hosting method. This section covers GitHub Pages, Codeberg Pages, and + self-hosting on your own server. + + +html.class note + > *Note*: This section won't include anything related to custom domains. If you want to use your + own domain with a third-party provider such as GitHub Pages, please refer to their documentation. + Additionally, this section will be upstreamed later to the Norgolith documentation. + +**** GitHub Pages + Deploying to GitHub Pages is a pretty straightforward process thanks to CI workflows (special + thanks to `@boltlessengineer`, from whom I yanked his CI workflow). Simply push your blog + source code alongside the `public` contents generated after running `lith build` into your + `username.github.io` repository. Make sure to add the following CI workflow to your + `.github/workflows` directory: + @code yaml + name: Publish to GH Pages + + permissions: + contents: write + on: + workflow_dispatch: + push: + paths: + - public/** + branches: + - main + + jobs: + publish: + runs-on: ubuntu-latest + steps: + - name: Checkout + uses: actions/checkout@v4 + - name: Deploy + uses: JamesIves/github-pages-deploy-action@v4 + with: + folder: public + @end + + If the CI didn't run during the first push, you can manually trigger it on the actions tab + thanks to the `workflow_dispatch` key in the workflow definition. + + +html.class important + > *Important*: make sure the `rootUrl` option in the site configuration file reflects the full URL + provided by GitHub Pages if using the default address (e.g. `https://.github.io`). + +**** Codeberg Pages + With Codeberg it gets a bit more complicated, since we normally don’t have access to its CI + system, for which you need to have or {https://codeberg.org/Codeberg-e.V./requests/issues/new?template=ISSUE_TEMPLATE%2fWoodpecker-CI.yaml}[request] access, as well as add a `.woodpecker.yaml` file in + the root of your project. Following the provided {https://docs.codeberg.org/codeberg-pages/}[getting started steps] in the documentation + should be enough to get your site up and running. *Make sure to deploy your `public/` directory + contents into the main branch, and the source code for your site in another branch such as + `source` to keep everything organized*. + + Since I personally never requested access to the CI and have never used it, I can’t provide + instructions for automated deployment. + + +html.class important + > *Important*: make sure the `rootUrl` option in the site configuration file reflects the full URL + provided by Codeberg Pages if using the default address (e.g. `https://.codeberg.page`). + +**** Self-hosting + If you're doing self-hosting then you should know what you're doing. However, since {https://caddyserver.com/}[Caddy] is + the only HTTP server I currently use, I can only provide support for it (sorry, I don't + remember how to use Nginx). + + The following is an example `Caddyfile` for serving your static site using Caddy: + @code nginx + { + email your_email@address.com + } + + yourdomain.com { + root * /srv/yourdomain.com + encode zstd gzip + + header { + Cache-Control max-age=3600 + } + + file_server { + precompressed br gzip + } + + log { + format json + output file /var/log/caddy/yourdomain.com_access.log { + roll_size 1gb + roll_keep 5 + roll_keep_for 720h + } + } + } + @end + + Where `/srv/yourdomain.com` is a directory that contains the artifacts produced by your `lith + build` in the `public/` directory. + +*** Current Caveats + Since Norgolith is still in what I like to call its infancy, it remains under constant + development (though I’ve taken a short break to avoid burnout). As a result, it may be missing + some features compared to other SSGs that don’t focus on—or even acknowledge—norg. Such + as: + - Custom 404 and 500 error templates. + - Complementary production files (`sitemap.xml`, `robots.txt`). + - HTML SEO meta tags and Open Graph tags ootb in the default templates. + - Archetypes equivalent, see {https://gohugo.io/content-management/archetypes/}[archetypes in Hugo] for a reference. + - Shortcodes support, see {https://gohugo.io/content-management/shortcodes/}[shortcodes in Hugo] for a reference. + - Miscellaneous templating functions such as `estimated reading time`. + +** Conclusion + Norgolith gives norg enthusiasts an /extremely/ fast, lightweight, and portable alternative that + zeroes in on norg for building their web content. The project is in constant dogfooding, brimming + with ideas, and has already been picked up by a few folks for their blogs. So what are you + waiting for—go for it! + + If you’d like to contribute, you’re more than welcome—whether it’s just filing bug + reports, requesting features, or fixing typos. Please take a look at the {https://norgolith.amartin.beer/docs/contributing/}[Contributing] section on + the documentation site to learn the guidelines and practices I recommend following when + contributing code to the project. Any contribution is welcome and you can join the {https://discord.gg/T6EgTAX7ht}[Neorg + community on Discord] for support in the `#norgolith` channel under the `3rd party` section! + + Make sure to check out the {https://github.com/NTBBloodbath/norgolith/issues?q=sort%3Aupdated-desc+is%3Aissue+is%3Aopen}[issues] currently in the repository, the {https://github.com/NTBBloodbath/norgolith/milestone/3}[milestone] for reaching version + 1.0 (/which includes a plugin system/), and if you’d like to help me dedicate more time to the + project, you can also do so through {https://ko-fi.com/ntbbloodbath}[sponsoring]. Thank you so much for reading this massive tome, + and I’ll see you in the next post! diff --git a/benches/inputs/small.norg b/benches/inputs/small.norg new file mode 100644 index 0000000..10a2594 --- /dev/null +++ b/benches/inputs/small.norg @@ -0,0 +1,41 @@ +* Heading Level One + Paragraph with *bold*, /italic/, and `verbatim` inline modifiers. + A {https://example.com}[link] and an [anchor] reference. +** Heading Level Two + - Unordered list item one + - Unordered list item two + -- Nested list item + ~ Ordered list item + > Quote block + > with content +$ Definition Term + Definition content +$$ Ranged Definition + Content here +$$ +^ Footnote Title + Footnote content +^^ Ranged Footnote + Content here +^^ +: Table Cell + Table content +:: Ranged Table + Content here +:: +@code +fn main() { + println!("hello"); +} +@end +|example +Some *ranged* content +|end +=macro.example +Some content +=end +.tag-name ++html.class note +> *Note*: This is a carryover note. +___ +Second paragraph after horizontal rule. \ No newline at end of file diff --git a/benches/parser_benchmarks.rs b/benches/parser_benchmarks.rs new file mode 100644 index 0000000..db261aa --- /dev/null +++ b/benches/parser_benchmarks.rs @@ -0,0 +1,90 @@ +use std::hint::black_box; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use chumsky::Parser as _; +use rust_norg::{parse, parse_tree, stage_1, stage_2, stage_3, stage_4, NorgASTFlat}; + +const SMALL: &str = include_str!("inputs/small.norg"); +const MEDIUM: &str = include_str!("inputs/medium.norg"); +const LARGE: &str = include_str!("inputs/large.norg"); + +fn parse_tree_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("parse_tree"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + group.throughput(Throughput::Elements(input.lines().count() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), input, |b, input| { + b.iter(|| parse_tree(black_box(input))) + }); + } + group.finish(); +} + +fn parse_flat_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("parse_flat"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + group.throughput(Throughput::Elements(input.lines().count() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), input, |b, input| { + b.iter(|| parse(black_box(input))) + }); + } + group.finish(); +} + +fn stage_1_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("stage_1"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + group.throughput(Throughput::Elements(input.lines().count() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), input, |b, input| { + b.iter(|| stage_1().parse(black_box(input))) + }); + } + group.finish(); +} + +fn stage_2_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("stage_2"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + let tokens = stage_1().parse(input).unwrap(); + group.throughput(Throughput::Elements(tokens.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), &tokens, |b, tokens| { + b.iter(|| stage_2().parse(black_box(tokens.clone())).unwrap()) + }); + } + group.finish(); +} + +fn stage_3_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("stage_3"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + let tokens = stage_1().parse(input).unwrap(); + let blocks = stage_2().parse(tokens).unwrap(); + group.throughput(Throughput::Elements(blocks.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), &blocks, |b, blocks| { + b.iter(|| stage_3().parse(black_box(blocks.clone())).unwrap()) + }); + } + group.finish(); +} + +fn stage_4_bench(c: &mut Criterion) { + let mut group = c.benchmark_group("stage_4"); + for (name, input) in [("small", SMALL), ("medium", MEDIUM), ("large", LARGE)] { + let flat: Vec = parse(input).unwrap(); + group.throughput(Throughput::Elements(flat.len() as u64)); + group.bench_with_input(BenchmarkId::from_parameter(name), &flat, |b, flat| { + b.iter(|| stage_4(black_box(flat.clone()))) + }); + } + group.finish(); +} + +criterion_group!( + name = benches; + config = Criterion::default().sample_size(50); + targets = parse_tree_bench, + parse_flat_bench, + stage_1_bench, + stage_2_bench, + stage_3_bench, + stage_4_bench +); +criterion_main!(benches); diff --git a/src/lib.rs b/src/lib.rs index 603c261..cce2fb2 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -3,7 +3,7 @@ use error::NorgParseError; pub use crate::stage_1::stage_1; pub use crate::stage_2::stage_2; -use crate::stage_4::stage_4; +pub use crate::stage_4::stage_4; pub use crate::stage_2::ParagraphSegmentToken; pub use crate::stage_3::*;