From 028e68a26f55a59ea5820cd763f261d07b30ec45 Mon Sep 17 00:00:00 2001 From: docker-run Date: Mon, 20 Jul 2026 21:10:43 +0200 Subject: [PATCH] fix: recognise inline verbatim opening after an attached modifier `_`Tree`_` is parsed by tree-sitter-norg as underline(verbatim), but this parser returned the backticks as plain Special tokens, so the verbatim was lost. The attached-modifier opener candidates end with .then(just(ParagraphSegmentToken::Whitespace).not()) which asserts "not followed by whitespace" but *consumes* the token it tests. When that token is a backtick the opener swallows it, so `inline_verbatim` never sees an opening delimiter. With more than one verbatim on the line the damage is worse than a dropped modifier: in `_`a` and `b`_` the swallowed first backtick shifts the pairing, so backticks 2 and 3 pair up and ` and ` becomes the verbatim. Peek instead of consuming, for '`' only, so every other parse path is byte-identical; `unravel_candidates` then skips re-emitting the token it no longer consumed. Rewinding unconditionally also works for verbatim but regresses `*/hello/*, world!`, where the left-empty opener stops consuming its right token and `/` is re-claimed as a new opener's left context. Adds a snapshot test covering verbatim directly after a modifier, multiple verbatims in one modifier, raw content inside a nested verbatim, and the previously-working cases as regression guards. --- src/lib.rs | 24 +++++++ ...sts__verbatim_after_attached_modifier.snap | 64 +++++++++++++++++++ src/stage_3.rs | 30 +++++++-- 3 files changed, 114 insertions(+), 4 deletions(-) create mode 100644 src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap diff --git a/src/lib.rs b/src/lib.rs index 603c261..3a6b355 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -683,6 +683,30 @@ mod tests { } } + #[test] + fn verbatim_after_attached_modifier() { + let examples: Vec<_> = [ + // An inline verbatim opening immediately after an attached + // modifier: tree-sitter-norg reads these as underline(verbatim). + "_`Tree`_", + "*`Tree`*", + "_`a *b* c`_", + // Several verbatims inside one modifier must pair up in order. + "_`a` and `b`_", + // Unchanged cases, kept here so a regression is obvious. + "_x `y` z_", + "`Tree`", + "_*Tree*_", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + #[test] fn modifiers() { let examples: Vec<_> = [ diff --git a/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap b/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap new file mode 100644 index 0000000..9714643 --- /dev/null +++ b/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap @@ -0,0 +1,64 @@ +--- +source: src/lib.rs +expression: examples +--- +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: "*" + content: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: a + - Whitespace + - Special: "*" + - Text: b + - Special: "*" + - Whitespace + - Text: c +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: a + - Token: Whitespace + - Token: + Text: and + - Token: Whitespace + - InlineVerbatim: + - Text: b +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - Token: + Text: x + - Token: Whitespace + - InlineVerbatim: + - Text: y + - Token: Whitespace + - Token: + Text: z +- - Paragraph: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - AttachedModifier: + modifier_type: "*" + content: + - Token: + Text: Tree diff --git a/src/stage_3.rs b/src/stage_3.rs index 0c066bf..dd5a014 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -7,6 +7,9 @@ use textwrap::dedent; use crate::stage_2::{NorgBlock, ParagraphSegmentToken, ParagraphTokenList}; +/// The inline-verbatim delimiter. +const VERBATIM: char = '`'; + #[derive(Clone, Copy, Hash, Debug, PartialEq, Eq, Serialize)] pub enum NestableDetachedModifier { Quote, @@ -119,9 +122,22 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< s @ ParagraphSegmentToken::Special(_) => s, }; + // An attached modifier may be immediately followed by an inline verbatim + // (`` _`Tree`_ ``, which tree-sitter-norg reads as underline(verbatim)). + // The generic branch *consumes* the token it checks, which swallowed the + // opening backtick so `inline_verbatim` below could never claim it — and + // with several verbatims on one line it shifted the pairing. Peek instead + // of consuming for '`' only, leaving every other case byte-identical. + let opener_rhs = || { + choice(( + just(ParagraphSegmentToken::Special(VERBATIM)).rewind(), + just(ParagraphSegmentToken::Whitespace).not(), + )) + }; + let opening_modifier_candidate = whitespace_or_special .then(modifier.repeated().at_least(1)) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .then(opener_rhs()) .map(|((left, modifiers), right)| { ParagraphSegment::AttachedModifierOpener((Some(left), modifiers, right)) }); @@ -129,7 +145,7 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< let left_empty_opening_modifier = modifier .repeated() .at_least(1) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .then(opener_rhs()) .map(|(modifiers, right)| { ParagraphSegment::AttachedModifierOpener((None, modifiers, right)) }); @@ -344,7 +360,11 @@ fn unravel_candidates(input: Vec) -> Vec { closer: None, } })); - acc.push(Token(right)); + // A peeked '`' was never consumed (see opener_rhs); it is + // still in the stream, so re-emitting it would duplicate it. + if right != ParagraphSegmentToken::Special(VERBATIM) { + acc.push(Token(right)); + } } AttachedModifierCloserCandidate((left, modifiers, right)) => { acc.push(*left); @@ -363,7 +383,9 @@ fn unravel_candidates(input: Vec) -> Vec { .into_iter() .map(|c| Token(ParagraphSegmentToken::Special(c))), ); - acc.push(Token(right)); + if right != ParagraphSegmentToken::Special(VERBATIM) { + acc.push(Token(right)); + } } others => acc.push(others), };