diff --git a/src/lib.rs b/src/lib.rs index 603c261..3a6b355 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -683,6 +683,30 @@ mod tests { } } + #[test] + fn verbatim_after_attached_modifier() { + let examples: Vec<_> = [ + // An inline verbatim opening immediately after an attached + // modifier: tree-sitter-norg reads these as underline(verbatim). + "_`Tree`_", + "*`Tree`*", + "_`a *b* c`_", + // Several verbatims inside one modifier must pair up in order. + "_`a` and `b`_", + // Unchanged cases, kept here so a regression is obvious. + "_x `y` z_", + "`Tree`", + "_*Tree*_", + ] + .into_iter() + .map(|example| example.to_string() + "\n") + .map(|str| parse(&str)) + .try_collect() + .unwrap(); + + assert_yaml_snapshot!(examples); + } + #[test] fn modifiers() { let examples: Vec<_> = [ diff --git a/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap b/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap new file mode 100644 index 0000000..9714643 --- /dev/null +++ b/src/snapshots/rust_norg__tests__verbatim_after_attached_modifier.snap @@ -0,0 +1,64 @@ +--- +source: src/lib.rs +expression: examples +--- +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: "*" + content: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: a + - Whitespace + - Special: "*" + - Text: b + - Special: "*" + - Whitespace + - Text: c +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - InlineVerbatim: + - Text: a + - Token: Whitespace + - Token: + Text: and + - Token: Whitespace + - InlineVerbatim: + - Text: b +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - Token: + Text: x + - Token: Whitespace + - InlineVerbatim: + - Text: y + - Token: Whitespace + - Token: + Text: z +- - Paragraph: + - InlineVerbatim: + - Text: Tree +- - Paragraph: + - AttachedModifier: + modifier_type: _ + content: + - AttachedModifier: + modifier_type: "*" + content: + - Token: + Text: Tree diff --git a/src/stage_3.rs b/src/stage_3.rs index 0c066bf..dd5a014 100644 --- a/src/stage_3.rs +++ b/src/stage_3.rs @@ -7,6 +7,9 @@ use textwrap::dedent; use crate::stage_2::{NorgBlock, ParagraphSegmentToken, ParagraphTokenList}; +/// The inline-verbatim delimiter. +const VERBATIM: char = '`'; + #[derive(Clone, Copy, Hash, Debug, PartialEq, Eq, Serialize)] pub enum NestableDetachedModifier { Quote, @@ -119,9 +122,22 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< s @ ParagraphSegmentToken::Special(_) => s, }; + // An attached modifier may be immediately followed by an inline verbatim + // (`` _`Tree`_ ``, which tree-sitter-norg reads as underline(verbatim)). + // The generic branch *consumes* the token it checks, which swallowed the + // opening backtick so `inline_verbatim` below could never claim it — and + // with several verbatims on one line it shifted the pairing. Peek instead + // of consuming for '`' only, leaving every other case byte-identical. + let opener_rhs = || { + choice(( + just(ParagraphSegmentToken::Special(VERBATIM)).rewind(), + just(ParagraphSegmentToken::Whitespace).not(), + )) + }; + let opening_modifier_candidate = whitespace_or_special .then(modifier.repeated().at_least(1)) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .then(opener_rhs()) .map(|((left, modifiers), right)| { ParagraphSegment::AttachedModifierOpener((Some(left), modifiers, right)) }); @@ -129,7 +145,7 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser< let left_empty_opening_modifier = modifier .repeated() .at_least(1) - .then(just(ParagraphSegmentToken::Whitespace).not()) + .then(opener_rhs()) .map(|(modifiers, right)| { ParagraphSegment::AttachedModifierOpener((None, modifiers, right)) }); @@ -344,7 +360,11 @@ fn unravel_candidates(input: Vec) -> Vec { closer: None, } })); - acc.push(Token(right)); + // A peeked '`' was never consumed (see opener_rhs); it is + // still in the stream, so re-emitting it would duplicate it. + if right != ParagraphSegmentToken::Special(VERBATIM) { + acc.push(Token(right)); + } } AttachedModifierCloserCandidate((left, modifiers, right)) => { acc.push(*left); @@ -363,7 +383,9 @@ fn unravel_candidates(input: Vec) -> Vec { .into_iter() .map(|c| Token(ParagraphSegmentToken::Special(c))), ); - acc.push(Token(right)); + if right != ParagraphSegmentToken::Special(VERBATIM) { + acc.push(Token(right)); + } } others => acc.push(others), };