Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
24 changes: 24 additions & 0 deletions src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -683,6 +683,30 @@ mod tests {
}
}

#[test]
fn verbatim_after_attached_modifier() {
let examples: Vec<_> = [
// An inline verbatim opening immediately after an attached
// modifier: tree-sitter-norg reads these as underline(verbatim).
"_`Tree`_",
"*`Tree`*",
"_`a *b* c`_",
// Several verbatims inside one modifier must pair up in order.
"_`a` and `b`_",
// Unchanged cases, kept here so a regression is obvious.
"_x `y` z_",
"`Tree`",
"_*Tree*_",
]
.into_iter()
.map(|example| example.to_string() + "\n")
.map(|str| parse(&str))
.try_collect()
.unwrap();

assert_yaml_snapshot!(examples);
}

#[test]
fn modifiers() {
let examples: Vec<_> = [
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,64 @@
---
source: src/lib.rs
expression: examples
---
- - Paragraph:
- AttachedModifier:
modifier_type: _
content:
- InlineVerbatim:
- Text: Tree
- - Paragraph:
- AttachedModifier:
modifier_type: "*"
content:
- InlineVerbatim:
- Text: Tree
- - Paragraph:
- AttachedModifier:
modifier_type: _
content:
- InlineVerbatim:
- Text: a
- Whitespace
- Special: "*"
- Text: b
- Special: "*"
- Whitespace
- Text: c
- - Paragraph:
- AttachedModifier:
modifier_type: _
content:
- InlineVerbatim:
- Text: a
- Token: Whitespace
- Token:
Text: and
- Token: Whitespace
- InlineVerbatim:
- Text: b
- - Paragraph:
- AttachedModifier:
modifier_type: _
content:
- Token:
Text: x
- Token: Whitespace
- InlineVerbatim:
- Text: y
- Token: Whitespace
- Token:
Text: z
- - Paragraph:
- InlineVerbatim:
- Text: Tree
- - Paragraph:
- AttachedModifier:
modifier_type: _
content:
- AttachedModifier:
modifier_type: "*"
content:
- Token:
Text: Tree
30 changes: 26 additions & 4 deletions src/stage_3.rs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,9 @@ use textwrap::dedent;

use crate::stage_2::{NorgBlock, ParagraphSegmentToken, ParagraphTokenList};

/// The inline-verbatim delimiter.
const VERBATIM: char = '`';

#[derive(Clone, Copy, Hash, Debug, PartialEq, Eq, Serialize)]
pub enum NestableDetachedModifier {
Quote,
Expand Down Expand Up @@ -119,17 +122,30 @@ fn paragraph_parser_opener_candidates_and_links() -> impl Parser<
s @ ParagraphSegmentToken::Special(_) => s,
};

// An attached modifier may be immediately followed by an inline verbatim
// (`` _`Tree`_ ``, which tree-sitter-norg reads as underline(verbatim)).
// The generic branch *consumes* the token it checks, which swallowed the
// opening backtick so `inline_verbatim` below could never claim it — and
// with several verbatims on one line it shifted the pairing. Peek instead
// of consuming for '`' only, leaving every other case byte-identical.
let opener_rhs = || {
choice((
just(ParagraphSegmentToken::Special(VERBATIM)).rewind(),
just(ParagraphSegmentToken::Whitespace).not(),
))
};

let opening_modifier_candidate = whitespace_or_special
.then(modifier.repeated().at_least(1))
.then(just(ParagraphSegmentToken::Whitespace).not())
.then(opener_rhs())
.map(|((left, modifiers), right)| {
ParagraphSegment::AttachedModifierOpener((Some(left), modifiers, right))
});

let left_empty_opening_modifier = modifier
.repeated()
.at_least(1)
.then(just(ParagraphSegmentToken::Whitespace).not())
.then(opener_rhs())
.map(|(modifiers, right)| {
ParagraphSegment::AttachedModifierOpener((None, modifiers, right))
});
Expand Down Expand Up @@ -344,7 +360,11 @@ fn unravel_candidates(input: Vec<ParagraphSegment>) -> Vec<ParagraphSegment> {
closer: None,
}
}));
acc.push(Token(right));
// A peeked '`' was never consumed (see opener_rhs); it is
// still in the stream, so re-emitting it would duplicate it.
if right != ParagraphSegmentToken::Special(VERBATIM) {
acc.push(Token(right));
}
}
AttachedModifierCloserCandidate((left, modifiers, right)) => {
acc.push(*left);
Expand All @@ -363,7 +383,9 @@ fn unravel_candidates(input: Vec<ParagraphSegment>) -> Vec<ParagraphSegment> {
.into_iter()
.map(|c| Token(ParagraphSegmentToken::Special(c))),
);
acc.push(Token(right));
if right != ParagraphSegmentToken::Special(VERBATIM) {
acc.push(Token(right));
}
}
others => acc.push(others),
};
Expand Down
Loading