diff --git a/CHANGELOG.md b/CHANGELOG.md index 0bd786f1..8793336f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,7 @@ ## [Unreleased] - [#211] Add support for the GFM tagfilter extension - treat emphasize inside link destination as text +- [#196] Fix GFM dollar math delimiter parsing ## [0.7.9] - [#210] Accept a `CancellationToken` in `EmptyStreamingMarkdownFile` diff --git a/src/commonMain/kotlin/org/intellij/markdown/html/GeneratingProviders.kt b/src/commonMain/kotlin/org/intellij/markdown/html/GeneratingProviders.kt index 4df462f0..92de38a1 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/html/GeneratingProviders.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/html/GeneratingProviders.kt @@ -335,7 +335,26 @@ open class ImageGeneratingProvider(linkMap: LinkMap, baseURI: URI?) : LinkGenera } private fun getPlainTextFrom(node: ASTNode, text: String): CharSequence { - return REGEX.replace(node.getTextInNode(text), "") + return when { + node.type == MarkdownElementTypes.LINK_DESTINATION -> REGEX.replace(node.getTextInNode(text), "") + node is LeafASTNode -> { + if (isMarkupToken(node.type)) "" else HtmlGenerator.leafText(text, node) + } + else -> node.children.joinToString(separator = "") { + getPlainTextFrom(it, text) + } + } + } + + private fun isMarkupToken(type: IElementType): Boolean { + return type == MarkdownTokenTypes.EMPH || + type == MarkdownTokenTypes.BACKTICK || + type == MarkdownTokenTypes.ESCAPED_BACKTICKS || + type == MarkdownTokenTypes.EXCLAMATION_MARK || + type == MarkdownTokenTypes.LBRACKET || + type == MarkdownTokenTypes.RBRACKET || + type == MarkdownTokenTypes.LPAREN || + type == MarkdownTokenTypes.RPAREN } companion object { diff --git a/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/impl/MathParser.kt b/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/impl/MathParser.kt index d7476525..5c7a684b 100644 --- a/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/impl/MathParser.kt +++ b/src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/impl/MathParser.kt @@ -1,7 +1,10 @@ package org.intellij.markdown.parser.sequentialparsers.impl +import org.intellij.markdown.MarkdownTokenTypes import org.intellij.markdown.flavours.gfm.GFMElementTypes import org.intellij.markdown.flavours.gfm.GFMTokenTypes +import org.intellij.markdown.html.isPunctuation +import org.intellij.markdown.html.isWhitespace import org.intellij.markdown.parser.sequentialparsers.RangesListBuilder import org.intellij.markdown.parser.sequentialparsers.SequentialParser import org.intellij.markdown.parser.sequentialparsers.TokensCache @@ -10,20 +13,39 @@ class MathParser : SequentialParser { override fun parse(tokens: TokensCache, rangesToGlue: List): SequentialParser.ParsingResult { val result = SequentialParser.ParsingResultBuilder() val delegateIndices = RangesListBuilder() - var iterator: TokensCache.Iterator = tokens.RangesListIterator(rangesToGlue) + var linkRangeIndex = 0 + var skipUntil = -1 + var iterator = tokens.RangesListIterator(rangesToGlue) + + val ranges = collectLinkRanges(tokens, rangesToGlue) + val nextClosers = collectNextClosingIndices(tokens, rangesToGlue, ranges) while (iterator.type != null) { - if (iterator.type == GFMTokenTypes.DOLLAR) { + if (iterator.index <= skipUntil) { + iterator = iterator.advance() + continue + } - val endIterator = findOfSize(iterator.advance(), iterator.length) + if (iterator.type == GFMTokenTypes.DOLLAR && canOpenMath(iterator)) { + while (linkRangeIndex < ranges.size && iterator.index > ranges[linkRangeIndex].last) { + linkRangeIndex++ + } - if (endIterator != null) { + val isInsideLink = linkRangeIndex < ranges.size && iterator.index >= ranges[linkRangeIndex].first + if (!isInsideLink) { + val endIndex = nextClosers[iterator.index] + if (endIndex == -1) { + delegateIndices.put(iterator.index) + iterator = iterator.advance() + continue + } if (iterator.length == 1) { - result.withNode(SequentialParser.Node(iterator.index..endIterator.index + 1, GFMElementTypes.INLINE_MATH)) + result.withNode(SequentialParser.Node(iterator.index..endIndex + 1, GFMElementTypes.INLINE_MATH)) } else { - result.withNode(SequentialParser.Node(iterator.index..endIterator.index + 1, GFMElementTypes.BLOCK_MATH)) + result.withNode(SequentialParser.Node(iterator.index..endIndex + 1, GFMElementTypes.BLOCK_MATH)) } - iterator = endIterator.advance() + skipUntil = endIndex + iterator = iterator.advance() continue } } @@ -34,17 +56,74 @@ class MathParser : SequentialParser { return result.withFurtherProcessing(delegateIndices.get()) } - private fun findOfSize(it: TokensCache.Iterator, length: Int): TokensCache.Iterator? { - var iterator = it - while (iterator.type != null) { - if (iterator.type == GFMTokenTypes.DOLLAR) { - if (iterator.length == length) { - return iterator + private fun collectNextClosingIndices( + tokens: TokensCache, + rangesToGlue: List, + linkRanges: List, + ): IntArray { + val result = IntArray(tokens.filteredTokens.size) { -1 } + val nextClosingByLength = HashMap() + var linkRangeIndex = linkRanges.lastIndex + + for (range in rangesToGlue.asReversed()) { + for (index in range.last downTo range.first) { + val iterator = tokens.Iterator(index) + if (iterator.type != GFMTokenTypes.DOLLAR) continue + + while (linkRangeIndex >= 0 && index < linkRanges[linkRangeIndex].first) { + linkRangeIndex-- + } + val isInsideLink = linkRangeIndex >= 0 && index <= linkRanges[linkRangeIndex].last + + result[index] = nextClosingByLength[iterator.length] ?: -1 + if (!isInsideLink && canCloseMath(iterator)) { + nextClosingByLength[iterator.length] = index } } + } + return result + } + + private fun canOpenMath(iterator: TokensCache.Iterator): Boolean { + val previous = iterator.charLookup(-1) + return !isWhitespace(iterator.charLookup(1)) && !previous.isWordCharacter() && !isPunctuation(previous) + } + + private fun canCloseMath(iterator: TokensCache.Iterator): Boolean { + return !isWhitespace(iterator.charLookup(-1)) && !iterator.charLookup(1).isWordCharacter() + } + private fun collectLinkRanges(tokens: TokensCache, rangesToGlue: List): List { + val result = ArrayList() + val inlineLinkStarts = LinkParserUtil.buildBracketStarts(tokens, rangesToGlue) { + it.rawLookup(1) == MarkdownTokenTypes.LPAREN + } + val referenceLinkStarts = LinkParserUtil.buildBracketStarts(tokens, rangesToGlue) + var iterator: TokensCache.Iterator = tokens.RangesListIterator(rangesToGlue) + + while (iterator.type != null) { + if (iterator.type == MarkdownTokenTypes.LBRACKET) { + val link = if (iterator.index in inlineLinkStarts) { + InlineLinkParser.parseInlineLink(iterator, inlineLinkStarts) + } else { + null + } ?: if (iterator.index in referenceLinkStarts) { + ReferenceLinkParser.parseReferenceLink(iterator) + } else { + null + } + if (link != null) { + result.add(iterator.index..link.iteratorPosition.index) + iterator = link.iteratorPosition.advance() + continue + } + } iterator = iterator.advance() } - return null + return result + } + + private fun Char.isWordCharacter(): Boolean { + return isLetterOrDigit() || this == '_' } } diff --git a/src/commonTest/kotlin/org/intellij/markdown/GfmTest.kt b/src/commonTest/kotlin/org/intellij/markdown/GfmTest.kt index 5fc6bde9..c70bd92e 100644 --- a/src/commonTest/kotlin/org/intellij/markdown/GfmTest.kt +++ b/src/commonTest/kotlin/org/intellij/markdown/GfmTest.kt @@ -41,4 +41,64 @@ class GfmTest: SpecTest(org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor( html = "

before <script> after</p>" ) } + + @Test + fun testDollarInsideStrongIsNotMath() = doTest( + markdown = "**$0.85 EPS** (vs. $0.70 est.)", + html = "<p><strong>$0.85 EPS</strong> (vs. $0.70 est.)</p>" + ) + + @Test + fun testInlineMathDoesNotStartInsideWordsOrAfterPunctuation() = doPlainParagraphTests( + "foo$1+2$ bar", + ".$1+2$", + "-$1+2$", + "_$1+2$", + "/$1+2$", + "+$1+2$", + "@$1+2$", + "#$1+2$", + ) + + @Test + fun testInlineMathDoesNotEndBeforeWordCharacters() = doPlainParagraphTests( + "$1+2$3", + "$1+2\$a", + "\$x\$_", + ) + + @Test + fun testMathIsNotParsedInsideLinkText() = doTest( + markdown = "[$1+2$](https://example.com)", + html = "<p><a href=\"https://example.com\">$1+2$</a></p>" + ) + + @Test + fun testMathIsNotParsedInsideLinkDestination() = doTest( + markdown = "[x](https://example.com/$1+2$) $1+2$", + html = "<p><a href=\"https://example.com/$1+2$\">x</a> <span class=\"math\" inline = \"true\">1+2</span></p>" + ) + + @Test + fun testMathIsNotParsedInsideImageText() = doTest( + markdown = "![$1+2$](x.png)", + html = "<p><img src=\"x.png\" alt=\"$1+2$\" /></p>" + ) + + @Test + fun testBlockMathDoesNotStartInsideWords() = doPlainParagraphTests( + "foo\$\$x+y\$\$bar", + ) + + @Test + fun testBlockMathDoesNotEndBeforeWordCharacters() = doPlainParagraphTests( + "\$\$x+y\$\$3", + "\$\$x+y\$\$a", + ) + + private fun doPlainParagraphTests(vararg markdowns: String) { + markdowns.forEach { markdown -> + doTest(markdown = markdown, html = "<p>$markdown</p>") + } + } } diff --git a/src/jvmTest/kotlin/org/intellij/markdown/ParserPerformanceTest.kt b/src/jvmTest/kotlin/org/intellij/markdown/ParserPerformanceTest.kt index 7992de2e..a78bc2b7 100644 --- a/src/jvmTest/kotlin/org/intellij/markdown/ParserPerformanceTest.kt +++ b/src/jvmTest/kotlin/org/intellij/markdown/ParserPerformanceTest.kt @@ -1,20 +1,28 @@ package org.intellij.markdown import junit.framework.TestCase +import org.intellij.markdown.flavours.MarkdownFlavourDescriptor import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor +import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor import org.intellij.markdown.parser.MarkdownParser import org.junit.experimental.categories.Category import java.io.File -import kotlin.test.* +import kotlin.test.Test +import kotlin.test.assertTrue @Category(ParserPerformanceTest::class) class ParserPerformanceTest : TestCase() { protected fun getTestDataPath(): String { return File(getIntellijMarkdownHome() + "/src/jvmTest/resources/data/performance").absolutePath } - private fun assertFast(content: String, fullParse: Boolean, expectedTimeMs: Int? = 1000) { + private fun assertFast( + content: String, + fullParse: Boolean, + expectedTimeMs: Int? = 1000, + flavour: MarkdownFlavourDescriptor = CommonMarkFlavourDescriptor(), + ) { val runnable = { i: Int -> - val root = MarkdownParser(CommonMarkFlavourDescriptor()). + val root = MarkdownParser(flavour). parse(MarkdownElementTypes.MARKDOWN_FILE, content, fullParse) assert(root.children.size > 0) } @@ -80,6 +88,20 @@ import kotlin.test.* assertFast("[a]: <" + "y".repeat(20_000_000), false) } + @Test + fun testRejectedMathClosersAreLinear() { + val input = (1..20_000).joinToString(" ") { "\$x\$1" } + assertFast(input, false, flavour = GFMFlavourDescriptor()) + } + + @Test + fun testMathLinkRangeLookupIsLinear() { + val input = (1..10_000).joinToString(" ") { i -> + "[link$i](url$i) \$x\$" + } + assertFast(input, false, flavour = GFMFlavourDescriptor()) + } + companion object { val WARM_UP_NUM = 10 val TEST_NUM = 100