Fix dollar math parsing boundaries

CHANGELOG.md

@@ -4,6 +4,7 @@
 ## [Unreleased]
 - [#211] Add support for the GFM tagfilter extension
 - treat emphasize inside link destination as text
+- [#196] Fix GFM dollar math delimiter parsing
 
 ## [0.7.9]
 - [#210] Accept a `CancellationToken` in `EmptyStreamingMarkdownFile`

src/commonMain/kotlin/org/intellij/markdown/html/GeneratingProviders.kt

@@ -335,7 +335,26 @@ open class ImageGeneratingProvider(linkMap: LinkMap, baseURI: URI?) : LinkGenera
     }
 
     private fun getPlainTextFrom(node: ASTNode, text: String): CharSequence {
-        return REGEX.replace(node.getTextInNode(text), "")
+        return when {
+            node.type == MarkdownElementTypes.LINK_DESTINATION -> REGEX.replace(node.getTextInNode(text), "")
+            node is LeafASTNode -> {
+                if (isMarkupToken(node.type)) "" else HtmlGenerator.leafText(text, node)
+            }
+            else -> node.children.joinToString(separator = "") {
+                getPlainTextFrom(it, text)
+            }
+        }
+    }
+
+    private fun isMarkupToken(type: IElementType): Boolean {
+        return type == MarkdownTokenTypes.EMPH ||
+                type == MarkdownTokenTypes.BACKTICK ||
+                type == MarkdownTokenTypes.ESCAPED_BACKTICKS ||
+                type == MarkdownTokenTypes.EXCLAMATION_MARK ||
+                type == MarkdownTokenTypes.LBRACKET ||
+                type == MarkdownTokenTypes.RBRACKET ||
+                type == MarkdownTokenTypes.LPAREN ||
+                type == MarkdownTokenTypes.RPAREN
     }
 
     companion object {

src/commonMain/kotlin/org/intellij/markdown/parser/sequentialparsers/impl/MathParser.kt

@@ -1,7 +1,10 @@
 package org.intellij.markdown.parser.sequentialparsers.impl
 
+import org.intellij.markdown.MarkdownTokenTypes
 import org.intellij.markdown.flavours.gfm.GFMElementTypes
 import org.intellij.markdown.flavours.gfm.GFMTokenTypes
+import org.intellij.markdown.html.isPunctuation
+import org.intellij.markdown.html.isWhitespace
 import org.intellij.markdown.parser.sequentialparsers.RangesListBuilder
 import org.intellij.markdown.parser.sequentialparsers.SequentialParser
 import org.intellij.markdown.parser.sequentialparsers.TokensCache
@@ -10,20 +13,39 @@ class MathParser : SequentialParser {
     override fun parse(tokens: TokensCache, rangesToGlue: List<IntRange>): SequentialParser.ParsingResult {
         val result = SequentialParser.ParsingResultBuilder()
         val delegateIndices = RangesListBuilder()
-        var iterator: TokensCache.Iterator = tokens.RangesListIterator(rangesToGlue)
+        var linkRangeIndex = 0
+        var skipUntil = -1
+        var iterator = tokens.RangesListIterator(rangesToGlue)
+
+        val ranges = collectLinkRanges(tokens, rangesToGlue)
+        val nextClosers = collectNextClosingIndices(tokens, rangesToGlue, ranges)
 
         while (iterator.type != null) {
-            if (iterator.type == GFMTokenTypes.DOLLAR) {
+            if (iterator.index <= skipUntil) {
+                iterator = iterator.advance()
+                continue
+            }
 
-                val endIterator = findOfSize(iterator.advance(), iterator.length)
+            if (iterator.type == GFMTokenTypes.DOLLAR && canOpenMath(iterator)) {
+                while (linkRangeIndex < ranges.size && iterator.index > ranges[linkRangeIndex].last) {
+                    linkRangeIndex++
+                }
 
-                if (endIterator != null) {
+                val isInsideLink = linkRangeIndex < ranges.size && iterator.index >= ranges[linkRangeIndex].first
+                if (!isInsideLink) {
+                    val endIndex = nextClosers[iterator.index]
+                    if (endIndex == -1) {
+                        delegateIndices.put(iterator.index)
+                        iterator = iterator.advance()
+                        continue
+                    }
                     if (iterator.length == 1) {
-                        result.withNode(SequentialParser.Node(iterator.index..endIterator.index + 1, GFMElementTypes.INLINE_MATH))
+                        result.withNode(SequentialParser.Node(iterator.index..endIndex + 1, GFMElementTypes.INLINE_MATH))
                     } else {
-                        result.withNode(SequentialParser.Node(iterator.index..endIterator.index + 1, GFMElementTypes.BLOCK_MATH))
+                        result.withNode(SequentialParser.Node(iterator.index..endIndex + 1, GFMElementTypes.BLOCK_MATH))
                     }
-                    iterator = endIterator.advance()
+                    skipUntil = endIndex
+                    iterator = iterator.advance()
                     continue
                 }
             }
@@ -34,17 +56,74 @@ class MathParser : SequentialParser {
         return result.withFurtherProcessing(delegateIndices.get())
     }
 
-    private fun findOfSize(it: TokensCache.Iterator, length: Int): TokensCache.Iterator? {
-        var iterator = it
-        while (iterator.type != null) {
-            if (iterator.type == GFMTokenTypes.DOLLAR) {
-                if (iterator.length == length) {
-                    return iterator
+    private fun collectNextClosingIndices(
+        tokens: TokensCache,
+        rangesToGlue: List<IntRange>,
+        linkRanges: List<IntRange>,
+    ): IntArray {
+        val result = IntArray(tokens.filteredTokens.size) { -1 }
+        val nextClosingByLength = HashMap<Int, Int>()
+        var linkRangeIndex = linkRanges.lastIndex
+
+        for (range in rangesToGlue.asReversed()) {
+            for (index in range.last downTo range.first) {
+                val iterator = tokens.Iterator(index)
+                if (iterator.type != GFMTokenTypes.DOLLAR) continue
+
+                while (linkRangeIndex >= 0 && index < linkRanges[linkRangeIndex].first) {
+                    linkRangeIndex--
+                }
+                val isInsideLink = linkRangeIndex >= 0 && index <= linkRanges[linkRangeIndex].last
+
+                result[index] = nextClosingByLength[iterator.length] ?: -1
+                if (!isInsideLink && canCloseMath(iterator)) {
+                    nextClosingByLength[iterator.length] = index
                 }
             }
+        }
+        return result
+    }
+
+    private fun canOpenMath(iterator: TokensCache.Iterator): Boolean {
+        val previous = iterator.charLookup(-1)
+        return !isWhitespace(iterator.charLookup(1)) && !previous.isWordCharacter() && !isPunctuation(previous)
+    }
+
+    private fun canCloseMath(iterator: TokensCache.Iterator): Boolean {
+        return !isWhitespace(iterator.charLookup(-1)) && !iterator.charLookup(1).isWordCharacter()
+    }
 
+    private fun collectLinkRanges(tokens: TokensCache, rangesToGlue: List<IntRange>): List<IntRange> {
+        val result = ArrayList<IntRange>()
+        val inlineLinkStarts = LinkParserUtil.buildBracketStarts(tokens, rangesToGlue) {
+            it.rawLookup(1) == MarkdownTokenTypes.LPAREN
+        }
+        val referenceLinkStarts = LinkParserUtil.buildBracketStarts(tokens, rangesToGlue)
+        var iterator: TokensCache.Iterator = tokens.RangesListIterator(rangesToGlue)
+
+        while (iterator.type != null) {
+            if (iterator.type == MarkdownTokenTypes.LBRACKET) {
+                val link = if (iterator.index in inlineLinkStarts) {
+                    InlineLinkParser.parseInlineLink(iterator, inlineLinkStarts)
+                } else {
+                    null
+                } ?: if (iterator.index in referenceLinkStarts) {
+                    ReferenceLinkParser.parseReferenceLink(iterator)
+                } else {
+                    null
+                }
+                if (link != null) {
+                    result.add(iterator.index..link.iteratorPosition.index)
+                    iterator = link.iteratorPosition.advance()
+                    continue
+                }
+            }
             iterator = iterator.advance()
         }
-        return null
+        return result
+    }
+
+    private fun Char.isWordCharacter(): Boolean {
+        return isLetterOrDigit() || this == '_'
     }
 }

src/commonTest/kotlin/org/intellij/markdown/GfmTest.kt

@@ -41,4 +41,64 @@ class GfmTest: SpecTest(org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor(
             html = "<p>before <title> <script> after</p>"
         )
     }
+
+    @Test
+    fun testDollarInsideStrongIsNotMath() = doTest(
+        markdown = "**$0.85 EPS** (vs. $0.70 est.)",
+        html = "<p><strong>$0.85 EPS</strong> (vs. $0.70 est.)</p>"
+    )
+
+    @Test
+    fun testInlineMathDoesNotStartInsideWordsOrAfterPunctuation() = doPlainParagraphTests(
+        "foo$1+2$ bar",
+        ".$1+2$",
+        "-$1+2$",
+        "_$1+2$",
+        "/$1+2$",
+        "+$1+2$",
+        "@$1+2$",
+        "#$1+2$",
+    )
+
+    @Test
+    fun testInlineMathDoesNotEndBeforeWordCharacters() = doPlainParagraphTests(
+        "$1+2$3",
+        "$1+2\$a",
+        "\$x\$_",
+    )
+
+    @Test
+    fun testMathIsNotParsedInsideLinkText() = doTest(
+        markdown = "[$1+2$](https://example.com)",
+        html = "<p><a href=\"https://example.com\">$1+2$</a></p>"
+    )
+
+    @Test
+    fun testMathIsNotParsedInsideLinkDestination() = doTest(
+        markdown = "[x](https://example.com/$1+2$) $1+2$",
+        html = "<p><a href=\"https://example.com/$1+2$\">x</a> <span class=\"math\" inline = \"true\">1+2</span></p>"
+    )
+
+    @Test
+    fun testMathIsNotParsedInsideImageText() = doTest(
+        markdown = "![$1+2$](x.png)",
+        html = "<p><img src=\"x.png\" alt=\"$1+2$\" /></p>"
+    )
+
+    @Test
+    fun testBlockMathDoesNotStartInsideWords() = doPlainParagraphTests(
+        "foo\$\$x+y\$\$bar",
+    )
+
+    @Test
+    fun testBlockMathDoesNotEndBeforeWordCharacters() = doPlainParagraphTests(
+        "\$\$x+y\$\$3",
+        "\$\$x+y\$\$a",
+    )
+
+    private fun doPlainParagraphTests(vararg markdowns: String) {
+        markdowns.forEach { markdown ->
+            doTest(markdown = markdown, html = "<p>$markdown</p>")
+        }
+    }
 }

src/jvmTest/kotlin/org/intellij/markdown/ParserPerformanceTest.kt

@@ -1,20 +1,28 @@
 package org.intellij.markdown
 
 import junit.framework.TestCase
+import org.intellij.markdown.flavours.MarkdownFlavourDescriptor
 import org.intellij.markdown.flavours.commonmark.CommonMarkFlavourDescriptor
+import org.intellij.markdown.flavours.gfm.GFMFlavourDescriptor
 import org.intellij.markdown.parser.MarkdownParser
 import org.junit.experimental.categories.Category
 import java.io.File
-import kotlin.test.*
+import kotlin.test.Test
+import kotlin.test.assertTrue
 
 @Category(ParserPerformanceTest::class) class ParserPerformanceTest : TestCase() {
     protected fun getTestDataPath(): String {
         return File(getIntellijMarkdownHome() + "/src/jvmTest/resources/data/performance").absolutePath
     }
 
-    private fun assertFast(content: String, fullParse: Boolean, expectedTimeMs: Int? = 1000) {
+    private fun assertFast(
+        content: String,
+        fullParse: Boolean,
+        expectedTimeMs: Int? = 1000,
+        flavour: MarkdownFlavourDescriptor = CommonMarkFlavourDescriptor(),
+    ) {
         val runnable = { i: Int ->
-            val root = MarkdownParser(CommonMarkFlavourDescriptor()).
+            val root = MarkdownParser(flavour).
             parse(MarkdownElementTypes.MARKDOWN_FILE, content, fullParse)
             assert(root.children.size > 0)
         }
@@ -80,6 +88,20 @@ import kotlin.test.*
         assertFast("[a]: <" + "y".repeat(20_000_000), false)
     }
 
+    @Test
+    fun testRejectedMathClosersAreLinear() {
+        val input = (1..20_000).joinToString(" ") { "\$x\$1" }
+        assertFast(input, false, flavour = GFMFlavourDescriptor())
+    }
+
+    @Test
+    fun testMathLinkRangeLookupIsLinear() {
+        val input = (1..10_000).joinToString(" ") { i ->
+            "[link$i](url$i) \$x\$"
+        }
+        assertFast(input, false, flavour = GFMFlavourDescriptor())
+    }
+
     companion object {
         val WARM_UP_NUM = 10
         val TEST_NUM = 100