diff --git a/CHANGELOG.md b/CHANGELOG.md index 9876144e..0bd39806 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0. - Negative integer literals load as numbers again instead of `${-N}` expression strings, matching negative floats and the pre-8.x behaviour. ([#307](https://github.com/amplify-education/python-hcl2/issues/307)) - `strip_string_quotes` no longer unquotes string literals nested inside expressions, which produced invalid HCL such as `${upper(x)}` from `upper("x")`. ([#310](https://github.com/amplify-education/python-hcl2/issues/310)) - `strip_string_quotes` now resolves escape sequences, so the values it yields match what the option documents. Escapes naming a codepoint outside the Unicode range, or a lone surrogate, are preserved verbatim rather than raising. ([#308](https://github.com/amplify-education/python-hcl2/issues/308)) +- Parse files with CRLF (`\r\n`) line endings, including heredocs. A `\r` acting as part of a line ending is ignored, so a CRLF file reconstructs with LF endings; a `\r` that is content — inside a quoted string or a heredoc body — is preserved. ([#315](https://github.com/amplify-education/python-hcl2/issues/315)) ## \[8.1.2\] - 2026-04-10 diff --git a/hcl2/hcl2.lark b/hcl2/hcl2.lark index ee875319..1a6af857 100644 --- a/hcl2/hcl2.lark +++ b/hcl2/hcl2.lark @@ -82,11 +82,23 @@ ELLIPSIS : "..." COLONS: "::" // Heredocs -HEREDOC_TEMPLATE : /<<(?P[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc)\n/ -HEREDOC_TEMPLATE_TRIM : /<<-(?P[a-zA-Z][a-zA-Z0-9._-]+)\n(?:(?:.|\n)*?\n)??\s*(?P=heredoc_trim)\n/ - -// Ignore whitespace (but not newlines, as they're significant in HCL) -%ignore /[ \t]+/ +// \r? accepts CRLF line endings around the markers. %ignore cannot help here: +// it applies between tokens, never inside a terminal's own pattern, so without +// this a heredoc in a CRLF file fails to match at all. The body group is lazy +// and optional so an empty body matches without the delimiter search running on +// to a later marker. +HEREDOC_TEMPLATE : /<<(?P[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc)\r?\n/ +HEREDOC_TEMPLATE_TRIM : /<<-(?P[a-zA-Z][a-zA-Z0-9._-]+)\r?\n(?:(?:.|\n)*?\r?\n)??\s*(?P=heredoc_trim)\r?\n/ + +// Ignore whitespace (but not newlines, as they're significant in HCL). +// \r is ignored too so CRLF line endings (\r\n) parse the same as LF: the +// \r before a significant \n is swallowed here, leaving NL_OR_COMMENT to +// match the \n as usual. A \r that belongs to a terminal's own content is +// unaffected, because that terminal matches a longer span from the same +// position and wins: STRING_CHARS keeps a literal CR inside a quoted string, +// and heredoc bodies keep theirs. Structural \r around heredoc markers is +// handled by the terminals above, not here. +%ignore /[ \t\r]+/ // ============================================================================ // Rules diff --git a/hcl2/rules/strings.py b/hcl2/rules/strings.py index 0b4f71a4..8e335523 100644 --- a/hcl2/rules/strings.py +++ b/hcl2/rules/strings.py @@ -127,7 +127,9 @@ class HeredocTemplateRule(LarkRule): """Rule for heredoc template strings (< str: diff --git a/hcl2/utils.py b/hcl2/utils.py index 597b5e9c..4769ebc2 100644 --- a/hcl2/utils.py +++ b/hcl2/utils.py @@ -5,8 +5,11 @@ from dataclasses import dataclass, replace from typing import Optional, Tuple -HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S) -HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\n([\s\S]*)\1", re.S) +# \r? mirrors the heredoc terminals in hcl2.lark: these run against a token the +# grammar already accepted, so failing to match a CRLF heredoc here would raise +# on input that parsed cleanly. +HEREDOC_PATTERN = re.compile(r"<<([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S) +HEREDOC_TRIM_PATTERN = re.compile(r"<<-([a-zA-Z][a-zA-Z0-9._-]+)\r?\n([\s\S]*)\1", re.S) @dataclass diff --git a/test/unit/test_crlf.py b/test/unit/test_crlf.py new file mode 100644 index 00000000..f4ad7509 --- /dev/null +++ b/test/unit/test_crlf.py @@ -0,0 +1,130 @@ +# pylint: disable=C0103,C0114,C0115,C0116 +r"""Regression tests for GH issue #315: CRLF (\r\n) line endings fail to parse. + +`hcl2.lark`'s `%ignore` rule only skipped spaces and tabs, so a bare `\r` +preceding the newline in a CRLF-terminated line had no terminal that could +consume it: `NL_OR_COMMENT` only matches starting from `\n`. The `\r` fell +through to `STRING_CHARS` and the parse failed with `UnexpectedToken`, for +every construct, as soon as a single CRLF line appeared anywhere. + +Line endings are handled in three places, so the tests here cross module +boundaries rather than mirroring one: `%ignore` and the heredoc terminals in +`hcl2.lark`, the heredoc patterns in `hcl2/utils.py`, and the trim characters +in `hcl2/rules/strings.py`. +""" + +from unittest import TestCase + +from hcl2.api import loads, parses_to_tree, reconstruct, transform +from hcl2.utils import SerializationOptions + +CR = "\r" + + +class TestCrlfLineEndings(TestCase): + """A CRLF source parses to the same dict as its LF equivalent.""" + + def test_bare_attribute(self): + self.assertEqual(loads("a = 1\r\n"), loads("a = 1\n")) + + def test_block(self): + crlf = loads("locals {\r\n a = 1\r\n}\r\n") + lf = loads("locals {\n a = 1\n}\n") + self.assertEqual(crlf, lf) + + def test_quoted_string(self): + crlf = loads('a = "x"\r\n') + lf = loads('a = "x"\n') + self.assertEqual(crlf, lf) + + def test_single_crlf_line_amid_lf_lines(self): + crlf = loads("a = 1\nb = 2\r\nc = 3\n") + lf = loads("a = 1\nb = 2\nc = 3\n") + self.assertEqual(crlf, lf) + + def test_tuple_and_object(self): + crlf = loads("a = [1,\r\n2]\r\nb = {\r\n x = 1\r\n}\r\n") + lf = loads("a = [1,\n2]\nb = {\n x = 1\n}\n") + self.assertEqual(crlf, lf) + + def test_comments(self): + for source in ("# hi\r\na = 1\r\n", "// hi\r\na = 1\r\n", "/* hi */\r\na = 1\r\n"): + with self.subTest(source=source): + options = SerializationOptions(with_comments=True) + result = loads(source, serialization_options=options) + self.assertEqual(result["a"], 1) + self.assertEqual(result["__comments__"], [{"value": "hi"}]) + + +class TestCrlfDoesNotEatContentCarriageReturns(TestCase): + r"""Only a `\r` that is insignificant whitespace between tokens is ignored. + + A `\r` inside a terminal's own content belongs to that terminal, which + matches a longer span from the same position and so wins. + """ + + def test_real_cr_inside_a_quoted_string_survives(self): + result = loads('a = "x' + CR + 'y"\n') + self.assertEqual(result, {"a": '"x' + CR + 'y"'}) + + def test_string_consisting_only_of_a_cr_survives(self): + self.assertEqual(loads('a = "' + CR + '"\n'), {"a": '"' + CR + '"'}) + + def test_cr_escape_sequence_is_untouched(self): + r"""`\r` as escape *text* is two ASCII characters, never at risk.""" + self.assertEqual(loads(r'a = "line1\r\nline2"' + "\n"), {"a": r'"line1\r\nline2"'}) + + def test_heredoc_body_keeps_its_carriage_returns(self): + r"""HCL treats `\r` around markers as structure but body text as content.""" + result = loads("a = <