From 2032778a691b6874428f6c741d1d798919ec6eac Mon Sep 17 00:00:00 2001 From: soyitiel Date: Mon, 12 Apr 2021 14:12:49 -0500 Subject: [PATCH 1/5] python parser can now indetify triple-quoted string comments --- comment_parser/parsers/python_parser.py | 36 ++++++++++++++++++++++--- 1 file changed, 33 insertions(+), 3 deletions(-) diff --git a/comment_parser/parsers/python_parser.py b/comment_parser/parsers/python_parser.py index e3e6fa3..92e2cca 100644 --- a/comment_parser/parsers/python_parser.py +++ b/comment_parser/parsers/python_parser.py @@ -5,12 +5,29 @@ import tokenize from comment_parser.parsers import common - def extract_comments(code): """Extracts a list of comments from the given Python script. - Comments are identified using the tokenize module. Does not include function, - class, or module docstrings. All comments are single line comments. + Comments are identified using the tokenize module. + - Single-lined comments which begin with the '#' character and end with a line-break. + - Multi-lined comments or docstrings, which are just triple-quoted strings (start + and end with ''' or 3 of these "), are told apart from regular strings by the + type of the previous token which should be a line-break or an indentation (NEWLINE, + NL, INDENT or DEDENT) or no token at all (it would mean it's the fisrt thing in + the script). Even in cases like this: + + my_string = \ + '''this should not be considered a comment''' + + my_string = \ + '''this should not either''' # <- notice the increasing indentation + + my_string = \ + '''weird syntax anyway''' # <- but still valid indentation + + the previous token to the string is the '=' operator and not a line-break or an + indentation. That way, only triple-quoted strings preceded by a line-break, an + indentation, or no token, will be considered intended as comments. Args: code: String containing code to extract comments from. @@ -19,11 +36,24 @@ def extract_comments(code): Raises: tokenize.TokenError """ + triplequotes = ['"""', "'''"] + multicommprevnums = [tokenize.NEWLINE, tokenize.NL, tokenize.INDENT, tokenize.DEDENT] + prevtoknum = None # Stores the previous token's type. comments = [] tokens = tokenize.tokenize(io.BytesIO(code.encode()).readline) for toknum, tokstring, tokloc, _, _ in tokens: + # Single-lined comment. if toknum is tokenize.COMMENT: # Removes leading '#' character. tokstring = tokstring[1:] comments.append(common.Comment(tokstring, tokloc[0], False)) + continue + # Multi-lined comment. + if toknum is tokenize.STRING: + if tokstring[:3] in triplequotes and tokstring[-3:] in triplequotes: + if (not prevtoknum) or prevtoknum in multicommprevnums: + # Removes the leading and preceding 3ple quotes (""" or '''). + tokstring = tokstring[3:-3] + comments.append(common.Comment(tokstring, tokloc[0], True)) + prevtoknum = toknum return comments From 5ffd0540d2ab32c40689365cf0dc3eb233c80a39 Mon Sep 17 00:00:00 2001 From: Konrad Weihmann Date: Tue, 21 Jun 2022 17:17:02 +0200 Subject: [PATCH 2/5] python_parser: add tokenize.ENCODING to parse triple quote comments even at the evry beginning of the file Signed-off-by: Konrad Weihmann --- comment_parser/parsers/python_parser.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comment_parser/parsers/python_parser.py b/comment_parser/parsers/python_parser.py index 92e2cca..4b7cfd5 100644 --- a/comment_parser/parsers/python_parser.py +++ b/comment_parser/parsers/python_parser.py @@ -37,7 +37,7 @@ def extract_comments(code): tokenize.TokenError """ triplequotes = ['"""', "'''"] - multicommprevnums = [tokenize.NEWLINE, tokenize.NL, tokenize.INDENT, tokenize.DEDENT] + multicommprevnums = [tokenize.ENCODING, tokenize.NEWLINE, tokenize.NL, tokenize.INDENT, tokenize.DEDENT] prevtoknum = None # Stores the previous token's type. comments = [] tokens = tokenize.tokenize(io.BytesIO(code.encode()).readline) From 3f8deab6bb3bce114faceea33a1cfaa7964fb9d6 Mon Sep 17 00:00:00 2001 From: Konrad Weihmann Date: Tue, 21 Jun 2022 17:17:30 +0200 Subject: [PATCH 3/5] tests: triple quote comments in python Signed-off-by: Konrad Weihmann --- .../parsers/tests/python_parser_test.py | 44 +++++++++++++++++++ 1 file changed, 44 insertions(+) diff --git a/comment_parser/parsers/tests/python_parser_test.py b/comment_parser/parsers/tests/python_parser_test.py index f6f1a38..41d2db8 100644 --- a/comment_parser/parsers/tests/python_parser_test.py +++ b/comment_parser/parsers/tests/python_parser_test.py @@ -45,3 +45,47 @@ def testEscapedDoubleQuote(self): comments = python_parser.extract_comments(code) expected = [common.Comment(code[3:], 1, multiline=False)] self.assertEqual(comments, expected) + + def testTripleQuoteCommentsDoubleQuoteMultiline(self): + code = '"""this is triple quote comment\n' + code += 'with\n' + code += 'multiple\n' + code += 'lines\n' + code += '"""' + comments = python_parser.extract_comments(code) + import logging + logging.warning(comments) + logging.warning(code) + expected = [common.Comment(code.strip('"'), 1, multiline=True)] + self.assertEqual(comments, expected) + + def testTripleQuoteCommentsDoubleQuoteSingleline(self): + code = '"""this is triple quote comment"""' + comments = python_parser.extract_comments(code) + import logging + logging.warning(comments) + logging.warning(code) + expected = [common.Comment(code.strip('"'), 1, multiline=True)] + self.assertEqual(comments, expected) + + def testTripleQuoteCommentsSingleQuoteMultiline(self): + code = '\'\'\'this is triple quote comment\n' + code += 'with\n' + code += 'multiple\n' + code += 'lines\n' + code += '\'\'\'' + comments = python_parser.extract_comments(code) + import logging + logging.warning(comments) + logging.warning(code) + expected = [common.Comment(code.strip('\''), 1, multiline=True)] + self.assertEqual(comments, expected) + + def testTripleQuoteCommentsSingleQuoteSingleline(self): + code = '\'\'\'this is triple quote comment\'\'\'' + comments = python_parser.extract_comments(code) + import logging + logging.warning(comments) + logging.warning(code) + expected = [common.Comment(code.strip('\''), 1, multiline=True)] + self.assertEqual(comments, expected) From d340dca3d7e0a96b3276f2758bc597eefc1e7d98 Mon Sep 17 00:00:00 2001 From: Konrad Weihmann Date: Tue, 21 Jun 2022 17:44:02 +0200 Subject: [PATCH 4/5] python_parser: fix pylint issues and a typo Signed-off-by: Konrad Weihmann --- comment_parser/parsers/python_parser.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/comment_parser/parsers/python_parser.py b/comment_parser/parsers/python_parser.py index 4b7cfd5..6eca416 100644 --- a/comment_parser/parsers/python_parser.py +++ b/comment_parser/parsers/python_parser.py @@ -10,12 +10,12 @@ def extract_comments(code): Comments are identified using the tokenize module. - Single-lined comments which begin with the '#' character and end with a line-break. - - Multi-lined comments or docstrings, which are just triple-quoted strings (start - and end with ''' or 3 of these "), are told apart from regular strings by the - type of the previous token which should be a line-break or an indentation (NEWLINE, - NL, INDENT or DEDENT) or no token at all (it would mean it's the fisrt thing in - the script). Even in cases like this: - + - Multi-lined comments or docstrings, which are just triple-quoted strings (start + and end with ''' or 3 of these "), are told apart from regular strings by the + type of the previous token which should be a line-break or an indentation (NEWLINE, + NL, INDENT or DEDENT) or no token at all (it would mean it's the first thing in + the script). Even in cases like this: + my_string = \ '''this should not be considered a comment''' @@ -24,9 +24,9 @@ def extract_comments(code): my_string = \ '''weird syntax anyway''' # <- but still valid indentation - - the previous token to the string is the '=' operator and not a line-break or an - indentation. That way, only triple-quoted strings preceded by a line-break, an + + the previous token to the string is the '=' operator and not a line-break or an + indentation. That way, only triple-quoted strings preceded by a line-break, an indentation, or no token, will be considered intended as comments. Args: @@ -37,7 +37,8 @@ def extract_comments(code): tokenize.TokenError """ triplequotes = ['"""', "'''"] - multicommprevnums = [tokenize.ENCODING, tokenize.NEWLINE, tokenize.NL, tokenize.INDENT, tokenize.DEDENT] + multicommprevnums = [tokenize.ENCODING, tokenize.NEWLINE, tokenize.NL, + tokenize.INDENT, tokenize.DEDENT] prevtoknum = None # Stores the previous token's type. comments = [] tokens = tokenize.tokenize(io.BytesIO(code.encode()).readline) From 2aad4606ff8736d0eb29fca05c35c5cf8b29a3a6 Mon Sep 17 00:00:00 2001 From: Konrad Weihmann Date: Tue, 21 Jun 2022 17:49:10 +0200 Subject: [PATCH 5/5] apply yapf styleguide Signed-off-by: Konrad Weihmann --- comment_parser/parsers/python_parser.py | 9 ++++++--- comment_parser/parsers/tests/python_parser_test.py | 4 ++-- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/comment_parser/parsers/python_parser.py b/comment_parser/parsers/python_parser.py index 6eca416..103287d 100644 --- a/comment_parser/parsers/python_parser.py +++ b/comment_parser/parsers/python_parser.py @@ -5,6 +5,7 @@ import tokenize from comment_parser.parsers import common + def extract_comments(code): """Extracts a list of comments from the given Python script. @@ -37,9 +38,11 @@ def extract_comments(code): tokenize.TokenError """ triplequotes = ['"""', "'''"] - multicommprevnums = [tokenize.ENCODING, tokenize.NEWLINE, tokenize.NL, - tokenize.INDENT, tokenize.DEDENT] - prevtoknum = None # Stores the previous token's type. + multicommprevnums = [ + tokenize.ENCODING, tokenize.NEWLINE, tokenize.NL, tokenize.INDENT, + tokenize.DEDENT + ] + prevtoknum = None # Stores the previous token's type. comments = [] tokens = tokenize.tokenize(io.BytesIO(code.encode()).readline) for toknum, tokstring, tokloc, _, _ in tokens: diff --git a/comment_parser/parsers/tests/python_parser_test.py b/comment_parser/parsers/tests/python_parser_test.py index 41d2db8..4930345 100644 --- a/comment_parser/parsers/tests/python_parser_test.py +++ b/comment_parser/parsers/tests/python_parser_test.py @@ -58,7 +58,7 @@ def testTripleQuoteCommentsDoubleQuoteMultiline(self): logging.warning(code) expected = [common.Comment(code.strip('"'), 1, multiline=True)] self.assertEqual(comments, expected) - + def testTripleQuoteCommentsDoubleQuoteSingleline(self): code = '"""this is triple quote comment"""' comments = python_parser.extract_comments(code) @@ -80,7 +80,7 @@ def testTripleQuoteCommentsSingleQuoteMultiline(self): logging.warning(code) expected = [common.Comment(code.strip('\''), 1, multiline=True)] self.assertEqual(comments, expected) - + def testTripleQuoteCommentsSingleQuoteSingleline(self): code = '\'\'\'this is triple quote comment\'\'\'' comments = python_parser.extract_comments(code)