gh-105259: Ensure we don't show newline characters for trailing NEWLINE tokens (#105364)

pablogsal · web-flow · commit c0a6ed39343b · 2023-06-06T12:52:16.000+01:00
diff --git a/Lib/test/test_tokenize.py b/Lib/test/test_tokenize.py
@@ -1870,7 +1870,7 @@ def readline(encoding):
             TokenInfo(type=NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1\n'),
             TokenInfo(type=OP, string='+', start=(1, 1), end=(1, 2), line='1+1\n'),
             TokenInfo(type=NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1\n'),
-            TokenInfo(type=NEWLINE, string='\n', start=(1, 3), end=(1, 4), line='1+1\n'),
+            TokenInfo(type=NEWLINE, string='', start=(1, 3), end=(1, 4), line='1+1\n'),
             TokenInfo(type=ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')
         ]
         for encoding in ["utf-8", "latin-1", "utf-16"]:
diff --git a/Misc/NEWS.d/next/Core and Builtins/2023-06-06-11-37-53.gh-issue-105259.E2BGKL.rst b/Misc/NEWS.d/next/Core and Builtins/2023-06-06-11-37-53.gh-issue-105259.E2BGKL.rst
@@ -0,0 +1,2 @@
+Don't include newline character for trailing ``NEWLINE`` tokens emitted in
+the :mod:`tokenize` module. Patch by Pablo Galindo
diff --git a/Parser/tokenizer.c b/Parser/tokenizer.c
@@ -114,6 +114,7 @@ tok_new(void)
     tok->report_warnings = 1;
     tok->tok_extra_tokens = 0;
     tok->comment_newline = 0;
+    tok->implicit_newline = 0;
     tok->tok_mode_stack[0] = (tokenizer_mode){.kind =TOK_REGULAR_MODE, .f_string_quote='\0', .f_string_quote_size = 0, .f_string_debug=0};
     tok->tok_mode_stack_index = 0;
     tok->tok_report_warnings = 1;
@@ -355,10 +356,12 @@ tok_concatenate_interactive_new_line(struct tok_state *tok, const char *line) {
         return -1;
     }
     strcpy(new_str + current_size, line);
+    tok->implicit_newline = 0;
     if (last_char != '\n') {
         /* Last line does not end in \n, fake one */
         new_str[current_size + line_size - 1] = '\n';
         new_str[current_size + line_size] = '\0';
+        tok->implicit_newline = 1;
     }
     tok->interactive_src_start = new_str;
     tok->interactive_src_end = new_str + current_size + line_size;
@@ -1262,11 +1265,13 @@ tok_underflow_file(struct tok_state *tok) {
         tok->done = E_EOF;
         return 0;
     }
+    tok->implicit_newline = 0;
     if (tok->inp[-1] != '\n') {
         assert(tok->inp + 1 < tok->end);
         /* Last line does not end in \n, fake one */
         *tok->inp++ = '\n';
         *tok->inp = '\0';
+        tok->implicit_newline = 1;
     }
 
     ADVANCE_LINENO();
@@ -1304,11 +1309,13 @@ tok_underflow_readline(struct tok_state* tok) {
         tok->done = E_EOF;
         return 0;
     }
+    tok->implicit_newline = 0;
     if (tok->inp[-1] != '\n') {
         assert(tok->inp + 1 < tok->end);
         /* Last line does not end in \n, fake one */
         *tok->inp++ = '\n';
         *tok->inp = '\0';
+        tok->implicit_newline = 1;
     }
 
     ADVANCE_LINENO();
diff --git a/Parser/tokenizer.h b/Parser/tokenizer.h
@@ -131,6 +131,7 @@ struct tok_state {
     int tok_report_warnings;
     int tok_extra_tokens;
     int comment_newline;
+    int implicit_newline;
 #ifdef Py_DEBUG
     int debug;
 #endif
diff --git a/Python/Python-tokenize.c b/Python/Python-tokenize.c
@@ -243,10 +243,12 @@ tokenizeriter_next(tokenizeriterobject *it)
         }
         else if (type == NEWLINE) {
             Py_DECREF(str);
-            if (it->tok->start[0] == '\r') {
-                str = PyUnicode_FromString("\r\n");
-            } else {
-                str = PyUnicode_FromString("\n");
+            if (!it->tok->implicit_newline) {
+                if (it->tok->start[0] == '\r') {
+                    str = PyUnicode_FromString("\r\n");
+                } else {
+                    str = PyUnicode_FromString("\n");
+                }
             }
             end_col_offset++;
         }

Original file line number	Diff line number	Diff line change
`@@ -1870,7 +1870,7 @@ def readline(encoding):`
`1870`	`1870`	`TokenInfo(type=NUMBER, string='1', start=(1, 0), end=(1, 1), line='1+1\n'),`
`1871`	`1871`	`TokenInfo(type=OP, string='+', start=(1, 1), end=(1, 2), line='1+1\n'),`
`1872`	`1872`	`TokenInfo(type=NUMBER, string='1', start=(1, 2), end=(1, 3), line='1+1\n'),`
`1873`		`- TokenInfo(type=NEWLINE, string='\n', start=(1, 3), end=(1, 4), line='1+1\n'),`
	`1873`	`+ TokenInfo(type=NEWLINE, string='', start=(1, 3), end=(1, 4), line='1+1\n'),`
`1874`	`1874`	`TokenInfo(type=ENDMARKER, string='', start=(2, 0), end=(2, 0), line='')`
`1875`	`1875`	`]`
`1876`	`1876`	`for encoding in ["utf-8", "latin-1", "utf-16"]:`
Original file line number	Diff line number	Diff line change
`@@ -0,0 +1,2 @@`
	`1`	+Don't include newline character for trailing ``NEWLINE`` tokens emitted in
	`2`	+the :mod:`tokenize` module. Patch by Pablo Galindo
Original file line number	Diff line number	Diff line change
`@@ -243,10 +243,12 @@ tokenizeriter_next(tokenizeriterobject *it)`
`243`	`243`	`}`
`244`	`244`	`else if (type == NEWLINE) {`
`245`	`245`	`Py_DECREF(str);`
`246`		`- if (it->tok->start[0] == '\r') {`
`247`		`- str = PyUnicode_FromString("\r\n");`
`248`		`- } else {`
`249`		`- str = PyUnicode_FromString("\n");`
	`246`	`+ if (!it->tok->implicit_newline) {`
	`247`	`+ if (it->tok->start[0] == '\r') {`
	`248`	`+ str = PyUnicode_FromString("\r\n");`
	`249`	`+ } else {`
	`250`	`+ str = PyUnicode_FromString("\n");`
	`251`	`+ }`
`250`	`252`	`}`
`251`	`253`	`end_col_offset++;`
`252`	`254`	`}`