@@ -2856,7 +2856,7 @@ describe("Grammar Tests", function() {
28562856
28572857 it("test/docstrings/continuation2.py",
28582858 function() {
2859- tokens = grammar.tokenizeLines("'\n'")
2859+ tokens = grammar.tokenizeLines("'\n'\n# comment ")
28602860 expect(tokens[0][0].value).toBe("'");
28612861 expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
28622862 expect(tokens[0][1].value).toBe("");
@@ -2865,6 +2865,10 @@ describe("Grammar Tests", function() {
28652865 expect(tokens[1][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
28662866 expect(tokens[1][1].value).toBe("");
28672867 expect(tokens[1][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]);
2868+ expect(tokens[2][0].value).toBe("#");
2869+ expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
2870+ expect(tokens[2][1].value).toBe(" comment");
2871+ expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
28682872 });
28692873
28702874 it("test/docstrings/continuation3.py",
@@ -3035,6 +3039,73 @@ describe("Grammar Tests", function() {
30353039 expect(tokens[12][1].scopes).toEqual(["source.python","string.quoted.docstring.raw.multi.python","punctuation.definition.string.end.python"]);
30363040 });
30373041
3042+ it("test/docstrings/continuation5.py",
3043+ function() {
3044+ tokens = grammar.tokenizeLines("'implicit ' \"concatenation\"\n\n'''implicit\n''' 'concatenation'\n\n'''implicit\n''' \"\"\"\nconcatenation\n\"\"\"\n\n'implicit' '''\nconcatenation\n'''")
3045+ expect(tokens[0][0].value).toBe("'");
3046+ expect(tokens[0][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3047+ expect(tokens[0][1].value).toBe("implicit ");
3048+ expect(tokens[0][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3049+ expect(tokens[0][2].value).toBe("'");
3050+ expect(tokens[0][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3051+ expect(tokens[0][3].value).toBe(" ");
3052+ expect(tokens[0][3].scopes).toEqual(["source.python"]);
3053+ expect(tokens[0][4].value).toBe("\"");
3054+ expect(tokens[0][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3055+ expect(tokens[0][5].value).toBe("concatenation");
3056+ expect(tokens[0][5].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3057+ expect(tokens[0][6].value).toBe("\"");
3058+ expect(tokens[0][6].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3059+ expect(tokens[1][0].value).toBe("");
3060+ expect(tokens[1][0].scopes).toEqual(["source.python"]);
3061+ expect(tokens[2][0].value).toBe("'''");
3062+ expect(tokens[2][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3063+ expect(tokens[2][1].value).toBe("implicit");
3064+ expect(tokens[2][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3065+ expect(tokens[3][0].value).toBe("'''");
3066+ expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3067+ expect(tokens[3][1].value).toBe(" ");
3068+ expect(tokens[3][1].scopes).toEqual(["source.python"]);
3069+ expect(tokens[3][2].value).toBe("'");
3070+ expect(tokens[3][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3071+ expect(tokens[3][3].value).toBe("concatenation");
3072+ expect(tokens[3][3].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3073+ expect(tokens[3][4].value).toBe("'");
3074+ expect(tokens[3][4].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3075+ expect(tokens[4][0].value).toBe("");
3076+ expect(tokens[4][0].scopes).toEqual(["source.python"]);
3077+ expect(tokens[5][0].value).toBe("'''");
3078+ expect(tokens[5][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3079+ expect(tokens[5][1].value).toBe("implicit");
3080+ expect(tokens[5][1].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3081+ expect(tokens[6][0].value).toBe("'''");
3082+ expect(tokens[6][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3083+ expect(tokens[6][1].value).toBe(" ");
3084+ expect(tokens[6][1].scopes).toEqual(["source.python"]);
3085+ expect(tokens[6][2].value).toBe("\"\"\"");
3086+ expect(tokens[6][2].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3087+ expect(tokens[7][0].value).toBe("concatenation");
3088+ expect(tokens[7][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3089+ expect(tokens[8][0].value).toBe("\"\"\"");
3090+ expect(tokens[8][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3091+ expect(tokens[9][0].value).toBe("");
3092+ expect(tokens[9][0].scopes).toEqual(["source.python"]);
3093+ expect(tokens[10][0].value).toBe("'");
3094+ expect(tokens[10][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
3095+ expect(tokens[10][1].value).toBe("implicit");
3096+ expect(tokens[10][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python"]);
3097+ expect(tokens[10][2].value).toBe("'");
3098+ expect(tokens[10][2].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.end.python"]);
3099+ expect(tokens[10][3].value).toBe(" ");
3100+ expect(tokens[10][3].scopes).toEqual(["source.python"]);
3101+ expect(tokens[10][4].value).toBe("'''");
3102+ expect(tokens[10][4].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.begin.python"]);
3103+ expect(tokens[11][0].value).toBe("concatenation");
3104+ expect(tokens[11][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python"]);
3105+ expect(tokens[12][0].value).toBe("'''");
3106+ expect(tokens[12][0].scopes).toEqual(["source.python","string.quoted.docstring.multi.python","punctuation.definition.string.end.python"]);
3107+ });
3108+
30383109 it("test/docstrings/def1.py",
30393110 function() {
30403111 tokens = grammar.tokenizeLines("def foo():\n '''TE\\'''ST'''\n\ndef foo():\n r'''TE\\'''ST'''\n\ndef foo():\n R'''TE\\'''ST'''\n\ndef foo():\n u'''TEST'''\n\ndef foo():\n U'''TEST'''\n\ndef foo():\n b'''TEST'''\n\ndef foo():\n B'''TEST'''")
@@ -6498,7 +6569,7 @@ describe("Grammar Tests", function() {
64986569
64996570 it("test/fstrings/nested3.py",
65006571 function() {
6501- tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"")
6572+ tokens = grammar.tokenizeLines("f\"result: {value:{60}.{16!s:2}{'qwerty'\n[2]}}\"\n# comment ")
65026573 expect(tokens[0][0].value).toBe("f");
65036574 expect(tokens[0][0].scopes).toEqual(["source.python","meta.fstring.python","storage.type.string.python string.quoted.single.python string.interpolated.python"]);
65046575 expect(tokens[0][1].value).toBe("\"");
@@ -6551,6 +6622,10 @@ describe("Grammar Tests", function() {
65516622 expect(tokens[1][4].scopes).toEqual(["source.python","string.quoted.single.python","punctuation.definition.string.begin.python"]);
65526623 expect(tokens[1][5].value).toBe("");
65536624 expect(tokens[1][5].scopes).toEqual(["source.python","string.quoted.single.python","invalid.illegal.newline.python"]);
6625+ expect(tokens[2][0].value).toBe("#");
6626+ expect(tokens[2][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
6627+ expect(tokens[2][1].value).toBe(" comment");
6628+ expect(tokens[2][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
65546629 });
65556630
65566631 it("test/fstrings/nested4.py",
@@ -9790,7 +9865,7 @@ describe("Grammar Tests", function() {
97909865
97919866 it("test/illegals/backticks3.py",
97929867 function() {
9793- tokens = grammar.tokenizeLines("a = lambda `123`")
9868+ tokens = grammar.tokenizeLines("a = lambda `123`\n# comment ")
97949869 expect(tokens[0][0].value).toBe("a");
97959870 expect(tokens[0][0].scopes).toEqual(["source.python"]);
97969871 expect(tokens[0][1].value).toBe(" ");
@@ -9811,6 +9886,10 @@ describe("Grammar Tests", function() {
98119886 expect(tokens[0][8].scopes).toEqual(["source.python","meta.lambda-function.python","meta.function.lambda.parameters.python","invalid.deprecated.backtick.python"]);
98129887 expect(tokens[0][9].value).toBe("");
98139888 expect(tokens[0][9].scopes).toEqual(["source.python","meta.lambda-function.python"]);
9889+ expect(tokens[1][0].value).toBe("#");
9890+ expect(tokens[1][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
9891+ expect(tokens[1][1].value).toBe(" comment");
9892+ expect(tokens[1][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
98149893 });
98159894
98169895 it("test/illegals/illegal1.py",
@@ -10951,7 +11030,7 @@ describe("Grammar Tests", function() {
1095111030
1095211031 it("test/regexp/python2.py",
1095311032 function() {
10954- tokens = grammar.tokenizeLines("a = r'\n (?x)\n foo\n'")
11033+ tokens = grammar.tokenizeLines("a = r'\n (?x)\n foo\n'\n# comment ")
1095511034 expect(tokens[0][0].value).toBe("a");
1095611035 expect(tokens[0][0].scopes).toEqual(["source.python"]);
1095711036 expect(tokens[0][1].value).toBe(" ");
@@ -10984,6 +11063,10 @@ describe("Grammar Tests", function() {
1098411063 expect(tokens[3][0].scopes).toEqual(["source.python","string.quoted.docstring.single.python","punctuation.definition.string.begin.python"]);
1098511064 expect(tokens[3][1].value).toBe("");
1098611065 expect(tokens[3][1].scopes).toEqual(["source.python","string.quoted.docstring.single.python","invalid.illegal.newline.python"]);
11066+ expect(tokens[4][0].value).toBe("#");
11067+ expect(tokens[4][0].scopes).toEqual(["source.python","comment.line.number-sign.python","punctuation.definition.comment.python"]);
11068+ expect(tokens[4][1].value).toBe(" comment");
11069+ expect(tokens[4][1].scopes).toEqual(["source.python","comment.line.number-sign.python"]);
1098711070 });
1098811071
1098911072 it("test/regexp/python3.py",
0 commit comments