bpo-39209: Manage correctly multi-line tokens in interactive mode (GH… · python/cpython@5ec91f7

File tree

3 files changed

lines changed

  • Misc/NEWS.d/next/Core and Builtins

3 files changed

lines changed

Original file line numberDiff line numberDiff line change

@@ -58,5 +58,41 @@ def test_no_memory(self):

5858

# Exit code 120: Py_FinalizeEx() failed to flush stdout and stderr.

5959

self.assertIn(p.returncode, (1, 120))

6060
61+

@cpython_only

62+

def test_multiline_string_parsing(self):

63+

# bpo-39209: Multiline string tokens need to be handled in the tokenizer

64+

# in two places: the interactive path and the non-interactive path.

65+

user_input = '''\

66+

x = """<?xml version="1.0" encoding="iso-8859-1"?>

67+

<test>

68+

<Users>

69+

<fun25>

70+

<limits>

71+

<total>0KiB</total>

72+

<kbps>0</kbps>

73+

<rps>1.3</rps>

74+

<connections>0</connections>

75+

</limits>

76+

<usages>

77+

<total>16738211KiB</total>

78+

<kbps>237.15</kbps>

79+

<rps>1.3</rps>

80+

<connections>0</connections>

81+

</usages>

82+

<time_to_refresh>never</time_to_refresh>

83+

<limit_exceeded_URL>none</limit_exceeded_URL>

84+

</fun25>

85+

</Users>

86+

</test>"""

87+

'''

88+

user_input = dedent(user_input)

89+

user_input = user_input.encode()

90+

p = spawn_repl()

91+

with SuppressCrashReport():

92+

p.stdin.write(user_input)

93+

output = kill_python(p)

94+

self.assertEqual(p.returncode, 0)

95+
96+
6197

if __name__ == "__main__":

6298

unittest.main()

Original file line numberDiff line numberDiff line change

@@ -0,0 +1,2 @@

1+

Correctly handle multi-line tokens in interactive mode. Patch by Pablo

2+

Galindo.

Original file line numberDiff line numberDiff line change

@@ -886,6 +886,7 @@ tok_nextc(struct tok_state *tok)

886886

size_t start = tok->start - tok->buf;

887887

size_t oldlen = tok->cur - tok->buf;

888888

size_t newlen = oldlen + strlen(newtok);

889+

Py_ssize_t cur_multi_line_start = tok->multi_line_start - tok->buf;

889890

char *buf = tok->buf;

890891

buf = (char *)PyMem_REALLOC(buf, newlen+1);

891892

tok->lineno++;

@@ -898,6 +899,7 @@ tok_nextc(struct tok_state *tok)

898899

}

899900

tok->buf = buf;

900901

tok->cur = tok->buf + oldlen;

902+

tok->multi_line_start = tok->buf + cur_multi_line_start;

901903

tok->line_start = tok->cur;

902904

strcpy(tok->buf + oldlen, newtok);

903905

PyMem_FREE(newtok);