Original file line numberDiff line numberDiff line change@@ -253,7 +253,7 @@ def testSyntaxErrorOffset(self):
253253check('try:\n pass\nexcept*:\n pass', 3, 8)
254254check('try:\n pass\nexcept*:\n pass\nexcept* ValueError:\n pass', 3, 8)
255255256-# Errors thrown by tokenizer.c
256+# Errors thrown by the tokenizer
257257check('(0x+1)', 1, 3)
258258check('x = 0xI', 1, 6)
259259check('0010 + 2', 1, 1)
Original file line numberDiff line numberDiff line change@@ -255,7 +255,7 @@ class UTF8ValidatorTest(unittest.TestCase):
255255deftest_invalid_utf8(self):
256256# This is a port of test_utf8_decode_invalid_sequences in
257257# test_unicode.py to exercise the separate utf8 validator in
258-# Parser/tokenizer.c used when reading source files.
258+# Parser/tokenizer/helpers.c used when reading source files.
259259260260# That file is written using low-level C file I/O, so the only way to
261261# test it is to write actual files to disk.
Original file line numberDiff line numberDiff line change@@ -1435,7 +1435,7 @@ def test_cookie_second_line_empty_first_line(self):
14351435self.assertEqual(consumed_lines, expected)
1436143614371437deftest_latin1_normalization(self):
1438-# See get_normal_name() in tokenizer.c.
1438+# See get_normal_name() in Parser/tokenizer/helpers.c.
14391439encodings= ("latin-1", "iso-8859-1", "iso-latin-1", "latin-1-unix",
14401440"iso-8859-1-unix", "iso-latin-1-mac")
14411441forencodinginencodings:
@@ -1460,7 +1460,7 @@ def test_syntaxerror_latin1(self):
146014601461146114621462deftest_utf8_normalization(self):
1463-# See get_normal_name() in tokenizer.c.
1463+# See get_normal_name() in Parser/tokenizer/helpers.c.
14641464encodings= ("utf-8", "utf-8-mac", "utf-8-unix")
14651465forencodinginencodings:
14661466forrepin ("-", "_"):
Original file line numberDiff line numberDiff line change@@ -298,7 +298,7 @@ def untokenize(iterable):
298298299299300300def_get_normal_name(orig_enc):
301-"""Imitates get_normal_name in tokenizer.c."""
301+"""Imitates get_normal_name in Parser/tokenizer/helpers.c."""
302302# Only care about the first 12 characters.
303303enc=orig_enc[:12].lower().replace("_", "-")
304304ifenc=="utf-8"orenc.startswith("utf-8-"):
Original file line numberDiff line numberDiff line change@@ -45,7 +45,7 @@ struct _inittab _PyImport_Inittab[] = {
4545/* This lives in Python/Python-ast.c */
4646 {"_ast", PyInit__ast},
474748-/* This lives in Python/Python-tokenizer.c */
48+/* This lives in Python/Python-tokenize.c */
4949 {"_tokenize", PyInit__tokenize},
50505151/* These entries are here for sys.builtin_module_names */
Original file line numberDiff line numberDiff line change@@ -1,5 +1,5 @@
112-/* Readline interface for tokenizer.c and [raw_]input() in bltinmodule.c.
2+/* Readline interface for the tokenizer and [raw_]input() in bltinmodule.c.
33 By default, or when stdin is not a tty device, we have a super
44 simple my_readline function using fgets.
55 Optionally, we can use the GNU readline library.
@@ -364,7 +364,7 @@ PyOS_StdioReadline(FILE *sys_stdin, FILE *sys_stdout, const char *prompt)
364364char*(*PyOS_ReadlineFunctionPointer)(FILE*, FILE*, constchar*) =NULL;
365365366366367-/* Interface used by tokenizer.c and bltinmodule.c */
367+/* Interface used by file_tokenizer.c and bltinmodule.c */
368368369369char*
370370PyOS_Readline(FILE*sys_stdin, FILE*sys_stdout, constchar*prompt)
Original file line numberDiff line numberDiff line change@@ -14,8 +14,9 @@ static int
1414warn_invalid_escape_sequence(Parser*p, constchar*first_invalid_escape, Token*t)
1515{
1616unsigned charc=*first_invalid_escape;
17-if ((t->type==FSTRING_MIDDLE||t->type==FSTRING_END) && (c=='{'||c=='}')) { // in this case the tokenizer has already emitted a warning,
18-// see tokenizer.c:warn_invalid_escape_sequence
17+if ((t->type==FSTRING_MIDDLE||t->type==FSTRING_END) && (c=='{'||c=='}')) {
18+// in this case the tokenizer has already emitted a warning,
19+// see Parser/tokenizer/helpers.c:warn_invalid_escape_sequence
1920return0;
2021 }
2122Original file line numberDiff line numberDiff line change@@ -32,7 +32,7 @@
3232#defineMAX_FRAME_DEPTH 100
3333#defineMAX_NTHREADS 100
343435-/* Function from Parser/tokenizer.c */
35+/* Function from Parser/tokenizer/file_tokenizer.c */
3636externchar*_PyTokenizer_FindEncodingFilename(int, PyObject*);
37373838/*[clinic input]
Original file line numberDiff line numberDiff line change@@ -428,8 +428,8 @@ Objects/typeobject.c:type_new():PyId___slots__ _Py_IDENTIFIER(
428428Objects/unicodeobject.c:unicodeiter_reduce():PyId_iter _Py_IDENTIFIER(iter)
429429Objects/weakrefobject.c:proxy_bytes():PyId___bytes__ _Py_IDENTIFIER(__bytes__)
430430Objects/weakrefobject.c:weakref_repr():PyId___name__ _Py_IDENTIFIER(__name__)
431-Parser/tokenizer.c:fp_setreadl():PyId_open _Py_IDENTIFIER(open)
432-Parser/tokenizer.c:fp_setreadl():PyId_readline _Py_IDENTIFIER(readline)
431+Parser/tokenizer/file_tokenizer.c:fp_setreadl():PyId_open _Py_IDENTIFIER(open)
432+Parser/tokenizer/file_tokenizer.c:fp_setreadl():PyId_readline _Py_IDENTIFIER(readline)
433433Python/Python-ast.c:ast_type_reduce():PyId___dict__ _Py_IDENTIFIER(__dict__)
434434Python/Python-ast.c:make_type():PyId___module__ _Py_IDENTIFIER(__module__)
435435Python/_warnings.c:PyId_stderr _Py_IDENTIFIER(stderr)
Original file line numberDiff line numberDiff line change@@ -123,7 +123,14 @@ def compile_c_extension(
123123common_sources= [
124124str(MOD_DIR.parent.parent.parent/"Python"/"Python-ast.c"),
125125str(MOD_DIR.parent.parent.parent/"Python"/"asdl.c"),
126-str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer.c"),
126+str(MOD_DIR.parent.parent.parent/"Parser"/"lexer"/"lexer.c"),
127+str(MOD_DIR.parent.parent.parent/"Parser"/"lexer"/"state.c"),
128+str(MOD_DIR.parent.parent.parent/"Parser"/"lexer"/"buffer.c"),
129+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"/"string_tokenizer.c"),
130+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"/"file_tokenizer.c"),
131+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"/"utf8_tokenizer.c"),
132+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"/"readline_tokenizer.c"),
133+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"/"helpers.c"),
127134str(MOD_DIR.parent.parent.parent/"Parser"/"pegen.c"),
128135str(MOD_DIR.parent.parent.parent/"Parser"/"pegen_errors.c"),
129136str(MOD_DIR.parent.parent.parent/"Parser"/"action_helpers.c"),
@@ -133,6 +140,8 @@ def compile_c_extension(
133140include_dirs= [
134141str(MOD_DIR.parent.parent.parent/"Include"/"internal"),
135142str(MOD_DIR.parent.parent.parent/"Parser"),
143+str(MOD_DIR.parent.parent.parent/"Parser"/"lexer"),
144+str(MOD_DIR.parent.parent.parent/"Parser"/"tokenizer"),
136145 ]
137146extension=Extension(
138147extension_name,