Original file line numberDiff line numberDiff line change@@ -449,16 +449,6 @@ def _tokenize(rl_gen, encoding):
449449source=b"".join(rl_gen).decode(encoding)
450450token=None
451451fortokenin_generate_tokens_from_c_tokenizer(source, extra_tokens=True):
452-# TODO: Marta -> limpiar esto
453-if6<token.type<=54:
454-token=token._replace(type=OP)
455-iftoken.typein {ASYNC, AWAIT}:
456-token=token._replace(type=NAME)
457-iftoken.type==NEWLINE:
458-l_start, c_start=token.start
459-l_end, c_end=token.end
460-token=token._replace(string='\n', start=(l_start, c_start), end=(l_end, c_end+1))
461-462452yieldtoken
463453iftokenisnotNone:
464454last_line, _=token.start
@@ -550,8 +540,7 @@ def _generate_tokens_from_c_tokenizer(source, extra_tokens=False):
550540"""Tokenize a source reading Python code as unicode strings using the internal C tokenizer"""
551541import_tokenizeasc_tokenizer
552542forinfoinc_tokenizer.TokenizerIter(source, extra_tokens=extra_tokens):
553-tok, type, lineno, end_lineno, col_off, end_col_off, line=info
554-yieldTokenInfo(type, tok, (lineno, col_off), (end_lineno, end_col_off), line)
543+yieldTokenInfo._make(info)
555544556545557546if__name__=="__main__":
Original file line numberDiff line numberDiff line change@@ -207,7 +207,22 @@ tokenizeriter_next(tokenizeriterobject *it)
207207end_col_offset=_PyPegen_byte_offset_to_character_offset(line, token.end-it->tok->line_start);
208208 }
209209210-result=Py_BuildValue("(NinnnnN)", str, type, lineno, end_lineno, col_offset, end_col_offset, line);
210+if (it->tok->tok_extra_tokens) {
211+// Necessary adjustments to match the original Python tokenize
212+// implementation
213+if (type>DEDENT&&type<OP) {
214+type=OP;
215+ }
216+elseif (type==ASYNC||type==AWAIT) {
217+type=NAME;
218+ }
219+elseif (type==NEWLINE) {
220+str=PyUnicode_FromString("\n");
221+end_col_offset++;
222+ }
223+ }
224+225+result=Py_BuildValue("(iN(nn)(nn)N)", type, str, lineno, col_offset, end_lineno, end_col_offset, line);
211226exit:
212227_PyToken_Free(&token);
213228returnresult;