@@ -12,6 +12,16 @@
1212#include"tokenizer/helpers.h"
1313#include"pegen.h"
141415+#defineIDENTIFIER_CACHE_SIZE 2048 // Must be a power of two.
16+#defineIDENTIFIER_CACHE_MAX_PROBES 8
17+18+struct_identifier_cache_entry {
19+constchar*key; // Borrowed from arena-owned token bytes.
20+Py_ssize_tlen;
21+Py_hash_thash;
22+PyObject*value; // Borrowed from an arena-owned identifier.
23+};
24+1525// Internal parser functions
16261727asdl_stmt_seq*
@@ -572,11 +582,44 @@ _PyPegen_name_from_token(Parser *p, Token* t)
572582p->error_indicator=1;
573583returnNULL;
574584 }
585+// Identifiers repeat constantly; a small span-keyed cache skips the
586+// UTF-8 decode + intern for repeated occurrences. Keys point into
587+// arena-owned token bytes and values are arena-owned interned strings,
588+// so borrowed references are valid for the lifetime of the parse
589+// (including the second error pass, which reuses parser and arena).
590+Py_ssize_tlen=PyBytes_GET_SIZE(t->bytes);
591+Py_hash_thash=PyObject_Hash(t->bytes);
592+if (hash==-1) {
593+p->error_indicator=1;
594+returnNULL;
595+ }
596+IdentifierCacheEntry*free_slot=NULL;
597+size_tidx= (size_t)hash& (IDENTIFIER_CACHE_SIZE-1);
598+for (intprobe=0; probe<IDENTIFIER_CACHE_MAX_PROBES; probe++) {
599+IdentifierCacheEntry*entry=&p->identifier_cache[
600+ (idx+probe) & (IDENTIFIER_CACHE_SIZE-1)];
601+if (entry->key==NULL) {
602+free_slot=entry;
603+break;
604+ }
605+if (entry->hash==hash&&entry->len==len&&
606+memcmp(entry->key, s, len) ==0)
607+ {
608+return_PyAST_Name(entry->value, Load, t->lineno, t->col_offset,
609+t->end_lineno, t->end_col_offset, p->arena);
610+ }
611+ }
575612PyObject*id=_PyPegen_new_identifier(p, s);
576613if (id==NULL) {
577614p->error_indicator=1;
578615returnNULL;
579616 }
617+if (free_slot!=NULL) {
618+free_slot->key=s;
619+free_slot->len=len;
620+free_slot->hash=hash;
621+free_slot->value=id;
622+ }
580623return_PyAST_Name(id, Load, t->lineno, t->col_offset, t->end_lineno,
581624t->end_col_offset, p->arena);
582625}
@@ -844,6 +887,15 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
844887p->flags=flags;
845888p->feature_version=feature_version;
846889p->known_err_token=NULL;
890+p->identifier_cache=PyMem_Calloc(
891+IDENTIFIER_CACHE_SIZE, sizeof(*p->identifier_cache));
892+if (p->identifier_cache==NULL) {
893+growable_comment_array_deallocate(&p->type_ignore_comments);
894+PyMem_Free(p->tokens[0]);
895+PyMem_Free(p->tokens);
896+PyMem_Free(p);
897+return (Parser*) PyErr_NoMemory();
898+ }
847899p->level=0;
848900p->call_invalid_rules=0;
849901p->last_stmt_location.lineno=0;
@@ -859,6 +911,7 @@ _PyPegen_Parser_New(struct tok_state *tok, int start_rule, int flags,
859911void
860912_PyPegen_Parser_Free(Parser*p)
861913{
914+PyMem_Free(p->identifier_cache);
862915Py_XDECREF(p->normalize);
863916for (inti=0; i<p->size; i++) {
864917PyMem_Free(p->tokens[i]);