bpo-46841: Use inline caching for `COMPARE_OP` (GH-31622) · python/cpython@7820a58

GitHub

@@ -3671,36 +3671,37 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

36713671SET_TOP(res);

36723672Py_DECREF(left);

36733673Py_DECREF(right);

3674-if (res==NULL)

3674+if (res==NULL) {

36753675 goto error;

3676+ }

3677+JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);

36763678PREDICT(POP_JUMP_IF_FALSE);

36773679PREDICT(POP_JUMP_IF_TRUE);

36783680DISPATCH();

36793681 }

3680368236813683TARGET(COMPARE_OP_ADAPTIVE) {

36823684assert(cframe.use_tracing==0);

3683-SpecializedCacheEntry*cache=GET_CACHE();

3684-if (cache->adaptive.counter==0) {

3685+_PyCompareOpCache*cache=(_PyCompareOpCache*)next_instr;

3686+if (cache->counter==0) {

36853687PyObject*right=TOP();

36863688PyObject*left=SECOND();

36873689next_instr--;

3688-_Py_Specialize_CompareOp(left, right, next_instr, cache);

3690+_Py_Specialize_CompareOp(left, right, next_instr, oparg);

36893691DISPATCH();

36903692 }

36913693else {

36923694STAT_INC(COMPARE_OP, deferred);

3693-cache->adaptive.counter--;

3694-oparg=cache->adaptive.original_oparg;

3695+cache->counter--;

36953696JUMP_TO_INSTRUCTION(COMPARE_OP);

36963697 }

36973698 }

3698369936993700TARGET(COMPARE_OP_FLOAT_JUMP) {

37003701assert(cframe.use_tracing==0);

37013702// Combined: COMPARE_OP (float ? float) + POP_JUMP_IF_(true/false)

3702-SpecializedCacheEntry*caches=GET_CACHE();

3703-intwhen_to_jump_mask=caches[0].adaptive.index;

3703+_PyCompareOpCache*cache=(_PyCompareOpCache*)next_instr;

3704+intwhen_to_jump_mask=cache->mask;

37043705PyObject*right=TOP();

37053706PyObject*left=SECOND();

37063707DEOPT_IF(!PyFloat_CheckExact(left), COMPARE_OP);

@@ -3711,6 +3712,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

37113712DEOPT_IF(isnan(dleft), COMPARE_OP);

37123713DEOPT_IF(isnan(dright), COMPARE_OP);

37133714STAT_INC(COMPARE_OP, hit);

3715+JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);

37143716NEXTOPARG();

37153717STACK_SHRINK(2);

37163718Py_DECREF(left);

@@ -3731,8 +3733,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

37313733TARGET(COMPARE_OP_INT_JUMP) {

37323734assert(cframe.use_tracing==0);

37333735// Combined: COMPARE_OP (int ? int) + POP_JUMP_IF_(true/false)

3734-SpecializedCacheEntry*caches=GET_CACHE();

3735-intwhen_to_jump_mask=caches[0].adaptive.index;

3736+_PyCompareOpCache*cache=(_PyCompareOpCache*)next_instr;

3737+intwhen_to_jump_mask=cache->mask;

37363738PyObject*right=TOP();

37373739PyObject*left=SECOND();

37383740DEOPT_IF(!PyLong_CheckExact(left), COMPARE_OP);

@@ -3744,6 +3746,7 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

37443746Py_ssize_tileft=Py_SIZE(left) * ((PyLongObject*)left)->ob_digit[0];

37453747Py_ssize_tiright=Py_SIZE(right) * ((PyLongObject*)right)->ob_digit[0];

37463748intsign= (ileft>iright) - (ileft<iright);

3749+JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);

37473750NEXTOPARG();

37483751STACK_SHRINK(2);

37493752Py_DECREF(left);

@@ -3764,8 +3767,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

37643767TARGET(COMPARE_OP_STR_JUMP) {

37653768assert(cframe.use_tracing==0);

37663769// Combined: COMPARE_OP (str == str or str != str) + POP_JUMP_IF_(true/false)

3767-SpecializedCacheEntry*caches=GET_CACHE();

3768-intinvert=caches[0].adaptive.index;

3770+_PyCompareOpCache*cache=(_PyCompareOpCache*)next_instr;

3771+intinvert=cache->mask;

37693772PyObject*right=TOP();

37703773PyObject*left=SECOND();

37713774DEOPT_IF(!PyUnicode_CheckExact(left), COMPARE_OP);

@@ -3775,8 +3778,8 @@ _PyEval_EvalFrameDefault(PyThreadState *tstate, _PyInterpreterFrame *frame, int

37753778if (res<0) {

37763779 goto error;

37773780 }

3778-assert(caches[0].adaptive.original_oparg==Py_EQ||

3779-caches[0].adaptive.original_oparg==Py_NE);

3781+assert(oparg==Py_EQ||oparg==Py_NE);

3782+JUMPBY(INLINE_CACHE_ENTRIES_COMPARE_OP);

37803783NEXTOPARG();

37813784assert(opcode==POP_JUMP_IF_TRUE||opcode==POP_JUMP_IF_FALSE);

37823785STACK_SHRINK(2);

@@ -5601,7 +5604,7 @@ MISS_WITH_CACHE(LOAD_METHOD)

56015604MISS_WITH_CACHE(PRECALL)

56025605MISS_WITH_CACHE(CALL)

56035606MISS_WITH_INLINE_CACHE(BINARY_OP)

5604-MISS_WITH_CACHE(COMPARE_OP)

5607+MISS_WITH_INLINE_CACHE(COMPARE_OP)

56055608MISS_WITH_CACHE(BINARY_SUBSCR)

56065609MISS_WITH_INLINE_CACHE(UNPACK_SEQUENCE)

56075610MISS_WITH_OPARG_COUNTER(STORE_SUBSCR)