gh-151461: Fix encoding-related exception handling in file tokenizer … · python/cpython@f1a5f68

GitHub

@@ -1,6 +1,8 @@

11#include"Python.h"

22#include"errcode.h"

3+#include"pycore_runtime.h"// _Py_ID()

34#include"pycore_token.h"

5+#include"pycore_tuple.h"// _PyTuple_FromPair

4657#include"../lexer/state.h"

68@@ -149,6 +151,53 @@ _PyTokenizer_warn_invalid_escape_sequence(struct tok_state *tok, int first_inval

149151return0;

150152}

151153154+void

155+_PyTokenizer_raise_init_error(PyObject*filename)

156+{

157+if (!(PyErr_ExceptionMatches(PyExc_LookupError)

158+||PyErr_ExceptionMatches(PyExc_SyntaxError)

159+||PyErr_ExceptionMatches(PyExc_ValueError)

160+||PyErr_ExceptionMatches(PyExc_UnicodeDecodeError))) {

161+return;

162+ }

163+PyObject*errstr=NULL;

164+PyObject*tuple=NULL;

165+PyObject*type;

166+PyObject*value;

167+PyObject*tback;

168+PyErr_Fetch(&type, &value, &tback);

169+if (PyErr_GivenExceptionMatches(value, PyExc_SyntaxError)) {

170+if (PyObject_SetAttr(value, &_Py_ID(filename), filename)) {

171+ goto error;

172+ }

173+PyErr_Restore(type, value, tback);

174+return;

175+ }

176+errstr=PyObject_Str(value);

177+if (!errstr) {

178+ goto error;

179+ }

180+181+PyObject*tmp=Py_BuildValue("(OiiO)", filename, 0, -1, Py_None);

182+if (!tmp) {

183+ goto error;

184+ }

185+186+tuple=_PyTuple_FromPair(errstr, tmp);

187+Py_DECREF(tmp);

188+if (!tuple) {

189+ goto error;

190+ }

191+PyErr_SetObject(PyExc_SyntaxError, tuple);

192+193+error:

194+Py_XDECREF(type);

195+Py_XDECREF(value);

196+Py_XDECREF(tback);

197+Py_XDECREF(errstr);

198+Py_XDECREF(tuple);

199+}

200+152201int

153202_PyTokenizer_parser_warn(structtok_state*tok, PyObject*category, constchar*format, ...)

154203{

@@ -418,8 +467,8 @@ _PyTokenizer_check_coding_spec(const char* line, Py_ssize_t size, struct tok_sta

418467if (tok->encoding==NULL) {

419468assert(tok->decoding_readline==NULL);

420469if (strcmp(cs, "utf-8") !=0&& !set_readline(tok, cs)) {

470+_PyTokenizer_raise_init_error(tok->filename);

421471_PyTokenizer_error_ret(tok);

422-PyErr_Format(PyExc_SyntaxError, "encoding problem: %s", cs);

423472PyMem_Free(cs);

424473return0;

425474 }