gh-155006: Encode an error handler's replacement strictly (GH-155008) · python/cpython@685ad02

GitHub

Original file line numberDiff line numberDiff line change@@ -1,3 +1,4 @@

1+import_codecs

12importcodecs

23importcontextlib

34importcopy

@@ -3735,6 +3736,17 @@ def test_encode_errors(self):

37353736self.assertEqual(codecs.iconv_encode(enc, 'a€b', 'xmlcharrefreplace')[0],

37363737b'a€b')

373737383739+deftest_encode_errors_unencodable_replacement(self):

3740+# Encoding the replacement must not call the error handler again.

3741+enc=self.require('ASCII')

3742+codecs.register_error('test.iconv', lambdaexc: ('€', exc.end))

3743+self.addCleanup(_codecs._unregister_error, 'test.iconv')

3744+withself.assertRaises(UnicodeEncodeError) ascm:

3745+codecs.iconv_encode(enc, 'a€b', 'test.iconv')

3746+self.assertEqual((cm.exception.start, cm.exception.end), (1, 2))

3747+self.assertEqual(cm.exception.reason,

3748+'unable to encode error handler result')

3749+37383750deftest_decode_errors(self):

37393751enc=self.require('ASCII')

37403752bad=b'a\xffb'

Original file line numberDiff line numberDiff line change@@ -8520,11 +8520,19 @@ _PyUnicode_EncodeIconv(const char *encoding, PyObject *unicode,

85208520replen=PyBytes_GET_SIZE(rep);

85218521 }

85228522else {

8523-/* A str replacement is encoded through the same codec. */

8523+/* A str replacement is encoded through the same codec, but

8524+ strictly: handling its errors in turn could never terminate. */

85248525assert(PyUnicode_Check(rep));

8525-repbytes=_PyUnicode_EncodeIconv(encoding, rep, errors);

8526+repbytes=_PyUnicode_EncodeIconv(encoding, rep, NULL);

85268527Py_DECREF(rep);

85278528if (repbytes==NULL) {

8529+if (PyErr_ExceptionMatches(PyExc_UnicodeEncodeError)) {

8530+/* Report the input the caller knows about, not the

8531+ replacement. */

8532+PyErr_Clear();

8533+raise_encode_exception(&exc, encoding, unicode, pos, pos+1,

8534+"unable to encode error handler result");

8535+ }

85288536 goto done;

85298537 }

85308538repdata=PyBytes_AS_STRING(repbytes);