From f8668169b663233bea08f502ab3957e78c7a9777 Mon Sep 17 00:00:00 2001 From: Toshio Kuratomi Date: Aug 22 2010 14:43:31 +0000 Subject: Merge changes from a subsequent revision that fixes lone surrogate detection compilation with wide unicode builds. --- diff --git a/python3-r80382-lone-surrogate-and-utf8-error-handler.patch b/python3-r80382-lone-surrogate-and-utf8-error-handler.patch deleted file mode 100644 index 6287d7f..0000000 --- a/python3-r80382-lone-surrogate-and-utf8-error-handler.patch +++ /dev/null @@ -1,172 +0,0 @@ -Index: Python-3.1.2/Objects/unicodeobject.c -=================================================================== ---- Python-3.1.2.orig/Objects/unicodeobject.c -+++ Python-3.1.2/Objects/unicodeobject.c -@@ -159,6 +159,12 @@ static PyObject *unicode_encode_call_err - const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, - Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); - -+static void raise_encode_exception(PyObject **exceptionObject, -+ const char *encoding, -+ const Py_UNICODE *unicode, Py_ssize_t size, -+ Py_ssize_t startpos, Py_ssize_t endpos, -+ const char *reason); -+ - /* Same for linebreaks */ - static unsigned char ascii_linebreak[] = { - 0, 0, 0, 0, 0, 0, 0, 0, -@@ -2461,59 +2467,88 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s - /* Encode Latin-1 */ - *p++ = (char)(0xc0 | (ch >> 6)); - *p++ = (char)(0x80 | (ch & 0x3f)); -- } -- else { -- /* Encode UCS2 Unicode ordinals */ -- if (ch < 0x10000) { -+ } else if (0xD800 <= ch && ch <= 0xDFFF) { - #ifndef Py_UNICODE_WIDE -- /* Special case: check for high surrogate */ -- if (0xD800 <= ch && ch <= 0xDBFF && i != size) { -- Py_UCS4 ch2 = s[i]; -- /* Check for low surrogate and combine the two to -- form a UCS4 value */ -- if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { -- ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; -- i++; -- goto encodeUCS4; -- } -- /* Fall through: handles isolated high surrogates */ -- } -+ /* Special case: check for high and low surrogate */ -+ if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) { -+ Py_UCS4 ch2 = s[i]; -+ /* Combine the two surrogates to form a UCS4 value */ -+ ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; -+ i++; -+ -+ /* Encode UCS4 Unicode ordinals */ -+ *p++ = (char)(0xf0 | (ch >> 18)); -+ *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); -+ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); -+ *p++ = (char)(0x80 | (ch & 0x3f)); -+ - #endif -- if (ch >= 0xd800 && ch <= 0xdfff) { -- Py_ssize_t newpos; -- PyObject *rep; -- char *prep; -- int k; -- rep = unicode_encode_call_errorhandler -- (errors, &errorHandler, "utf-8", "surrogates not allowed", -- s, size, &exc, i-1, i, &newpos); -- if (!rep) -- goto error; -- /* Implementation limitations: only support error handler that return -- bytes, and only support up to four replacement bytes. */ -- if (!PyBytes_Check(rep)) { -- PyErr_SetString(PyExc_TypeError, "error handler should have returned bytes"); -- Py_DECREF(rep); -+ } else { -+ Py_ssize_t newpos; -+ PyObject *rep; -+ Py_ssize_t repsize, k; -+ rep = unicode_encode_call_errorhandler -+ (errors, &errorHandler, "utf-8", "surrogates not allowed", -+ s, size, &exc, i-1, i, &newpos); -+ if (!rep) -+ goto error; -+ -+ if (PyBytes_Check(rep)) -+ repsize = PyBytes_GET_SIZE(rep); -+ else -+ repsize = PyUnicode_GET_SIZE(rep); -+ -+ if (repsize > 4) { -+ Py_ssize_t offset; -+ -+ if (result == NULL) -+ offset = p - stackbuf; -+ else -+ offset = p - PyBytes_AS_STRING(result); -+ -+ if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { -+ /* integer overflow */ -+ PyErr_NoMemory(); - goto error; - } -- if (PyBytes_Size(rep) > 4) { -- PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes"); -- Py_DECREF(rep); -- goto error; -+ nallocated += repsize - 4; -+ if (result != NULL) { -+ if (_PyBytes_Resize(&result, nallocated) < 0) -+ goto error; -+ } else { -+ result = PyBytes_FromStringAndSize(NULL, nallocated); -+ if (result == NULL) -+ goto error; -+ Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); - } -- prep = PyBytes_AsString(rep); -- for(k = PyBytes_Size(rep); k > 0; k--) -+ p = PyBytes_AS_STRING(result) + offset; -+ } -+ -+ if (PyBytes_Check(rep)) { -+ char *prep = PyBytes_AS_STRING(rep); -+ for(k = repsize; k > 0; k--) - *p++ = *prep++; -- Py_DECREF(rep); -- continue; -- -+ } else /* rep is unicode */ { -+ Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); -+ Py_UNICODE c; -+ -+ for(k=0; k> 12)); -- *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); -- *p++ = (char)(0x80 | (ch & 0x3f)); -- continue; -+ Py_DECREF(rep); - } -- encodeUCS4: -+ } else if (ch < 0x10000) { -+ *p++ = (char)(0xe0 | (ch >> 12)); -+ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); -+ *p++ = (char)(0x80 | (ch & 0x3f)); -+ } else /* ch >= 0x10000 */ { - /* Encode UCS4 Unicode ordinals */ - *p++ = (char)(0xf0 | (ch >> 18)); - *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); -Index: Python-3.1.2/Lib/test/test_codecs.py -=================================================================== ---- Python-3.1.2.orig/Lib/test/test_codecs.py -+++ Python-3.1.2/Lib/test/test_codecs.py -@@ -571,6 +571,16 @@ class UTF8Test(ReadTest): - def test_lone_surrogates(self): - self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8") - self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8") -+ self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"), -+ b'[\\udc80]') -+ self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"), -+ b'[�]') -+ self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"), -+ b'[\x80]') -+ self.assertEqual("[\uDC80]".encode("utf-8", "ignore"), -+ b'[]') -+ self.assertEqual("[\uDC80]".encode("utf-8", "replace"), -+ b'[?]') - - def test_surrogatepass_handler(self): - self.assertEquals("abc\ud800def".encode("utf-8", "surrogatepass"), diff --git a/python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch b/python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch new file mode 100644 index 0000000..93420a8 --- /dev/null +++ b/python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch @@ -0,0 +1,184 @@ +Index: Python-3.1.2/Objects/unicodeobject.c +=================================================================== +--- Python-3.1.2.orig/Objects/unicodeobject.c ++++ Python-3.1.2/Objects/unicodeobject.c +@@ -159,6 +159,12 @@ static PyObject *unicode_encode_call_err + const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject, + Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos); + ++static void raise_encode_exception(PyObject **exceptionObject, ++ const char *encoding, ++ const Py_UNICODE *unicode, Py_ssize_t size, ++ Py_ssize_t startpos, Py_ssize_t endpos, ++ const char *reason); ++ + /* Same for linebreaks */ + static unsigned char ascii_linebreak[] = { + 0, 0, 0, 0, 0, 0, 0, 0, +@@ -2453,67 +2459,98 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s + for (i = 0; i < size;) { + Py_UCS4 ch = s[i++]; + +- if (ch < 0x80) ++ if (ch < 0x80) { + /* Encode ASCII */ + *p++ = (char) ch; + +- else if (ch < 0x0800) { ++ } else if (ch < 0x0800) { + /* Encode Latin-1 */ + *p++ = (char)(0xc0 | (ch >> 6)); + *p++ = (char)(0x80 | (ch & 0x3f)); +- } +- else { +- /* Encode UCS2 Unicode ordinals */ +- if (ch < 0x10000) { ++ } else if (0xD800 <= ch && ch <= 0xDFFF) { + #ifndef Py_UNICODE_WIDE +- /* Special case: check for high surrogate */ +- if (0xD800 <= ch && ch <= 0xDBFF && i != size) { +- Py_UCS4 ch2 = s[i]; +- /* Check for low surrogate and combine the two to +- form a UCS4 value */ +- if (0xDC00 <= ch2 && ch2 <= 0xDFFF) { +- ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; +- i++; +- goto encodeUCS4; +- } +- /* Fall through: handles isolated high surrogates */ +- } ++ /* Special case: check for high and low surrogate */ ++ if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) { ++ Py_UCS4 ch2 = s[i]; ++ /* Combine the two surrogates to form a UCS4 value */ ++ ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000; ++ i++; ++ ++ /* Encode UCS4 Unicode ordinals */ ++ *p++ = (char)(0xf0 | (ch >> 18)); ++ *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); ++ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); ++ *p++ = (char)(0x80 | (ch & 0x3f)); ++ ++ } else { + #endif +- if (ch >= 0xd800 && ch <= 0xdfff) { +- Py_ssize_t newpos; +- PyObject *rep; +- char *prep; +- int k; +- rep = unicode_encode_call_errorhandler +- (errors, &errorHandler, "utf-8", "surrogates not allowed", +- s, size, &exc, i-1, i, &newpos); +- if (!rep) +- goto error; +- /* Implementation limitations: only support error handler that return +- bytes, and only support up to four replacement bytes. */ +- if (!PyBytes_Check(rep)) { +- PyErr_SetString(PyExc_TypeError, "error handler should have returned bytes"); +- Py_DECREF(rep); ++ Py_ssize_t newpos; ++ PyObject *rep; ++ Py_ssize_t repsize, k; ++ rep = unicode_encode_call_errorhandler ++ (errors, &errorHandler, "utf-8", "surrogates not allowed", ++ s, size, &exc, i-1, i, &newpos); ++ if (!rep) ++ goto error; ++ ++ if (PyBytes_Check(rep)) ++ repsize = PyBytes_GET_SIZE(rep); ++ else ++ repsize = PyUnicode_GET_SIZE(rep); ++ ++ if (repsize > 4) { ++ Py_ssize_t offset; ++ ++ if (result == NULL) ++ offset = p - stackbuf; ++ else ++ offset = p - PyBytes_AS_STRING(result); ++ ++ if (nallocated > PY_SSIZE_T_MAX - repsize + 4) { ++ /* integer overflow */ ++ PyErr_NoMemory(); + goto error; + } +- if (PyBytes_Size(rep) > 4) { +- PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes"); +- Py_DECREF(rep); +- goto error; ++ nallocated += repsize - 4; ++ if (result != NULL) { ++ if (_PyBytes_Resize(&result, nallocated) < 0) ++ goto error; ++ } else { ++ result = PyBytes_FromStringAndSize(NULL, nallocated); ++ if (result == NULL) ++ goto error; ++ Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset); + } +- prep = PyBytes_AsString(rep); +- for(k = PyBytes_Size(rep); k > 0; k--) ++ p = PyBytes_AS_STRING(result) + offset; ++ } ++ ++ if (PyBytes_Check(rep)) { ++ char *prep = PyBytes_AS_STRING(rep); ++ for(k = repsize; k > 0; k--) + *p++ = *prep++; +- Py_DECREF(rep); +- continue; +- ++ } else /* rep is unicode */ { ++ Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep); ++ Py_UNICODE c; ++ ++ for(k=0; k> 12)); +- *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); +- *p++ = (char)(0x80 | (ch & 0x3f)); +- continue; ++ Py_DECREF(rep); ++#ifndef Py_UNICODE_WIDE + } +- encodeUCS4: ++#endif ++ } else if (ch < 0x10000) { ++ *p++ = (char)(0xe0 | (ch >> 12)); ++ *p++ = (char)(0x80 | ((ch >> 6) & 0x3f)); ++ *p++ = (char)(0x80 | (ch & 0x3f)); ++ } else /* ch >= 0x10000 */ { + /* Encode UCS4 Unicode ordinals */ + *p++ = (char)(0xf0 | (ch >> 18)); + *p++ = (char)(0x80 | ((ch >> 12) & 0x3f)); +Index: Python-3.1.2/Lib/test/test_codecs.py +=================================================================== +--- Python-3.1.2.orig/Lib/test/test_codecs.py ++++ Python-3.1.2/Lib/test/test_codecs.py +@@ -571,6 +571,16 @@ class UTF8Test(ReadTest): + def test_lone_surrogates(self): + self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8") + self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8") ++ self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"), ++ b'[\\udc80]') ++ self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"), ++ b'[�]') ++ self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"), ++ b'[\x80]') ++ self.assertEqual("[\uDC80]".encode("utf-8", "ignore"), ++ b'[]') ++ self.assertEqual("[\uDC80]".encode("utf-8", "replace"), ++ b'[?]') + + def test_surrogatepass_handler(self): + self.assertEquals("abc\ud800def".encode("utf-8", "surrogatepass"), diff --git a/python3.spec b/python3.spec index 9ec77d6..75f1203 100644 --- a/python3.spec +++ b/python3.spec @@ -227,7 +227,7 @@ Patch110: python-3.1.2-fix-expat-issue9054.patch # Fix encoding to utf8 when lone surrogates are present and error handler is # set to ignore, replace, or others that return a unicode str. # http://bugs.python.org/issue8092 -Patch111: python3-r80382-lone-surrogate-and-utf8-error-handler.patch +Patch111: python3-r80382-r80385-lone-surrogate-and-utf8-error-handler.patch BuildRoot: %{_tmppath}/%{name}-%{version}-root BuildRequires: readline-devel, openssl-devel, gmp-devel @@ -389,7 +389,7 @@ rm -r Modules/zlib || exit 1 %patch110 -p0 -b .fix-expat-issue9054 -%patch111 -p0 -b .surrogate-utf8 +%patch111 -p1 -b .surrogate-utf8 # Currently (2010-01-15), http://docs.python.org/library is for 2.6, and there # are many differences between 2.6 and the Python 3 library.