torsava / rpms / python3

Forked from rpms/python3 2 years ago
Clone
e9848a
Index: Python-3.1.2/Objects/unicodeobject.c
211f42
===================================================================
e9848a
--- Python-3.1.2.orig/Objects/unicodeobject.c
e9848a
+++ Python-3.1.2/Objects/unicodeobject.c
e9848a
@@ -159,6 +159,12 @@ static PyObject *unicode_encode_call_err
211f42
        const Py_UNICODE *unicode, Py_ssize_t size, PyObject **exceptionObject,
211f42
        Py_ssize_t startpos, Py_ssize_t endpos, Py_ssize_t *newpos);
211f42
 
211f42
+static void raise_encode_exception(PyObject **exceptionObject,
211f42
+                                   const char *encoding,
211f42
+                                   const Py_UNICODE *unicode, Py_ssize_t size,
211f42
+                                   Py_ssize_t startpos, Py_ssize_t endpos,
211f42
+                                   const char *reason);
211f42
+
211f42
 /* Same for linebreaks */
211f42
 static unsigned char ascii_linebreak[] = {
211f42
     0, 0, 0, 0, 0, 0, 0, 0,
e9848a
@@ -2453,67 +2459,98 @@ PyUnicode_EncodeUTF8(const Py_UNICODE *s
e9848a
     for (i = 0; i < size;) {
e9848a
         Py_UCS4 ch = s[i++];
e9848a
 
e9848a
-        if (ch < 0x80)
e9848a
+        if (ch < 0x80) {
e9848a
             /* Encode ASCII */
e9848a
             *p++ = (char) ch;
e9848a
 
e9848a
-        else if (ch < 0x0800) {
e9848a
+        } else if (ch < 0x0800) {
211f42
             /* Encode Latin-1 */
211f42
             *p++ = (char)(0xc0 | (ch >> 6));
211f42
             *p++ = (char)(0x80 | (ch & 0x3f));
211f42
-        }
211f42
-        else {
211f42
-            /* Encode UCS2 Unicode ordinals */
211f42
-            if (ch < 0x10000) {
211f42
+        } else if (0xD800 <= ch && ch <= 0xDFFF) {
211f42
 #ifndef Py_UNICODE_WIDE
211f42
-                /* Special case: check for high surrogate */
211f42
-                if (0xD800 <= ch && ch <= 0xDBFF && i != size) {
211f42
-                    Py_UCS4 ch2 = s[i];
211f42
-                    /* Check for low surrogate and combine the two to
211f42
-                       form a UCS4 value */
211f42
-                    if (0xDC00 <= ch2 && ch2 <= 0xDFFF) {
211f42
-                        ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
211f42
-                        i++;
211f42
-                        goto encodeUCS4;
211f42
-                    }
211f42
-                    /* Fall through: handles isolated high surrogates */
211f42
-                }
211f42
+            /* Special case: check for high and low surrogate */
211f42
+            if (ch <= 0xDBFF && i != size && 0xDC00 <= s[i] && s[i] <= 0xDFFF) {
211f42
+                Py_UCS4 ch2 = s[i];
211f42
+                /* Combine the two surrogates to form a UCS4 value */
211f42
+                ch = ((ch - 0xD800) << 10 | (ch2 - 0xDC00)) + 0x10000;
211f42
+                i++;
211f42
+
211f42
+                /* Encode UCS4 Unicode ordinals */
211f42
+                *p++ = (char)(0xf0 | (ch >> 18));
211f42
+                *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
211f42
+                *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
211f42
+                *p++ = (char)(0x80 | (ch & 0x3f));
211f42
+
e9848a
+            } else {
211f42
 #endif
211f42
-                if (ch >= 0xd800 && ch <= 0xdfff) {
211f42
-                    Py_ssize_t newpos;
211f42
-                    PyObject *rep;
211f42
-                    char *prep;
211f42
-                    int k;
211f42
-                    rep = unicode_encode_call_errorhandler
211f42
-                        (errors, &errorHandler, "utf-8", "surrogates not allowed", 
211f42
-                         s, size, &exc, i-1, i, &newpos);
211f42
-                    if (!rep)
e9848a
-                        goto error;
e9848a
-                    /* Implementation limitations: only support error handler that return
e9848a
-                       bytes, and only support up to four replacement bytes. */
e9848a
-                    if (!PyBytes_Check(rep)) {
e9848a
-                        PyErr_SetString(PyExc_TypeError, "error handler should have returned bytes");
e9848a
-                        Py_DECREF(rep);
211f42
+                Py_ssize_t newpos;
211f42
+                PyObject *rep;
211f42
+                Py_ssize_t repsize, k;
211f42
+                rep = unicode_encode_call_errorhandler
211f42
+                    (errors, &errorHandler, "utf-8", "surrogates not allowed",
211f42
+                     s, size, &exc, i-1, i, &newpos);
211f42
+                if (!rep)
211f42
+                    goto error;
211f42
+
211f42
+                if (PyBytes_Check(rep))
211f42
+                    repsize = PyBytes_GET_SIZE(rep);
211f42
+                else
211f42
+                    repsize = PyUnicode_GET_SIZE(rep);
211f42
+
211f42
+                if (repsize > 4) {
211f42
+                    Py_ssize_t offset;
211f42
+
211f42
+                    if (result == NULL)
211f42
+                        offset = p - stackbuf;
211f42
+                    else
211f42
+                        offset = p - PyBytes_AS_STRING(result);
211f42
+
211f42
+                    if (nallocated > PY_SSIZE_T_MAX - repsize + 4) {
211f42
+                        /* integer overflow */
211f42
+                        PyErr_NoMemory();
211f42
                         goto error;
211f42
                     }
211f42
-                    if (PyBytes_Size(rep) > 4) {
211f42
-                        PyErr_SetString(PyExc_TypeError, "error handler returned too many bytes");
211f42
-                        Py_DECREF(rep);
211f42
-                        goto error;
211f42
+                    nallocated += repsize - 4;
211f42
+                    if (result != NULL) {
211f42
+                        if (_PyBytes_Resize(&result, nallocated) < 0)
211f42
+                            goto error;
211f42
+                    } else {
211f42
+                        result = PyBytes_FromStringAndSize(NULL, nallocated);
211f42
+                        if (result == NULL)
211f42
+                            goto error;
211f42
+                        Py_MEMCPY(PyBytes_AS_STRING(result), stackbuf, offset);
211f42
                     }
211f42
-                    prep = PyBytes_AsString(rep);
211f42
-                    for(k = PyBytes_Size(rep); k > 0; k--)
211f42
+                    p = PyBytes_AS_STRING(result) + offset;
211f42
+                }
211f42
+
211f42
+                if (PyBytes_Check(rep)) {
211f42
+                    char *prep = PyBytes_AS_STRING(rep);
211f42
+                    for(k = repsize; k > 0; k--)
211f42
                         *p++ = *prep++;
211f42
-                    Py_DECREF(rep);
211f42
-                    continue;
211f42
-                    
211f42
+                } else /* rep is unicode */ {
211f42
+                    Py_UNICODE *prep = PyUnicode_AS_UNICODE(rep);
211f42
+                    Py_UNICODE c;
211f42
+
211f42
+                    for(k=0; k
211f42
+                        c = prep[k];
211f42
+                        if (0x80 <= c) {
211f42
+                            raise_encode_exception(&exc, "utf-8", s, size,
211f42
+                                                   i-1, i, "surrogates not allowed");
211f42
+                            goto error;
211f42
+                        }
211f42
+                        *p++ = (char)prep[k];
211f42
+                    }
211f42
                 }
211f42
-                *p++ = (char)(0xe0 | (ch >> 12));
211f42
-                *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
211f42
-                *p++ = (char)(0x80 | (ch & 0x3f));
211f42
-                continue;
211f42
+                Py_DECREF(rep);
e9848a
+#ifndef Py_UNICODE_WIDE
211f42
             }
211f42
-          encodeUCS4:
e9848a
+#endif
211f42
+        } else if (ch < 0x10000) {
211f42
+            *p++ = (char)(0xe0 | (ch >> 12));
211f42
+            *p++ = (char)(0x80 | ((ch >> 6) & 0x3f));
211f42
+            *p++ = (char)(0x80 | (ch & 0x3f));
211f42
+        } else /* ch >= 0x10000 */ {
211f42
             /* Encode UCS4 Unicode ordinals */
211f42
             *p++ = (char)(0xf0 | (ch >> 18));
211f42
             *p++ = (char)(0x80 | ((ch >> 12) & 0x3f));
e9848a
Index: Python-3.1.2/Lib/test/test_codecs.py
211f42
===================================================================
e9848a
--- Python-3.1.2.orig/Lib/test/test_codecs.py
e9848a
+++ Python-3.1.2/Lib/test/test_codecs.py
e9848a
@@ -571,6 +571,16 @@ class UTF8Test(ReadTest):
211f42
     def test_lone_surrogates(self):
211f42
         self.assertRaises(UnicodeEncodeError, "\ud800".encode, "utf-8")
211f42
         self.assertRaises(UnicodeDecodeError, b"\xed\xa0\x80".decode, "utf-8")
211f42
+        self.assertEqual("[\uDC80]".encode("utf-8", "backslashreplace"),
211f42
+                         b'[\\udc80]')
211f42
+        self.assertEqual("[\uDC80]".encode("utf-8", "xmlcharrefreplace"),
211f42
+                         b'[�]')
211f42
+        self.assertEqual("[\uDC80]".encode("utf-8", "surrogateescape"),
211f42
+                         b'[\x80]')
211f42
+        self.assertEqual("[\uDC80]".encode("utf-8", "ignore"),
211f42
+                         b'[]')
211f42
+        self.assertEqual("[\uDC80]".encode("utf-8", "replace"),
211f42
+                         b'[?]')
211f42
 
211f42
     def test_surrogatepass_handler(self):
211f42
         self.assertEquals("abc\ud800def".encode("utf-8", "surrogatepass"),