fix expandtabs overflow detection to be consistent and not rely on signed overflow

benjaminp · benjaminp · commit 23cf403ca17d · 2014-03-30T19:47:57.000-04:00
diff --git a/Objects/stringlib/transmogrify.h b/Objects/stringlib/transmogrify.h
@@ -15,7 +15,7 @@ stringlib_expandtabs(PyObject *self, PyObject *args)
 {
     const char *e, *p;
     char *q;
-    size_t i, j;
+    Py_ssize_t i, j;
     PyObject *u;
     int tabsize = 8;
     
@@ -25,35 +25,31 @@ stringlib_expandtabs(PyObject *self, PyObject *args)
     /* First pass: determine size of output string */
     i = j = 0;
     e = STRINGLIB_STR(self) + STRINGLIB_LEN(self);
-    for (p = STRINGLIB_STR(self); p < e; p++)
+    for (p = STRINGLIB_STR(self); p < e; p++) {
         if (*p == '\t') {
             if (tabsize > 0) {
-                j += tabsize - (j % tabsize);
-                if (j > PY_SSIZE_T_MAX) {
-                    PyErr_SetString(PyExc_OverflowError,
-                                    "result is too long");
-                    return NULL;
-                }
+                Py_ssize_t incr = tabsize - (j % tabsize);
+                if (j > PY_SSIZE_T_MAX - incr)
+                    goto overflow;
+                j += incr;
             }
         }
         else {
+            if (j > PY_SSIZE_T_MAX - 1)
+                goto overflow;
             j++;
             if (*p == '\n' || *p == '\r') {
+                if (i > PY_SSIZE_T_MAX - j)
+                    goto overflow;
                 i += j;
                 j = 0;
-                if (i > PY_SSIZE_T_MAX) {
-                    PyErr_SetString(PyExc_OverflowError,
-                                    "result is too long");
-                    return NULL;
-                }
             }
         }
-    
-    if ((i + j) > PY_SSIZE_T_MAX) {
-        PyErr_SetString(PyExc_OverflowError, "result is too long");
-        return NULL;
     }
     
+    if (i > PY_SSIZE_T_MAX - j)
+        goto overflow;
+    
     /* Second pass: create output string and fill it */
     u = STRINGLIB_NEW(NULL, i + j);
     if (!u)
@@ -62,7 +58,7 @@ stringlib_expandtabs(PyObject *self, PyObject *args)
     j = 0;
     q = STRINGLIB_STR(u);
     
-    for (p = STRINGLIB_STR(self); p < e; p++)
+    for (p = STRINGLIB_STR(self); p < e; p++) {
         if (*p == '\t') {
             if (tabsize > 0) {
                 i = tabsize - (j % tabsize);
@@ -77,8 +73,12 @@ stringlib_expandtabs(PyObject *self, PyObject *args)
             if (*p == '\n' || *p == '\r')
                 j = 0;
         }
-    
+    }
+
     return u;
+  overflow:
+    PyErr_SetString(PyExc_OverflowError, "result too long");
+    return NULL;
 }
 
 Py_LOCAL_INLINE(PyObject *)
