fix integer overflow in Resample.c

Author: nedwill

libImaging/Resample.c

@@ -138,11 +138,23 @@ ImagingResampleHorizontal(Imaging imIn, int xsize, int filter)
     /* maximum number of coofs */
     kmax = (int) ceil(support) * 2 + 1;
 
+    // check for overflow
+    if (kmax > 0 && xsize > SIZE_MAX / kmax)
+        return (Imaging) ImagingError_MemoryError();
+
+    // sizeof(float) should be greater than 0
+    if (xsize * kmax > SIZE_MAX / sizeof(float))
+        return (Imaging) ImagingError_MemoryError();
+
     /* coefficient buffer */
     kk = malloc(xsize * kmax * sizeof(float));
     if ( ! kk)
         return (Imaging) ImagingError_MemoryError();
 
+    // sizeof(int) should be greater than 0 as well
+    if (xsize > SIZE_MAX / (2 * sizeof(int)))
+        return (Imaging) ImagingError_MemoryError();
+
     xbounds = malloc(xsize * 2 * sizeof(int));
     if ( ! xbounds) {
         free(kk);