Validate margin in ImagingExpand

Author: radarhere

Tests/test_image_filter.py

@@ -152,6 +152,9 @@ def test_rankfilter_properties() -> None:
 
     with pytest.raises(ValueError, match="filter size too large"):
         ImageFilter.RankFilter(23171, 1)
+    im = Image.new("1", (1, 1))
+    with pytest.raises(ValueError, match="filter size too large"):
+        im.im.expand(23171)
 
     with pytest.raises(ValueError, match="bad rank value"):
         ImageFilter.RankFilter(1, 1)

src/libImaging/Filter.c

@@ -59,6 +59,9 @@ ImagingExpand(Imaging imIn, int margin) {
     if (margin < 0) {
         return (Imaging)ImagingError_ValueError("bad kernel size");
     }
+    if (margin > INT_MAX / (margin * (int)sizeof(FLOAT32))) {
+        return (Imaging)ImagingError_ValueError("filter size too large");
+    }
 
     imOut =
         ImagingNewDirty(imIn->mode, imIn->xsize + 2 * margin, imIn->ysize + 2 * margin);