Validate large filter sizes when initializing RankFilter (#9695)

Author: radarhere

Tests/test_image_filter.py

@@ -150,6 +150,12 @@ def test_rankfilter_properties() -> None:
     with pytest.raises(ValueError, match="bad filter size"):
         ImageFilter.MinFilter(2)
 
+    with pytest.raises(ValueError, match="filter size too large"):
+        ImageFilter.RankFilter(23171, 1)
+    im = Image.new("1", (1, 1))
+    with pytest.raises(ValueError, match="filter size too large"):
+        im.im.expand(23171)
+
     with pytest.raises(ValueError, match="bad rank value"):
         ImageFilter.RankFilter(1, 1)
 

src/PIL/ImageFilter.py

@@ -102,6 +102,9 @@ def __init__(self, size: int, rank: int) -> None:
         if size % 2 == 0:
             msg = "bad filter size"
             raise ValueError(msg)
+        if size * size * 4 > (2**31 - 1):
+            msg = "filter size too large"
+            raise ValueError(msg)
         if rank < 0 or rank >= size * size:
             msg = "bad rank value"
             raise ValueError(msg)

src/libImaging/Filter.c

@@ -59,6 +59,9 @@ ImagingExpand(Imaging imIn, int margin) {
     if (margin < 0) {
         return (Imaging)ImagingError_ValueError("bad kernel size");
     }
+    if (margin > INT_MAX / (margin * (int)sizeof(FLOAT32))) {
+        return (Imaging)ImagingError_ValueError("filter size too large");
+    }
 
     imOut =
         ImagingNewDirty(imIn->mode, imIn->xsize + 2 * margin, imIn->ysize + 2 * margin);