Fix segfault opining very large jpeg files in editor, #5170

rtengine/iimage.h

@@ -139,7 +139,7 @@ protected:
     AlignedBuffer<T*> ab;
 public:
 #if CHECK_BOUNDS
-    int width_, height_;
+    size_t width_, height_;
 #endif
     T** ptrs;
 
@@ -149,7 +149,7 @@ public:
     PlanarPtr() : ptrs (nullptr) {}
 #endif
 
-    bool resize(int newSize)
+    bool resize(size_t newSize)
     {
         if (ab.resize(newSize)) {
             ptrs = ab.data;
@@ -167,7 +167,7 @@ public:
         ptrs = tmpsPtrs;
 
 #if CHECK_BOUNDS
-        int tmp = other.width_;
+        size_t tmp = other.width_;
         other.width_ = width_;
         width_ = tmp;
         tmp = other.height_;
@@ -176,7 +176,7 @@ public:
 #endif
     }
 
-    T*&       operator() (unsigned row)
+    T*&       operator() (size_t row)
     {
 #if CHECK_BOUNDS
         assert (row < height_);
@@ -184,7 +184,7 @@ public:
         return ptrs[row];
     }
     // Will send back the start of a row, starting with a red, green or blue value
-    T*        operator() (unsigned row) const
+    T*        operator() (size_t row) const
     {
 #if CHECK_BOUNDS
         assert (row < height_);
@@ -192,14 +192,14 @@ public:
         return ptrs[row];
     }
     // Will send back a value at a given row, col position
-    T&        operator() (unsigned row, unsigned col)
+    T&        operator() (size_t row, size_t col)
     {
 #if CHECK_BOUNDS
         assert (row < height_ && col < width_);
 #endif
         return ptrs[row][col];
     }
-    const T   operator() (unsigned row, unsigned col) const
+    const T   operator() (size_t row, size_t col) const
     {
 #if CHECK_BOUNDS
         assert (row < height_ && col < width_);
@@ -1180,10 +1180,10 @@ class ChunkyPtr
 {
 private:
     T* ptr;
-    int width;
+    ssize_t width;
 public:
 #if CHECK_BOUNDS
-    int width_, height_;
+    size_t width_, height_;
 #endif
 
 #if CHECK_BOUNDS
@@ -1191,7 +1191,7 @@ public:
 #else
     ChunkyPtr() : ptr (nullptr), width(-1) {}
 #endif
-    void init(T* base, int w = -1)
+    void init(T* base, ssize_t w = -1)
     {
         ptr = base;
         width = w;
@@ -1202,12 +1202,12 @@ public:
         other.ptr = ptr;
         ptr = tmpsPtr;
 
-        int tmpWidth = other.width;
+        ssize_t tmpWidth = other.width;
         other.width = width;
         width = tmpWidth;
 
 #if CHECK_BOUNDS
-        int tmp = other.width_;
+        size_t tmp = other.width_;
         other.width_ = width_;
         width_ = tmp;
         tmp = other.height_;
@@ -1218,7 +1218,7 @@ public:
     }
 
     // Will send back the start of a row, starting with a red, green or blue value
-    T* operator() (unsigned row) const
+    T* operator() (size_t row) const
     {
 #if CHECK_BOUNDS
         assert (row < height_);
@@ -1226,14 +1226,14 @@ public:
         return &ptr[3 * (row * width)];
     }
     // Will send back a value at a given row, col position
-    T& operator() (unsigned row, unsigned col)
+    T& operator() (size_t row, size_t col)
     {
 #if CHECK_BOUNDS
         assert (row < height_ && col < width_);
 #endif
         return ptr[3 * (row * width + col)];
     }
-    const T  operator() (unsigned row, unsigned col) const
+    const T  operator() (size_t row, size_t col) const
     {
 #if CHECK_BOUNDS
         assert (row < height_ && col < width_);
@@ -1315,7 +1315,7 @@ public:
         b.height_ = height;
 #endif
 
-        abData.resize(width * height * 3u);
+        abData.resize((size_t)width * (size_t)height * (size_t)3);
 
         if (!abData.isEmpty()) {
             data = abData.data;

rtengine/image8.cc

@@ -69,7 +69,7 @@ void Image8::setScanline (int row, unsigned char* buffer, int bps, unsigned int
                 data[row * width * 3 + 3 * i] = data[row * width * 3 + 3 * i + 1] = data[row * width * 3 + 3 * i + 2] = buffer[i];
             }
         } else {
-            memcpy (data + row * width * 3u, buffer, width * 3);
+            memcpy (data + (uint64_t)row * (uint64_t)width * (uint64_t)3u, buffer, width * 3);
         }
         break;