From 1813d880d024095f14bec898fff02f80823bbe75 Mon Sep 17 00:00:00 2001 From: heckflosse Date: Mon, 11 Feb 2019 20:35:52 +0100 Subject: [PATCH 1/4] Fix segfault on very large files, #5170 --- rtengine/improcfun.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rtengine/improcfun.cc b/rtengine/improcfun.cc index fb8093de9..3c8b0d430 100644 --- a/rtengine/improcfun.cc +++ b/rtengine/improcfun.cc @@ -5376,7 +5376,7 @@ void ImProcFunctions::getAutoExp (const LUTu &histogram, int histcompr, double histogram.getSumAndAverage (sum, ave); //find median of luminance - int median = 0, count = histogram[0]; + size_t median = 0, count = histogram[0]; while (count < sum / 2) { median++; From ff00226d9751745e95541ab15a7d2bfc81ff379f Mon Sep 17 00:00:00 2001 From: heckflosse Date: Mon, 11 Feb 2019 22:40:31 +0100 Subject: [PATCH 2/4] Fix segfault opining very large jpeg files in editor, #5170 --- rtengine/iimage.h | 32 ++++++++++++++++---------------- rtengine/image8.cc | 2 +- 2 files changed, 17 insertions(+), 17 deletions(-) diff --git a/rtengine/iimage.h b/rtengine/iimage.h index 3549d4dae..9cb303946 100644 --- a/rtengine/iimage.h +++ b/rtengine/iimage.h @@ -139,7 +139,7 @@ protected: AlignedBuffer ab; public: #if CHECK_BOUNDS - int width_, height_; + size_t width_, height_; #endif T** ptrs; @@ -149,7 +149,7 @@ public: PlanarPtr() : ptrs (nullptr) {} #endif - bool resize(int newSize) + bool resize(size_t newSize) { if (ab.resize(newSize)) { ptrs = ab.data; @@ -167,7 +167,7 @@ public: ptrs = tmpsPtrs; #if CHECK_BOUNDS - int tmp = other.width_; + size_t tmp = other.width_; other.width_ = width_; width_ = tmp; tmp = other.height_; @@ -176,7 +176,7 @@ public: #endif } - T*& operator() (unsigned row) + T*& operator() (size_t row) { #if CHECK_BOUNDS assert (row < height_); @@ -184,7 +184,7 @@ public: return ptrs[row]; } // Will send back the start of a row, starting with a red, green or blue value - T* operator() (unsigned row) const + T* operator() (size_t row) const { #if CHECK_BOUNDS assert (row < height_); @@ -192,14 +192,14 @@ public: return ptrs[row]; } // Will send back a value at a given row, col position - T& operator() (unsigned row, unsigned col) + T& operator() (size_t row, size_t col) { #if CHECK_BOUNDS assert (row < height_ && col < width_); #endif return ptrs[row][col]; } - const T operator() (unsigned row, unsigned col) const + const T operator() (size_t row, size_t col) const { #if CHECK_BOUNDS assert (row < height_ && col < width_); @@ -1180,10 +1180,10 @@ class ChunkyPtr { private: T* ptr; - int width; + ssize_t width; public: #if CHECK_BOUNDS - int width_, height_; + size_t width_, height_; #endif #if CHECK_BOUNDS @@ -1191,7 +1191,7 @@ public: #else ChunkyPtr() : ptr (nullptr), width(-1) {} #endif - void init(T* base, int w = -1) + void init(T* base, ssize_t w = -1) { ptr = base; width = w; @@ -1202,12 +1202,12 @@ public: other.ptr = ptr; ptr = tmpsPtr; - int tmpWidth = other.width; + ssize_t tmpWidth = other.width; other.width = width; width = tmpWidth; #if CHECK_BOUNDS - int tmp = other.width_; + size_t tmp = other.width_; other.width_ = width_; width_ = tmp; tmp = other.height_; @@ -1218,7 +1218,7 @@ public: } // Will send back the start of a row, starting with a red, green or blue value - T* operator() (unsigned row) const + T* operator() (size_t row) const { #if CHECK_BOUNDS assert (row < height_); @@ -1226,14 +1226,14 @@ public: return &ptr[3 * (row * width)]; } // Will send back a value at a given row, col position - T& operator() (unsigned row, unsigned col) + T& operator() (size_t row, size_t col) { #if CHECK_BOUNDS assert (row < height_ && col < width_); #endif return ptr[3 * (row * width + col)]; } - const T operator() (unsigned row, unsigned col) const + const T operator() (size_t row, size_t col) const { #if CHECK_BOUNDS assert (row < height_ && col < width_); @@ -1315,7 +1315,7 @@ public: b.height_ = height; #endif - abData.resize(width * height * 3u); + abData.resize((size_t)width * (size_t)height * (size_t)3); if (!abData.isEmpty()) { data = abData.data; diff --git a/rtengine/image8.cc b/rtengine/image8.cc index edced2fe5..513b0049e 100644 --- a/rtengine/image8.cc +++ b/rtengine/image8.cc @@ -69,7 +69,7 @@ void Image8::setScanline (int row, unsigned char* buffer, int bps, unsigned int data[row * width * 3 + 3 * i] = data[row * width * 3 + 3 * i + 1] = data[row * width * 3 + 3 * i + 2] = buffer[i]; } } else { - memcpy (data + row * width * 3u, buffer, width * 3); + memcpy (data + (uint64_t)row * (uint64_t)width * (uint64_t)3u, buffer, width * 3); } break; From 2d167d923b66655e612f50dc77100a1145b19e1b Mon Sep 17 00:00:00 2001 From: heckflosse Date: Wed, 13 Feb 2019 00:36:57 +0100 Subject: [PATCH 3/4] Reduce peak memorey usage for processing non raw files in queue by width * height * 3 * bytedepth, #5170 --- rtengine/stdimagesource.cc | 5 +++++ rtengine/stdimagesource.h | 1 + 2 files changed, 6 insertions(+) diff --git a/rtengine/stdimagesource.cc b/rtengine/stdimagesource.cc index 37d438863..298cf16ef 100644 --- a/rtengine/stdimagesource.cc +++ b/rtengine/stdimagesource.cc @@ -338,5 +338,10 @@ ColorTemp StdImageSource::getSpotWB (std::vector &red, std::vectorallocate(0, 0); +}; + + } diff --git a/rtengine/stdimagesource.h b/rtengine/stdimagesource.h index 500641f72..8f16880dc 100644 --- a/rtengine/stdimagesource.h +++ b/rtengine/stdimagesource.h @@ -101,6 +101,7 @@ public: void getRawValues(int x, int y, int rotate, int &R, int &G, int &B) override { R = G = B = 0;} + void flushRGB () override; }; } From 8eba1c40a65b2f6e730f5e09d1f91edd6db55a13 Mon Sep 17 00:00:00 2001 From: heckflosse Date: Wed, 13 Feb 2019 00:38:11 +0100 Subject: [PATCH 4/4] Fix segfaults when processing very large files in queue, #5170 --- rtengine/iimage.h | 17 ++++++++--------- rtengine/labimage.cc | 8 ++++---- rtengine/labimage.h | 2 +- 3 files changed, 13 insertions(+), 14 deletions(-) diff --git a/rtengine/iimage.h b/rtengine/iimage.h index 9cb303946..3b518e223 100644 --- a/rtengine/iimage.h +++ b/rtengine/iimage.h @@ -215,7 +215,7 @@ class PlanarWhateverData : virtual public ImageDatas private: AlignedBuffer abData; - int rowstride; // Plan size, in bytes (all padding bytes included) + size_t rowstride; // Plan size, in bytes (all padding bytes included) public: T* data; @@ -228,7 +228,7 @@ public: } // Send back the row stride. WARNING: unit = byte, not element! - int getRowStride () const + size_t getRowStride () const { return rowstride; } @@ -259,7 +259,6 @@ public: * Can be safely used to reallocate an existing image */ void allocate (int W, int H) override { - if (W == width && H == height) { return; } @@ -591,8 +590,8 @@ class PlanarRGBData : virtual public ImageDatas private: AlignedBuffer abData; - int rowstride; // Plan size, in bytes (all padding bytes included) - int planestride; // Row length, in bytes (padding bytes included) + size_t rowstride; // Plan size, in bytes (all padding bytes included) + size_t planestride; // Row length, in bytes (padding bytes included) protected: T* data; @@ -602,18 +601,18 @@ public: PlanarPtr b; PlanarRGBData() : rowstride(0), planestride(0), data (nullptr) {} - PlanarRGBData(int w, int h) : rowstride(0), planestride(0), data (nullptr) + PlanarRGBData(size_t w, size_t h) : rowstride(0), planestride(0), data (nullptr) { allocate(w, h); } // Send back the row stride. WARNING: unit = byte, not element! - int getRowStride () const + size_t getRowStride () const { return rowstride; } // Send back the plane stride. WARNING: unit = byte, not element! - int getPlaneStride () const + size_t getPlaneStride () const { return planestride; } @@ -710,7 +709,7 @@ public: char *bluestart = (char*)(data) + 2 * planestride; for (int i = 0; i < height; ++i) { - int k = i * rowstride; + size_t k = i * rowstride; r(i) = (T*)(redstart + k); g(i) = (T*)(greenstart + k); b(i) = (T*)(bluestart + k); diff --git a/rtengine/labimage.cc b/rtengine/labimage.cc index f4e13049a..81b1a4303 100644 --- a/rtengine/labimage.cc +++ b/rtengine/labimage.cc @@ -64,7 +64,7 @@ void LabImage::getPipetteData (float &v1, float &v2, float &v3, int posX, int po v3 = n ? accumulator_b / float(n) : 0.f; } -void LabImage::allocLab(int w, int h) +void LabImage::allocLab(size_t w, size_t h) { L = new float*[h]; a = new float*[h]; @@ -73,19 +73,19 @@ void LabImage::allocLab(int w, int h) data = new float [w * h * 3]; float * index = data; - for (int i = 0; i < h; i++) { + for (size_t i = 0; i < h; i++) { L[i] = index + i * w; } index += w * h; - for (int i = 0; i < h; i++) { + for (size_t i = 0; i < h; i++) { a[i] = index + i * w; } index += w * h; - for (int i = 0; i < h; i++) { + for (size_t i = 0; i < h; i++) { b[i] = index + i * w; } } diff --git a/rtengine/labimage.h b/rtengine/labimage.h index 93f3887b6..28bb891a9 100644 --- a/rtengine/labimage.h +++ b/rtengine/labimage.h @@ -25,7 +25,7 @@ namespace rtengine class LabImage { private: - void allocLab(int w, int h); + void allocLab(size_t w, size_t h); public: int W, H;