From 07d26c16edca47fd2ac0b8b1c3eaf47cbed8d197 Mon Sep 17 00:00:00 2001
From: Ingo Weyrich <heckflosse67@gmx.de>
Date: Sat, 15 Aug 2020 17:35:24 +0200
Subject: [PATCH] ImProcCoordinator::updateVectorscope(): fix crash caused by
 large stack allocation, speedup, fixed scaling from [0;255] to [0;65535]
 range

---
 rtengine/improccoordinator.cc | 73 +++++++++++++++++------------------
 rtengine/rtengine.h           |  2 +-
 rtgui/editorpanel.cc          |  2 +-
 rtgui/editorpanel.h           |  2 +-
 4 files changed, 38 insertions(+), 41 deletions(-)

diff --git a/rtengine/improccoordinator.cc b/rtengine/improccoordinator.cc
index 7ae814f8c..0a752784a 100644
--- a/rtengine/improccoordinator.cc
+++ b/rtengine/improccoordinator.cc
@@ -45,7 +45,8 @@
 #ifdef _OPENMP
 #include <omp.h>
 #endif
-
+#define BENCHMARK
+#include "StopWatch.h"
 namespace
 {
 
@@ -1855,57 +1856,53 @@ void ImProcCoordinator::updateVectorscope()
     if (!workimg) {
         return;
     }
-
+BENCHFUN
     int x1, y1, x2, y2;
     params->crop.mapToResized(pW, pH, scale, x1, x2, y1, y2);
 
     constexpr int size = VECTORSCOPE_SIZE;
     memset((int*)vectorscope, 0, size * size * sizeof(vectorscope[0][0]));
 
-    const int lab_img_size = (hListener->vectorscopeType() == 1) ? (x2 - x1) * (y2 - y1) : 0;
-    float a[lab_img_size], b[lab_img_size];
+    vectorscopeScale = (x2 - x1) * (y2 - y1);
+    const int lab_img_size = (hListener->vectorscopeType() == 1) ? vectorscopeScale : 0;
+
+    std::unique_ptr<float[]> a;
+    std::unique_ptr<float[]> b;
     if (lab_img_size) {
-        float L[lab_img_size];
-        ipf.rgb2lab(*workimg, x1, y1, x2 - x1, y2 - y1, L, a, b, params->icm);
+        a.reset(new float[lab_img_size]);
+        b.reset(new float[lab_img_size]);
+        std::unique_ptr<float []> L(new float[lab_img_size]);
+        ipf.rgb2lab(*workimg, x1, y1, x2 - x1, y2 - y1, L.get(), a.get(), b.get(), params->icm);
     }
 
-    int ofs_lab = 0;
-    for (int i = y1; i < y2; i++) {
-        int ofs = (i * pW + x1) * 3;
-
-        for (int j = x1; j < x2; j++) {
-            switch (hListener->vectorscopeType()) {
-                case 0: {
-                    // HS
-                    int red = 256 * workimg->data[ofs++];
-                    int green = 256 * workimg->data[ofs++];
-                    int blue = 256 * workimg->data[ofs++];
-                    float h, s, l;
-                    Color::rgb2hsl(red, green, blue, h, s, l);
-                    const int col = s * cos(2 * RT_PI * h) * (size / 2) + size / 2;
-                    const int row = s * sin(2 * RT_PI * h) * (size / 2) + size / 2;
-                    if (col >= 0 && col < size && row >= 0 && row < size) {
-                        vectorscope[row][col]++;
-                    }
-                    break;
+    if (hListener->vectorscopeType() == 0) { // HS
+        for (int i = y1; i < y2; ++i) {
+            int ofs = (i * pW + x1) * 3;
+            for (int j = x1; j < x2; ++j) {
+                const float red = 257.f * workimg->data[ofs++];
+                const float green = 257.f * workimg->data[ofs++];
+                const float blue = 257.f * workimg->data[ofs++];
+                float h, s, l;
+                Color::rgb2hslfloat(red, green, blue, h, s, l);
+                const auto sincosval = xsincosf(2.f * RT_PI_F * h);
+                const int col = s * sincosval.y * (size / 2) + size / 2;
+                const int row = s * sincosval.x * (size / 2) + size / 2;
+                if (col >= 0 && col < size && row >= 0 && row < size) {
+                    vectorscope[row][col]++;
                 }
-
-                case 1: {
-                    // CH
-                    const int col = (size / 96000.0) * a[ofs_lab] + size / 2;
-                    const int row = (size / 96000.0) * b[ofs_lab] + size / 2;
-
-                    if (col >= 0 && col < size && row >= 0 && row < size) {
-                        vectorscope[row][col]++;
-                    }
-                    ofs_lab++;
-                    break;
+            }
+        }
+    } else if (hListener->vectorscopeType() == 1) { // CH
+        for (int i = y1; i < y2; ++i) {
+            for (int j = x1, ofs_lab = (i - y1) * (x2 - x1); j < x2; ++j, ++ofs_lab) {
+                const int col = (size / 96000.f) * a[ofs_lab] + size / 2;
+                const int row = (size / 96000.f) * b[ofs_lab] + size / 2;
+                if (col >= 0 && col < size && row >= 0 && row < size) {
+                    vectorscope[row][col]++;
                 }
             }
         }
     }
-
-    vectorscopeScale = (x2 - x1) * (y2 - y1);
 }
 
 void ImProcCoordinator::updateWaveforms()
diff --git a/rtengine/rtengine.h b/rtengine/rtengine.h
index 9c4092f2b..be33ea41e 100644
--- a/rtengine/rtengine.h
+++ b/rtengine/rtengine.h
@@ -350,7 +350,7 @@ public:
     /** Returns if the listener wants the waveform to be updated. */
     virtual bool updateWaveform(void) = 0;
     /** Returns the vectorscope type: 0 for H-S and 1 for H-C. */
-    virtual int vectorscopeType(void) = 0;
+    virtual int vectorscopeType(void) const = 0;
 };
 
 class HistogramObservable
diff --git a/rtgui/editorpanel.cc b/rtgui/editorpanel.cc
index 7f5c99997..d80da8ef2 100644
--- a/rtgui/editorpanel.cc
+++ b/rtgui/editorpanel.cc
@@ -2289,7 +2289,7 @@ bool EditorPanel::updateWaveform(void)
         || histogram_scope_type == HistogramPanelListener::NONE;
 }
 
-int EditorPanel::vectorscopeType(void)
+int EditorPanel::vectorscopeType(void) const
 {
     switch (histogram_scope_type) {
         case HistogramPanelListener::VECTORSCOPE_HS:
diff --git a/rtgui/editorpanel.h b/rtgui/editorpanel.h
index 2916dc822..93e373924 100644
--- a/rtgui/editorpanel.h
+++ b/rtgui/editorpanel.h
@@ -146,7 +146,7 @@ public:
     bool updateHistogram(void) override;
     bool updateVectorscope(void) override;
     bool updateWaveform(void) override;
-    int vectorscopeType(void) override;
+    int vectorscopeType(void) const override;
 
     // HistogramPanelListener
     void scopeTypeChanged(ScopeType new_type) override;