improved out-of-gamut warning, taken from PhotoFlow

rtengine/improcfun.cc

@@ -260,10 +260,18 @@ extern const Settings* settings;
 
 ImProcFunctions::~ImProcFunctions ()
 {
-
     if (monitorTransform) {
         cmsDeleteTransform (monitorTransform);
     }
+    if (gw_softproof2refTransform) {
+        cmsDeleteTransform(gw_softproof2refTransform);
+    }
+    if (gw_lab2refTransform) {
+        cmsDeleteTransform(gw_lab2refTransform);
+    }
+    if (gw_lab2softproofTransform) {
+        cmsDeleteTransform(gw_lab2softproofTransform);
+    }
 }
 
 void ImProcFunctions::setScale (double iscale)
@@ -271,14 +279,34 @@ void ImProcFunctions::setScale (double iscale)
     scale = iscale;
 }
 
+
+static void cms_log_handler(cmsContext ContextID, cmsUInt32Number ErrorCode, const char *Text)
+{
+    std::cout << "\n*** LCMS ERROR: " << ErrorCode << ": " << Text << "\n" << std::endl;
+}
+
 void ImProcFunctions::updateColorProfiles (const Glib::ustring& monitorProfile, RenderingIntent monitorIntent, bool softProof, bool gamutCheck)
 {
     // set up monitor transform
     if (monitorTransform) {
         cmsDeleteTransform (monitorTransform);
     }
+    if (gw_softproof2refTransform) {
+        cmsDeleteTransform(gw_softproof2refTransform);
+    }
+    if (gw_lab2refTransform) {
+        cmsDeleteTransform(gw_lab2refTransform);
+    }
+    if (gw_lab2softproofTransform) {
+        cmsDeleteTransform(gw_lab2softproofTransform);
+    }
+
+    cmsSetLogErrorHandler(&cms_log_handler);
 
     monitorTransform = nullptr;
+    gw_softproof2refTransform = nullptr;
+    gw_lab2refTransform = nullptr;
+    gw_lab2softproofTransform = nullptr;
 
     cmsHPROFILE monitor = nullptr;
 
@@ -295,6 +323,8 @@ void ImProcFunctions::updateColorProfiles (const Glib::ustring& monitorProfile,
 
         cmsUInt32Number flags;
         cmsHPROFILE iprof  = cmsCreateLab4Profile (nullptr);
+        cmsHPROFILE gamutprof = nullptr;
+        cmsUInt32Number gamutbpc = 0;
 
         bool softProofCreated = false;
 
@@ -321,9 +351,9 @@ void ImProcFunctions::updateColorProfiles (const Glib::ustring& monitorProfile,
             if (oprof) {
                 // NOCACHE is for thread safety, NOOPTIMIZE for precision
 
-                if (gamutCheck) {
+                // if (gamutCheck) {
-                    flags |= cmsFLAGS_GAMUTCHECK;
+                //     flags |= cmsFLAGS_GAMUTCHECK;
-                }
+                // }
 
                 monitorTransform = cmsCreateProofingTransform (
                                        iprof, TYPE_Lab_FLT,
@@ -336,17 +366,28 @@ void ImProcFunctions::updateColorProfiles (const Glib::ustring& monitorProfile,
                 if (monitorTransform) {
                     softProofCreated = true;
                 }
+
+                if (gamutCheck) {
+                    gamutprof = oprof;
+                    if (params->icm.outputBPC) {
+                        gamutbpc = cmsFLAGS_BLACKPOINTCOMPENSATION;
+                    }
+                }
             }
         } else if (gamutCheck) {
-            flags = cmsFLAGS_GAMUTCHECK | cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE;
+            // flags = cmsFLAGS_GAMUTCHECK | cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE;
+            // if (settings->monitorBPC) {
+            //     flags |= cmsFLAGS_BLACKPOINTCOMPENSATION;
+            // }
+
+            // monitorTransform = cmsCreateProofingTransform(iprof, TYPE_Lab_FLT, monitor, TYPE_RGB_8, monitor, monitorIntent, monitorIntent, flags);
+
+            // if (monitorTransform) {
+            //     softProofCreated = true;
+            // }
+            gamutprof = monitor;
             if (settings->monitorBPC) {
-                flags |= cmsFLAGS_BLACKPOINTCOMPENSATION;
+                gamutbpc = cmsFLAGS_BLACKPOINTCOMPENSATION;
-            }
-
-            monitorTransform = cmsCreateProofingTransform(iprof, TYPE_Lab_FLT, monitor, TYPE_RGB_8, monitor, monitorIntent, monitorIntent, flags);
-
-            if (monitorTransform) {
-                softProofCreated = true;
             }
         }
 
@@ -360,6 +401,33 @@ void ImProcFunctions::updateColorProfiles (const Glib::ustring& monitorProfile,
             monitorTransform = cmsCreateTransform (iprof, TYPE_Lab_FLT, monitor, TYPE_RGB_8, monitorIntent, flags);
         }
 
+        if (gamutCheck) {
+            if (cmsIsMatrixShaper(gamutprof)) {
+                cmsHPROFILE aces = ICCStore::getInstance()->getProfile("ACES");
+                if (aces) {
+                    gw_lab2refTransform = cmsCreateTransform(iprof, TYPE_Lab_FLT, aces, TYPE_RGB_FLT, INTENT_ABSOLUTE_COLORIMETRIC, cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE);
+                    gw_lab2softproofTransform = cmsCreateTransform(iprof, TYPE_Lab_FLT, gamutprof, TYPE_RGB_FLT, INTENT_ABSOLUTE_COLORIMETRIC, cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE);
+                    gw_softproof2refTransform = cmsCreateTransform(gamutprof, TYPE_RGB_FLT, aces, TYPE_RGB_FLT, INTENT_ABSOLUTE_COLORIMETRIC, cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE | gamutbpc);
+                }
+            } else {
+                gw_lab2refTransform = nullptr;
+                gw_lab2softproofTransform = cmsCreateTransform(iprof, TYPE_Lab_FLT, gamutprof, TYPE_RGB_FLT, INTENT_ABSOLUTE_COLORIMETRIC, cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE);
+                gw_softproof2refTransform = cmsCreateTransform(gamutprof, TYPE_RGB_FLT, iprof, TYPE_Lab_FLT, INTENT_ABSOLUTE_COLORIMETRIC, cmsFLAGS_NOOPTIMIZE | cmsFLAGS_NOCACHE | gamutbpc);
+            }
+
+            if (!gw_softproof2refTransform) {
+                if (gw_lab2softproofTransform) {
+                    cmsDeleteTransform(gw_lab2softproofTransform);
+                    gw_lab2softproofTransform = nullptr;
+                }
+            } else if (!gw_lab2softproofTransform) {
+                if (gw_softproof2refTransform) {
+                    cmsDeleteTransform(gw_softproof2refTransform);
+                    gw_softproof2refTransform = nullptr;
+                }
+            }
+        }
+
         cmsCloseProfile (iprof);
     }
 }

rtengine/improcfun.h

@@ -45,8 +45,9 @@ class ImProcFunctions
 
 
     cmsHTRANSFORM monitorTransform;
-    cmsHTRANSFORM lab2outputTransform;
+    cmsHTRANSFORM gw_lab2refTransform;
-    cmsHTRANSFORM output2monitorTransform;
+    cmsHTRANSFORM gw_lab2softproofTransform;
+    cmsHTRANSFORM gw_softproof2refTransform;
 
     const ProcParams* params;
     double scale;
@@ -197,7 +198,7 @@ public:
     double lumimul[3];
 
     ImProcFunctions       (const ProcParams* iparams, bool imultiThread = true)
-        : monitorTransform (nullptr), lab2outputTransform (nullptr), output2monitorTransform (nullptr), params (iparams), scale (1), multiThread (imultiThread), lumimul{} {}
+        : monitorTransform (nullptr), gw_lab2refTransform(nullptr), gw_lab2softproofTransform(nullptr), gw_softproof2refTransform(nullptr), params (iparams), scale (1), multiThread (imultiThread), lumimul{} {}
     ~ImProcFunctions      ();
     bool needsLuminanceOnly() { return !(needsCA() || needsDistortion() || needsRotation() || needsPerspective() || needsLCP() || needsLensfun()) && (needsVignetting() || needsPCVignetting() || needsGradient());}
     void setScale         (double iscale);

rtengine/iplab2rgb.cc

@@ -46,12 +46,28 @@ void ImProcFunctions::lab2monitorRgb (LabImage* lab, Image8* image)
         int H = lab->H;
         unsigned char * data = image->data;
 
+        const auto set_gamut_warning =
+            [](Image8 *image, int y, int x) -> void
+            {
+                image->r(y, x) = 0;
+                image->g(y, x) = 255;
+                image->b(y, x) = 255;
+            };
+
         // cmsDoTransform is relatively expensive
 #ifdef _OPENMP
         #pragma omp parallel firstprivate(lab, data, W, H)
 #endif
         {
             AlignedBuffer<float> pBuf(3 * lab->W);
+
+            AlignedBuffer<float> gwBuf1;
+            AlignedBuffer<float> gwBuf2;
+            if (gw_softproof2refTransform) {
+                gwBuf1.resize(3 * lab->W);
+                gwBuf2.resize(3 * lab->W);
+            }
+            
             float *buffer = pBuf.data;
 
 #ifdef _OPENMP
@@ -74,6 +90,40 @@ void ImProcFunctions::lab2monitorRgb (LabImage* lab, Image8* image)
                 }
 
                 cmsDoTransform (monitorTransform, buffer, data + ix, W);
+
+                if (gw_softproof2refTransform) {
+                    float delta_max = gw_lab2refTransform ? 0.0001f : 4.9999f;
+                    cmsDoTransform(gw_lab2softproofTransform, buffer, gwBuf2.data, W);
+                    
+                    cmsDoTransform(gw_softproof2refTransform, gwBuf2.data, gwBuf1.data, W);
+                    float *proofdata = gwBuf1.data;
+                    float *refdata = buffer;
+                    if (gw_lab2refTransform) {
+                        cmsDoTransform(gw_lab2refTransform, buffer, gwBuf2.data, W);
+                        refdata = gwBuf2.data;
+
+                        int iy = 0;
+                        for (int j = 0; j < W; ++j) {
+                            float delta = max(std::abs(proofdata[iy] - refdata[iy]), std::abs(proofdata[iy+1] - refdata[iy+1]), std::abs(proofdata[iy+2] - refdata[iy+2]));
+                            iy += 3;
+
+                            if (delta > delta_max) {
+                                set_gamut_warning(image, i, j);
+                            }
+                        }
+                    } else {
+                        int iy = 0;
+                        for (int j = 0; j < W; ++j) {
+                            cmsCIELab lab1 = { proofdata[iy], proofdata[iy+1], proofdata[iy+2] };
+                            cmsCIELab lab2 = { refdata[iy], refdata[iy+1], refdata[iy+2] };
+                            iy += 3;
+                            float delta = cmsDeltaE(&lab1, &lab2);
+                            if (delta > delta_max) {
+                                set_gamut_warning(image, i, j);
+                            }
+                        }
+                    }
+                }
             }
         } // End of parallelization
     } else {