Merge pull request #3094 from adamreichold/fix-cluts-dir-listing

Fix slow parsing of CLUT directory

rtdata/languages/default

@@ -1466,6 +1466,7 @@ TP_EXPOS_WHITEPOINT_LABEL;Raw White Points
 TP_FILMSIMULATION_LABEL;Film Simulation
 TP_FILMSIMULATION_STRENGTH;Strength
 TP_FILMSIMULATION_ZEROCLUTSFOUND;Set HaldCLUT directory in Preferences
+TP_FILMSIMULATION_SLOWPARSEDIR;The Film Simulation HaldCLUT folder you pointed RawTherapee to is taking too long to load. Reduce the number of files in that folder or point RawTherapee to an empty one instead.\n\nSee Preferences > Image Processing > Film Simulation
 TP_FLATFIELD_AUTOSELECT;Auto-selection
 TP_FLATFIELD_BLURRADIUS;Blur radius
 TP_FLATFIELD_BLURTYPE;Blur type

rtengine/iccstore.cc

@@ -54,7 +54,7 @@ void loadProfiles (const Glib::ustring& dirName,
 
             const Glib::ustring extension = fileName.substr (fileName.size () - 4).casefold ();
 
-            if (extension.compare(".icc") == 0 && extension.compare(".icm") == 0)
+            if (extension.compare (".icc") != 0 && extension.compare (".icm") != 0)
                 continue;
 
             const Glib::ustring filePath = Glib::build_filename (dirName, fileName);

rtgui/filmsimulation.cc

@@ -1,4 +1,7 @@
 #include "filmsimulation.h"
+
+#include <chrono>
+
 #include "options.h"
 #include "../rtengine/clutstore.h"
 #include "../rtengine/safegtk.h"
@@ -6,6 +9,30 @@
 using namespace rtengine;
 using namespace rtengine::procparams;
 
+namespace
+{
+
+void notifySlowParseDir (const std::chrono::system_clock::time_point& startedAt)
+{
+    static bool alreadyNotified = false;
+
+    if (alreadyNotified) {
+        return;
+    }
+
+    const auto now = std::chrono::system_clock::now ();
+    if (now - startedAt < std::chrono::seconds (10)) {
+        return;
+    }
+
+    Gtk::MessageDialog dialog (M ("TP_FILMSIMULATION_SLOWPARSEDIR"), false, Gtk::MESSAGE_WARNING, Gtk::BUTTONS_OK, true);
+    dialog.run ();
+
+    alreadyNotified = true;
+}
+
+}
+
 typedef std::vector<Glib::ustring> Strings;
 
 FilmSimulation::FilmSimulation()
@@ -143,83 +170,119 @@ ClutComboBox::ClutColumns::ClutColumns()
     add( clutFilename );
 }
 
-int ClutComboBox::fillFromDir( Glib::ustring path )
+int ClutComboBox::fillFromDir (const Glib::ustring& path)
 {
-    int result = 0;
+    m_model = Gtk::TreeStore::create (m_columns);
+    set_model (m_model);
 
-    if ( !path.empty() ) {
+    const auto result = parseDir (path);
-        m_model.clear();
+
-        m_model = Gtk::TreeStore::create( m_columns );
+    if (result > 0) {
-        set_model( m_model );
+        pack_start (m_columns.label, false);
-        result = parseDir( path, 0 );
-        pack_start( m_columns.label, false );
     }
 
     return result;
 }
 
-Gtk::TreeIter appendToModel( Glib::RefPtr<Gtk::TreeStore> model, Gtk::TreeModel::Row *parent )
+int ClutComboBox::parseDir (const Glib::ustring& path)
 {
-    Gtk::TreeIter result;
+    if (path.empty () || !Glib::file_test (path, Glib::FILE_TEST_IS_DIR)) {
-
+        return 0;
-    if ( parent ) {
-        result = model->append( parent->children() );
-
-    } else {
-        result = model->append();
     }
 
-    return result;
+    const auto startedAt = std::chrono::system_clock::now ();
-}
 
-int ClutComboBox::parseDir( Glib::ustring path, Gtk::TreeModel::Row *parentRow )
+    // Build menu of limited directory structure using breadth-first search
-{
+    using Dirs = std::vector<std::pair<Glib::ustring, Gtk::TreeModel::Row>>;
-    int result = 0;
+    Dirs dirs;
 
-    if ( path.empty() || !safe_file_test( path, Glib::FILE_TEST_EXISTS ) || !safe_file_test ( path, Glib::FILE_TEST_IS_DIR ) ) {
+    {
-        return result;
+        Dirs currDirs;
+        Dirs nextDirs;
+
+        currDirs.emplace_back (path, Gtk::TreeModel::Row ());
+
+        while (!currDirs.empty ()) {
+
+            for (auto& dir : currDirs) {
+
+                const auto& path = dir.first;
+                const auto& row = dir.second;
+
+                try {
+                    for (const auto& entry : Glib::Dir (path)) {
+
+                        const auto entryPath = Glib::build_filename (path, entry);
+
+                        if (!Glib::file_test (entryPath, Glib::FILE_TEST_IS_DIR)) {
+                            continue;
                         }
 
-    Glib::Dir* dir = new Glib::Dir( path );
+                        auto newRow = row ? *m_model->append (row.children ()) : *m_model->append ();
+                        newRow[m_columns.label] = entry;
 
-    Strings names;
+                        nextDirs.emplace_back (entryPath, newRow);
+                    }
+                } catch (Glib::Exception&) {}
 
-    for( Glib::DirIterator it = dir->begin(); it != dir->end(); ++it ) {
+                dirs.push_back (std::move (dir));
-        Glib::ustring current = *it;
 
-        if ( current != "." && current != ".." ) {
+                notifySlowParseDir (startedAt);
-            names.push_back( current );
+            }
+
+            currDirs.clear ();
+            currDirs.swap (nextDirs);
         }
     }
 
-    std::sort( names.begin(), names.end() );
+    // Fill menu structure with CLUT files
+    Strings entries;
 
-    for ( Strings::iterator it = names.begin(); it != names.end(); ++it ) {
+    auto fileCount = 0;
-        Glib::ustring current = *it;
-        Glib::ustring fullname = Glib::build_filename( path, current );
 
-        if ( safe_file_test( fullname, Glib::FILE_TEST_IS_DIR ) ) {
+    for (const auto& dir : dirs) {
+
+        const auto& path = dir.first;
+        const auto& row = dir.second;
+
+        entries.clear ();
+
+        try {
+            for (const auto& entry : Glib::Dir (path)) {
+
+                const auto entryPath = Glib::build_filename (path, entry);
+
+                if (!Glib::file_test (entryPath, Glib::FILE_TEST_IS_REGULAR)) {
+                    continue;
+                }
+
+                entries.push_back (entryPath);
+            }
+        } catch (Glib::Exception&) {}
+
+        std::sort (entries.begin (), entries.end ());
+
+        for (const auto& entry : entries) {
 
-            Gtk::TreeModel::Row newFolderMenu = *appendToModel( m_model, parentRow );
-            newFolderMenu[ m_columns.label ] = current;
-            result += parseDir( fullname, &newFolderMenu );
-        } else {
             Glib::ustring name, extension, profileName;
-            splitClutFilename( current, name, extension, profileName );
+            splitClutFilename (entry, name, extension, profileName);
 
-            if ( extension == "tif" ||
+            extension = extension.casefold ();
-                    extension == "TIF" ||
+            if (extension.compare ("tif") != 0 && extension.compare ("png") != 0) {
-                    extension == "png" ||
+                continue;
-                    extension == "PNG" ) {
-                Gtk::TreeModel::Row newClut = *appendToModel( m_model, parentRow );
-                newClut[ m_columns.label ] = name;
-                newClut[ m_columns.clutFilename ] = fullname;
-                ++result;
             }
+
+            auto newRow = row ? *m_model->append (row.children ()) : *m_model->append ();
+            newRow[m_columns.label] = name;
+            newRow[m_columns.clutFilename] = entry;
+
+            ++fileCount;
+
+            notifySlowParseDir (startedAt);
         }
     }
 
-    return result;
+    return fileCount;
 }
 
 Glib::ustring ClutComboBox::getSelectedClut()

rtgui/filmsimulation.h

@@ -11,7 +11,7 @@
 class ClutComboBox : public MyComboBox
 {
 public:
-    int fillFromDir( Glib::ustring path );
+    int fillFromDir (const Glib::ustring& path);
     Glib::ustring getSelectedClut();
     void setSelectedClut( Glib::ustring filename );
     void addUnchangedEntry();
@@ -25,7 +25,7 @@ private:
         ClutColumns();
     };
 
-    int parseDir( Glib::ustring path, Gtk::TreeModel::Row *parentRow );
+    int parseDir (const Glib::ustring& path);
     Gtk::TreeIter findRowByClutFilename(  Gtk::TreeModel::Children childs, Glib::ustring filename );
 
     Glib::RefPtr<Gtk::TreeStore> m_model;