Improve preview loader parallelism

rtgui/CMakeLists.txt

@@ -14,6 +14,7 @@ set (BASESOURCEFILES
     whitebalance.cc   vignetting.cc rotate.cc distortion.cc
     crophandler.cc curveeditor.cc dirbrowser.cc
     filecatalog.cc
+    previewloader.cc
     histogrampanel.cc history.cc  imagearea.cc
     imageareapanel.cc iptcpanel.cc labcurve.cc lumadenoise.cc main.cc
     multilangmgr.cc mycurve.cc options.cc

rtgui/filecatalog.cc

@@ -27,10 +27,6 @@
 #include <thumbimageupdater.h>
 #include <safegtk.h>
 
-#ifdef _OPENMP
-#include <omp.h>
-#endif
-
 #define CHECKTIME 2000
 
 extern Glib::ustring argv0;
@@ -48,10 +44,8 @@ int _directoryUpdater (void* cat) {
 }
 #endif
 
-FileCatalog::FileCatalog (CoarsePanel* cp, ToolBar* tb) : listener(NULL), fslistener(NULL), hasValidCurrentEFS(false), filterPanel(NULL), coarsePanel(cp), toolBar(tb) {
+FileCatalog::FileCatalog (CoarsePanel* cp, ToolBar* tb) : selectedDirectoryId(1), listener(NULL), fslistener(NULL), hasValidCurrentEFS(false), filterPanel(NULL), coarsePanel(cp), toolBar(tb) {
 
-    previewLoader.setPreviewLoaderListener (this);
-    
     //  construct and initialize thumbnail browsers
         fileBrowser = new FileBrowser();
         fileBrowser->setFileBrowserListener (this);
@@ -224,19 +218,23 @@ void FileCatalog::closeDir () {
         wdMonitor = NULL;
     }
 #endif
+
+	// ignore old requests
+	++selectedDirectoryId;
+
     // terminate thumbnail preview loading
-    previewLoader.terminate ();
+    previewLoader->removeAllJobs ();
 
     // terminate thumbnail updater
     thumbImageUpdater->removeAllJobs ();
 
     // remove entries
+    selectedDirectory = "";
     fileBrowser->close ();  
 	fileNameList.clear ();
 	
     dirEFS.clear ();
     hasValidCurrentEFS = false;
-    selectedDirectory = "";
     redrawAll ();
 }
 
@@ -272,7 +270,6 @@ void FileCatalog::dirSelected (const Glib::ustring& dirname, const Glib::ustring
         }
 
         _refreshProgressBar ();
-        previewLoader.process ();
 
 #ifdef _WIN32
       wdMonitor = new WinDirMonitor (selectedDirectory, this);
@@ -314,7 +311,14 @@ int refreshpb (void* data) {
     return 0;
 }
 
-void FileCatalog::previewReady (FileBrowserEntry* fdn) {
+void FileCatalog::previewReady (int dir_id, FileBrowserEntry* fdn) {
+
+    GThreadLock lock;
+	
+	if ( dir_id != selectedDirectoryId )
+	{
+		return;
+	}
 
     // put it into the "full directory" browser
     fdn->setImageAreaToolListener (iatlistener);
@@ -372,42 +376,18 @@ void FileCatalog::_previewsFinished () {
 	}
 }
 
-void FileCatalog::previewsFinished () {
+void FileCatalog::previewsFinished (int dir_id) {
+
+	if ( dir_id != selectedDirectoryId )
+	{
+		return;
+	}
 
     if (!hasValidCurrentEFS) 
         currentEFS = dirEFS;
     g_idle_add (prevfinished, this);
 }
 
-void PreviewLoader::remove (Glib::ustring fname) {
-	std::list<DirEntry>::iterator i;
-	for (i=jqueue.begin(); i!=jqueue.end(); i++) 
-		if (i->fullName==fname)
-			break;
-	if (i!=jqueue.end())
-		jqueue.erase (i);	
-}
-
-void PreviewLoader::start () {
-
-    jqueue.sort ();
-}
-
-void PreviewLoader::process (DirEntry& current) {
-
-	if (Glib::file_test (current.fullName, Glib::FILE_TEST_EXISTS)) {
-	    Thumbnail* tmb = cacheMgr->getEntry (current.fullName);
-	    if (tmb && pl) 
-	  	    pl->previewReady (new FileBrowserEntry (tmb, current.fullName));
-	}
-}
-
-void PreviewLoader::end () {
-
-    if (pl)
-        pl->previewsFinished ();
-}
-
 void FileCatalog::setEnabled (bool e) {
 
     enabled = e;
@@ -426,13 +406,11 @@ void FileCatalog::refreshAll () {
 void FileCatalog::_openImage (std::vector<Thumbnail*> tmb) {
 
     if (enabled && listener!=NULL) {
-        previewLoader.stop ();
         for (size_t i=0; i<tmb.size(); i++) {
             if (editedFiles.find (tmb[i]->getFileName())==editedFiles.end())
                 listener->fileSelected (tmb[i]);
             tmb[i]->decreaseRef ();
         }
-        previewLoader.process ();
     }
 }
 
@@ -761,10 +739,8 @@ int FileCatalog::reparseDirectory () {
 				break;
 			}
 		if (!found) {
-			previewLoader.stop ();
 			checkAndAddFile (Gio::File::create_for_parse_name (nfileNameList[i]));
             _refreshProgressBar ();
-			previewLoader.process ();
 		}
 	}
 
@@ -801,7 +777,7 @@ void FileCatalog::checkAndAddFile (Glib::RefPtr<Gio::File> file) {
     if (info && info->get_file_type() != Gio::FILE_TYPE_DIRECTORY && (!info->is_hidden() || !options.fbShowHidden)) {
         int lastdot = info->get_name().find_last_of ('.');
         if (options.is_extention_enabled(lastdot!=(int)Glib::ustring::npos ? info->get_name().substr (lastdot+1) : "")){
-						previewLoader.add (DirEntry (file->get_parse_name()));
+						previewLoader->add (selectedDirectoryId,file->get_parse_name(),this);
             previewsToLoad++;
 				}
     }
@@ -823,7 +799,7 @@ void FileCatalog::addAndOpenFile (const Glib::ustring& fname) {
         Thumbnail* tmb = cacheMgr->getEntry (file->get_parse_name());
         if (tmb) {
             FileBrowserEntry* entry = new FileBrowserEntry (tmb, file->get_parse_name());
-  	        previewReady (entry);
+  	        previewReady (selectedDirectoryId,entry);
             // open the file
             FCOIParams* params = new FCOIParams;
             params->catalog = this;
@@ -856,25 +832,15 @@ bool FileCatalog::trashIsEmpty () {
 
 void FileCatalog::zoomIn () {
 
-    bool pLoad = previewLoader.runs();
-    if (pLoad)
-        previewLoader.stop ();
         
     fileBrowser->zoomIn ();
         
-    if (pLoad)
-        previewLoader.process ();
 }
 void FileCatalog::zoomOut () {
 
-    bool pLoad = previewLoader.runs();
-    if (pLoad)
-        previewLoader.stop ();
         
     fileBrowser->zoomOut ();
         
-    if (pLoad)
-        previewLoader.process ();
 }
 void FileCatalog::refreshEditedState (const std::set<Glib::ustring>& efiles) {
 
@@ -970,50 +936,27 @@ bool FileCatalog::handleShortcutKey (GdkEventKey* event) {
     return false;
 }
 
-PreviewMultiLoader::PreviewMultiLoader () {
+#if 0
-	next=0;
+void PreviewMultiLoader::setPreviewLoaderListener (PreviewLoaderListener* p) { 
-	loaderCount=1;
+	loadA.setPreviewLoaderListener(p); loadB.setPreviewLoaderListener(p);
-
-	#ifdef _OPENMP
-	loaderCount=omp_get_num_procs();
-	// If there are pleny of processor, spare one for snappy image editing
-	if (loaderCount>3) loaderCount--;
-	#endif
-
-	loaders=new PreviewLoader[loaderCount];
-}
-
-void PreviewMultiLoader::setPreviewLoaderListener (PreviewLoaderListener* p) {
-	for (int i=0;i<loaderCount;i++) loaders[i].setPreviewLoaderListener(p);
 }
 
 // Simple round robin
 void PreviewMultiLoader::add(DirEntry de) {
-	loaders[next].add(de);
+	if (next==0) {
-	next++;
+		loadA.add(de);
-	if (next>=loaderCount) next=0;
+		next=1;
+	} else {
+		loadB.add(de);
+		next=0;
+	}
 }
 
-void PreviewMultiLoader::start () { 
+void PreviewMultiLoader::start () { loadA.start(); loadB.start(); }
-	for (int i=0;i<loaderCount;i++) loaders[i].start();
+void PreviewMultiLoader::process () { loadA.process (); loadB.process(); }
-}
+void PreviewMultiLoader::remove  (Glib::ustring fname) { loadA.remove(fname); loadB.remove(fname); }
-void PreviewMultiLoader::process () { 
+void PreviewMultiLoader::end () { loadA.end(); loadB.end(); }
-	for (int i=0;i<loaderCount;i++) loaders[i].process (); 
+bool PreviewMultiLoader::runs () { return loadA.runs() || loadB.runs(); }
-}
+void PreviewMultiLoader::terminate () { loadA.terminate(); loadB.terminate(); }
-void PreviewMultiLoader::remove  (Glib::ustring fname) { 
+void PreviewMultiLoader::stop () { loadA.stop(); loadB.stop(); }
-	for (int i=0;i<loaderCount;i++) loaders[i].remove(fname); 
+#endif
-}
-void PreviewMultiLoader::end () { 
-	for (int i=0;i<loaderCount;i++) loaders[i].end();
-}
-bool PreviewMultiLoader::runs () { 
-	for (int i=0;i<loaderCount;i++) if (loaders[i].runs()) return true;
-	
-	return false;
-}
-void PreviewMultiLoader::terminate () { 
-	for (int i=0;i<loaderCount;i++) loaders[i].terminate(); 
-}
-void PreviewMultiLoader::stop () { 
-	for (int i=0;i<loaderCount;i++) loaders[i].stop();
-}

rtgui/filecatalog.h

@@ -33,12 +33,7 @@
 #include <coarsepanel.h>
 #include <toolbar.h>
 #include <filterpanel.h>
-
+#include <previewloader.h>
-class PreviewLoaderListener {
-  public:
-    virtual void previewReady (FileBrowserEntry* fd) {}
-    virtual void previewsFinished () {}
-};
 
 class DirEntry {
 
@@ -52,6 +47,7 @@ class DirEntry {
 	}
 };
 
+#if 0
 class PreviewLoader : public ProcessingThread<DirEntry> {
 
   protected:
@@ -71,11 +67,11 @@ class PreviewLoader : public ProcessingThread<DirEntry> {
 // Same interface as a normal PreviewLoader to minimize effects on code
 class PreviewMultiLoader {
 protected:
-	PreviewLoader *loaders;
+	PreviewLoader loadA,loadB;
-	int next,loaderCount;
+	int next;
 
 public:
-	PreviewMultiLoader ();
+	PreviewMultiLoader () { next=0; }
     
 	void setPreviewLoaderListener (PreviewLoaderListener* p);
 
@@ -90,6 +86,7 @@ public:
 	void terminate ();
 	void stop ();
 };
+#endif
 
 class FileCatalog : public Gtk::VBox,
                     public DirSelectionListener, 
@@ -104,11 +101,12 @@ class FileCatalog : public Gtk::VBox,
 
         Gtk::HBox* hBox;
         Glib::ustring selectedDirectory;
+        int selectedDirectoryId;
         bool enabled;
         
 		// Restore PreviewLoader if the new threadsafe is not that threadsafe ;-)
         //PreviewLoader previewLoader;
-        PreviewMultiLoader previewLoader;
+        //PreviewMultiLoader previewLoader;
         FileSelectionListener* listener;
         FileSelectionChangeListener* fslistener;
         ImageAreaToolListener* iatlistener;
@@ -177,8 +175,8 @@ class FileCatalog : public Gtk::VBox,
                 void refreshEditedState (const std::set<Glib::ustring>& efiles);
                 
                 // previewloaderlistener interface
-				void previewReady (FileBrowserEntry* fdn);
+				void previewReady (int dir_id, FileBrowserEntry* fdn);
-				void previewsFinished ();
+				void previewsFinished (int dir_id);
                 void _previewsFinished ();
                 void _refreshProgressBar ();
 

rtgui/previewloader.cc

@@ -0,0 +1,163 @@
+/*
+ *  This file is part of RawTherapee.
+ *
+ *  Copyright (c) 2004-2010 Gabor Horvath <hgabor@rawtherapee.com>
+ *
+ *  RawTherapee is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  RawTherapee is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with RawTherapee.  If not, see <http://www.gnu.org/licenses/>.
+ */
+
+#include <set>
+#include <previewloader.h>
+#include <guiutils.h>
+
+
+#define THREAD_NUM 4
+
+#define DEBUG(format,args...) 
+//#define DEBUG(format,args...) printf("PreviewLoader::%s: " format "\n", __FUNCTION__, ## args)
+
+class PreviewLoader::Impl
+{
+public:
+	struct Job
+	{
+		Job(int dir_id, const Glib::ustring& dir_entry, PreviewLoaderListener* listener):
+			dir_id_(dir_id),
+			dir_entry_(dir_entry),
+			listener_(listener)
+		{}
+
+		Job():
+			dir_id_(0),
+			listener_(0)
+		{}
+
+		int dir_id_;
+		Glib::ustring dir_entry_;
+		PreviewLoaderListener* listener_;
+	};
+
+	struct JobCompare
+	{
+		bool operator()(const Job& lhs, const Job& rhs)
+		{
+			if ( lhs.dir_id_ == rhs.dir_id_ )
+			{
+				return lhs.dir_entry_ < rhs.dir_entry_;
+			}
+			return lhs.dir_id_ < rhs.dir_id_;
+		}
+	};
+
+	typedef std::set<Job,JobCompare> JobSet;
+
+	Impl():
+		threadPool_(new Glib::ThreadPool(THREAD_NUM,0))
+	{}
+
+	Glib::ThreadPool* threadPool_;
+
+	Glib::Mutex mutex_;
+
+	JobSet jobs_;
+
+	void
+	processNextJob(void)
+	{ 
+		Job j;
+		{
+			Glib::Mutex::Lock lock(mutex_);
+
+			// nothing to do; could be jobs have been removed
+			if ( jobs_.empty() )
+			{
+				DEBUG("processing: nothing to do");
+				return;
+			}
+
+			// copy and remove front job
+			j = *jobs_.begin();
+			jobs_.erase(jobs_.begin());
+			DEBUG("processing %s",j.dir_entry_.c_str());
+			DEBUG("%d job(s) remaining",jobs_.size());
+		}
+
+		// unlock and do processing; will relock on block exit, then call listener
+		// if something got
+		Thumbnail* tmb = 0;
+		{
+			if (Glib::file_test(j.dir_entry_, Glib::FILE_TEST_EXISTS)) 
+			{
+				tmb = cacheMgr->getEntry(j.dir_entry_);
+			}
+		}
+
+		// we got something so notify listener
+		if ( tmb )
+		{
+			j.listener_->previewReady(j.dir_id_,new FileBrowserEntry(tmb,j.dir_entry_));
+		}
+
+		// signal at end
+		if ( jobs_.empty() )
+		{
+			j.listener_->previewsFinished(j.dir_id_);
+		}
+	}
+};
+
+PreviewLoader::PreviewLoader():
+	impl_(new Impl())
+{
+}
+
+PreviewLoader*
+PreviewLoader::getInstance(void)
+{
+	// this will not be deleted...
+	static PreviewLoader* instance_ = 0;
+	if ( instance_ == 0 )
+	{
+		instance_ = new PreviewLoader();
+	}
+	return instance_;
+}
+
+void 
+PreviewLoader::add(int dir_id, const Glib::ustring& dir_entry, PreviewLoaderListener* l)
+{
+	// somebody listening?
+	if ( l != 0 )
+	{
+		Glib::Mutex::Lock lock(impl_->mutex_);
+
+		// create a new job and append to queue
+		DEBUG("saving job %s",dir_entry.c_str());
+		impl_->jobs_.insert(Impl::Job(dir_id,dir_entry,l));
+
+		// queue a run request
+		DEBUG("adding run request %s",dir_entry.c_str());
+		impl_->threadPool_->push(sigc::mem_fun(*impl_, &PreviewLoader::Impl::processNextJob));
+	}
+}
+
+void 
+PreviewLoader::removeAllJobs(void) 
+{ 
+	DEBUG("stop");
+
+	impl_->jobs_.clear();
+}
+
+

rtgui/previewloader.h

@@ -0,0 +1,93 @@
+/*
+ *  This file is part of RawTherapee.
+ *
+ *  Copyright (c) 2004-2010 Gabor Horvath <hgabor@rawtherapee.com>
+ *
+ *  RawTherapee is free software: you can redistribute it and/or modify
+ *  it under the terms of the GNU General Public License as published by
+ *  the Free Software Foundation, either version 3 of the License, or
+ *  (at your option) any later version.
+ * 
+ *  RawTherapee is distributed in the hope that it will be useful,
+ *  but WITHOUT ANY WARRANTY; without even the implied warranty of
+ *  MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ *  GNU General Public License for more details.
+ *
+ *  You should have received a copy of the GNU General Public License
+ *  along with RawTherapee.  If not, see <http://www.gnu.org/licenses/>.
+ */
+#ifndef _PREVIEWLOADER_
+#define _PREVIEWLOADER_
+
+#include <set>
+#include <glibmm.h>
+#include <filebrowserentry.h>
+
+class PreviewLoaderListener 
+{
+public:
+
+	/** 
+	 * @brief a preview is ready
+	 * 
+	 * @param dir_id directory ID this is for
+	 * @param fd entry
+	 */
+	virtual void previewReady (int dir_id, FileBrowserEntry* fd) {}
+
+	/** 
+	 * @brief all previews have finished loading
+	 */
+	virtual void previewsFinished (int dir_id_) {}
+};
+
+class PreviewLoader
+{
+  public:
+
+	/** 
+	 * @brief Singleton entry point.
+	 * 
+	 * @note expects to be called inside gtk thread lock
+	 *
+	 * @return Pointer to thumbnail image updater.
+	 */
+	static PreviewLoader* getInstance(void);
+
+	/** 
+	 * @brief Add an thumbnail image update request.
+	 *
+	 * Code will add the request to the queue and, if needed, start a pool
+	 * thread to process it.
+	 * 
+	 * @param dir_id directory we're looking at
+	 * @param dir_entry entry in it
+	 * @param l listener
+	 */
+    void add(int dir_id, const Glib::ustring& dir_entry, PreviewLoaderListener* l);
+
+	/** 
+	 * @brief Stop processing and remove all jobs.
+	 *
+	 * Will not return till all jobs have completed.
+	 *
+	 * @note expects to be called inside gtk thread lock
+	 */
+    void removeAllJobs(void);
+
+  private:
+
+    PreviewLoader();
+
+	class Impl;
+	Impl* impl_;
+};
+
+/** 
+ * @brief Singleton boiler plate.
+ *
+ * To use: \c previewLoader->start() ,
+ */
+#define previewLoader PreviewLoader::getInstance()
+
+#endif

rtgui/thumbimageupdater.cc

@@ -27,37 +27,37 @@
 #define DEBUG(format,args...) 
 //#define DEBUG(format,args...) printf("ThumbImageUpdate::%s: " format "\n", __FUNCTION__, ## args)
 
-struct 
-Job
-{
-	Job(Thumbnail* thumbnail, const rtengine::procparams::ProcParams& pparams,
-				int height, bool* priority,
-				ThumbImageUpdateListener* listener):
-		thumbnail_(thumbnail),
-		pparams_(pparams),
-		height_(height),
-		priority_(priority),
-		listener_(listener)
-	{}
-
-	Job():
-		thumbnail_(0),
-		listener_(0)
-	{}
-
-	Thumbnail* thumbnail_;
-	rtengine::procparams::ProcParams pparams_;
-	int height_;
-	bool* priority_;
-	ThumbImageUpdateListener* listener_;
-};
-
-typedef std::list<Job> JobList;
-
 class
 ThumbImageUpdater::Impl
 {
 public:
+
+	struct Job
+	{
+		Job(Thumbnail* thumbnail, const rtengine::procparams::ProcParams& pparams,
+					int height, bool* priority,
+					ThumbImageUpdateListener* listener):
+			thumbnail_(thumbnail),
+			pparams_(pparams),
+			height_(height),
+			priority_(priority),
+			listener_(listener)
+		{}
+
+		Job():
+			thumbnail_(0),
+			listener_(0)
+		{}
+
+		Thumbnail* thumbnail_;
+		rtengine::procparams::ProcParams pparams_;
+		int height_;
+		bool* priority_;
+		ThumbImageUpdateListener* listener_;
+	};
+
+	typedef std::list<Job> JobList;
+
 	Impl():
 		threadPool_(new Glib::ThreadPool(THREAD_NUM,0)),
 		active_(0),
@@ -176,7 +176,7 @@ ThumbImageUpdater::add(Thumbnail* t, const rtengine::procparams::ProcParams& par
 	Glib::Mutex::Lock lock(impl_->mutex_);
 
     // look up if an older version is in the queue
-    JobList::iterator i(impl_->jobs_.begin());
+	Impl::JobList::iterator i(impl_->jobs_.begin());
 	for ( ; i != impl_->jobs_.end(); ++i )
 	{
 		if ( i->thumbnail_ == t &&
@@ -193,7 +193,7 @@ ThumbImageUpdater::add(Thumbnail* t, const rtengine::procparams::ProcParams& par
 
 	// create a new job and append to queue
 	DEBUG("queing job %s",t->getFileName().c_str());
-	impl_->jobs_.push_back(Job(t,params,height,priority,l));
+	impl_->jobs_.push_back(Impl::Job(t,params,height,priority,l));
 
 	DEBUG("adding run request %s",t->getFileName().c_str());
 	impl_->threadPool_->push(sigc::mem_fun(*impl_, &ThumbImageUpdater::Impl::processNextJob));
@@ -207,12 +207,12 @@ ThumbImageUpdater::removeJobs(ThumbImageUpdateListener* listener)
 
 	Glib::Mutex::Lock lock(impl_->mutex_);
 
-	for( JobList::iterator i(impl_->jobs_.begin()); i != impl_->jobs_.end(); )
+	for( Impl::JobList::iterator i(impl_->jobs_.begin()); i != impl_->jobs_.end(); )
 	{
 		if (i->listener_ == listener)
 		{
 			DEBUG("erasing specific job");
-			JobList::iterator e(i++);
+			Impl::JobList::iterator e(i++);
 			impl_->jobs_.erase(e);
 		}
 		else