fast file access with memory mapped files

(compile option WITH_MYFILE_MMAP)

CMakeLists.txt

@@ -11,6 +11,7 @@ endif (APPLE)
 
 option (BUILD_SHARED "Build rawtherapee with shared libraries" OFF)
 option (WITH_RAWZOR "Build with Rawzor support" OFF)
+option (WITH_MYFILE_MMAP "Build using memory mapped file" OFF)
 option (OPTION_OMP "Build with OpenMP support" ON)
 
 # set install directories
@@ -123,6 +124,10 @@ if (WITH_RAWZOR)
     endif (WIN32)
 endif (WITH_RAWZOR)
 
+if (WITH_MYFILE_MMAP)
+	add_definitions (-DMYFILE_MMAP)
+endif (WITH_MYFILE_MMAP)
+
 if (OPTION_OMP)
 	find_package(OpenMP)
 	if (OPENMP_FOUND)

rtengine/myfile.cc

@@ -23,6 +23,85 @@
 #include <rwz_sdk.h>
 #endif
 
+// get mmap() sorted out
+#ifdef MYFILE_MMAP
+
+#ifdef WIN32
+
+#include <fcntl.h>
+#include <ddk/ntifs.h>
+
+// dummy values
+#define MAP_PRIVATE 1
+#define PROT_READ 1
+
+void* mmap(void *start, size_t length, int prot, int flags, int fd, off_t offset)
+{
+	HANDLE handle = CreateFileMapping((HANDLE)_get_osfhandle(fd), NULL, PAGE_WRITECOPY, 0, 0, NULL);
+
+	if (handle != NULL) {
+		start = MapViewOfFile(handle, FILE_MAP_COPY, 0, offset, length);
+		CloseHandle(handle);
+	}
+
+	return start;
+}
+
+int munmap(void *start, size_t length)
+{
+	UnmapViewOfFile(start);
+	return 0;
+}
+
+#else // WIN32
+
+#include <fcntl.h>
+#include <sys/mman.h>
+
+#endif // WIN32
+#endif // MYFILE_MMAP
+
+#ifdef MYFILE_MMAP
+
+IMFILE* fopen (const char* fname)
+{
+	int fd = ::open(fname,O_RDONLY);
+	if ( fd < 0 )
+		return 0;
+
+	struct stat stat_buffer;
+	if ( fstat(fd,&stat_buffer) < 0 )
+	{
+		printf("no stat\n");
+		close(fd);
+		return 0;
+	}
+
+	void* data = mmap(0,stat_buffer.st_size,PROT_READ,MAP_PRIVATE,fd,0);
+	if ( data == 0 )
+	{
+		printf("no mmap\n");
+		close(fd);
+		return 0;
+	}
+
+	IMFILE* mf = new IMFILE;
+
+	mf->fd = fd;
+	mf->pos = 0;
+	mf->size = stat_buffer.st_size;
+	mf->data = (char*)data;
+	mf->eof = false;
+
+	return mf;
+}
+
+IMFILE* gfopen (const char* fname)
+{
+	return fopen(fname);
+}
+#else
+
 IMFILE* fopen (const char* fname) {
 
 	FILE* f = fopen (fname, "rb");
@@ -98,21 +177,34 @@ IMFILE* gfopen (const char* fname) {
 #endif
 	return mf;
 }
+#endif //MYFILE_MMAP
 
 IMFILE* fopen (unsigned* buf, int size) {
 
 	IMFILE* mf = new IMFILE;
+	mf->fd = -1;
 	mf->size = size;
 	mf->data = new char [mf->size];
-	memcpy (mf->data, buf, size);
+	memcpy ((void*)mf->data, buf, size);
 	mf->pos = 0;
 	mf->eof = false;
 	return mf;
 }
 
 void fclose (IMFILE* f) {
-
+#ifdef MYFILE_MMAP
+	if ( f->fd == -1 )
+	{
+		delete [] f->data;
+	}
+	else
+	{
+		munmap((void*)f->data,f->size);
+		close(f->fd);
+	}
+#else
 	delete [] f->data;
+#endif
 	delete f;
 }
 

rtengine/myfile.h

@@ -23,7 +23,7 @@
 #include <stdio.h>
 #include <string.h>
 struct IMFILE {
-
+	int fd;
 	int pos;
 	int size;
 	char* data;
@@ -86,8 +86,8 @@ inline int fread (void* dst, int es, int count, IMFILE* f) {
 	}
 }
 
-inline char* fdata(int offset, IMFILE* f) {
+inline unsigned char* fdata(int offset, IMFILE* f) {
-	return f->data + offset;
+	return (unsigned char*)f->data + offset;
 }
 
 int fscanf (IMFILE* f, const char* s ...);

rtengine/rtthumbnail.cc

@@ -207,7 +207,7 @@ Thumbnail* Thumbnail::loadQuickFromRaw (const Glib::ustring& fname, RawMetaDataL
     rml.ciffBase = ri->get_ciffBase();
     rml.ciffLength = ri->get_ciffLen();
 
-	Thumbnail* tpp = Thumbnail::loadFromMemory(fdata(ri->get_thumbOffset(),ri->get_file()),ri->get_thumbLength(),w,h,fixwh);
+	Thumbnail* tpp = Thumbnail::loadFromMemory((const char*)fdata(ri->get_thumbOffset(),ri->get_file()),ri->get_thumbLength(),w,h,fixwh);
 
 	if ( tpp == 0 )
 	{