mmap #22

cmake/CMakeLists.txt

@@ -29,6 +29,7 @@ OPTION(USE_IFC4 "Use IFC 4 instead of IFC 2x3 (full rebuild recommended when swi
 OPTION(BUILD_IFCPYTHON "Build IfcPython." ON)
 OPTION(BUILD_EXAMPLES "Build example applications." ON)
 OPTION(USE_VLD "Use Visual Leak Detector for debugging memory leaks, MSVC-only." OFF)
+OPTION(USE_MMAP "Adds a command line options to parse IFC files from memory mapped files using Boost.Iostreams" OFF)
 OPTION(BUILD_IFCMAX "Build IfcMax, a 3ds Max plug-in, Windows-only." OFF)
 OPTION(BUILD_SHARED_LIBS "Build IfcParse and IfcGeom as shared libs (SO/DLL)." OFF)
 # TODO QtViewer is deprecated ATM as it uses the 0.4 API
@@ -112,7 +113,19 @@ IF(WIN32)
 	SET(Boost_USE_STATIC_RUNTIME ON)
 	SET(Boost_USE_MULTITHREADED ON)
 ENDIF()
-FIND_PACKAGE(Boost REQUIRED COMPONENTS system program_options regex thread date_time)
+
+set(BOOST_COMPONENTS system program_options regex thread date_time)
+if(USE_MMAP)
+    if(MSVC)
+        # filesystem is necessary for the utf-16 wpath
+        set(BOOST_COMPONENTS ${BOOST_COMPONENTS} iostreams filesystem)
+    else()
+        set(BOOST_COMPONENTS ${BOOST_COMPONENTS} iostreams)
+    endif()
+    add_definitions(-DUSE_MMAP)
+endif()
+
+FIND_PACKAGE(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS})
 MESSAGE(STATUS "Boost include files found in ${Boost_INCLUDE_DIRS}")
 MESSAGE(STATUS "Boost libraries found in ${Boost_LIBRARY_DIRS}")
 

nix/build-all.py

@@ -454,7 +454,7 @@ os.environ["CXXFLAGS"]=OLD_CXX_FLAGS
 os.environ["CFLAGS"]=OLD_C_FLAGS
 
 str_concat = lambda prefix: lambda postfix: "=".join((prefix, postfix.strip()))
-build_dependency("boost-%s" % (BOOST_VERSION,), mode="bjam", build_tool_args=["--stagedir=%s/install/boost-%s" % (DEPS_DIR, BOOST_VERSION), "--with-system", "--with-program_options", "--with-regex", "--with-thread", "--with-date_time", "link=static"]+BOOST_ADDRESS_MODEL+list(map(str_concat("cxxflags"), CXXFLAGS.strip().split(' '))) + list(map(str_concat("linkflags"), LDFLAGS.strip().split(' '))) + ["stage"], download_url="http://downloads.sourceforge.net/project/boost/boost/%s/" % (BOOST_VERSION,), download_name="boost_%s.tar.bz2" % (BOOST_VERSION_UNDERSCORE,))
+build_dependency("boost-%s" % (BOOST_VERSION,), mode="bjam", build_tool_args=["--stagedir=%s/install/boost-%s" % (DEPS_DIR, BOOST_VERSION), "--with-system", "--with-program_options", "--with-regex", "--with-thread", "--with-date_time", "--with-iostreams", "link=static"]+BOOST_ADDRESS_MODEL+list(map(str_concat("cxxflags"), CXXFLAGS.strip().split(' '))) + list(map(str_concat("linkflags"), LDFLAGS.strip().split(' '))) + ["stage"], download_url="http://downloads.sourceforge.net/project/boost/boost/%s/" % (BOOST_VERSION,), download_name="boost_%s.tar.bz2" % (BOOST_VERSION_UNDERSCORE,))
 
 build_dependency(name="icu-%s" % (ICU_VERSION,), mode="icu", build_tool_args=["--enable-static", "--disable-shared"], download_url="http://download.icu-project.org/files/icu4c/%s/" % (ICU_VERSION,), download_name="icu4c-%s-src.tgz" % (ICU_VERSION_UNDERSCORE,))
 
@@ -481,6 +481,7 @@ run_cmake("", cmake_args=[
     "-DICU_LIBRARY_DIR="         "%s/install/icu-%s/lib" % (DEPS_DIR, ICU_VERSION),
     "-DPCRE_LIBRARY_DIR="        "%s/install/pcre-%s/lib" % (DEPS_DIR, PCRE_VERSION),
     "-DBUILD_IFCPYTHON="         "OFF",
+    "-DUSE_MMAP="                "ON",
     "-DCMAKE_INSTALL_PREFIX="    "%s/install/ifcopenshell" % (DEPS_DIR,)], cmake_dir=CMAKE_DIR, cwd=executables_dir)
 
 logger.info("\rBuilding executables...   ")

src/ifcconvert/IfcConvert.cpp

@@ -148,7 +148,7 @@ size_t read_filters_from_file(const std::string&, inclusion_filter&, inclusion_t
 void parse_filter(geom_filter &, const std::vector<std::string>&);
 std::vector<IfcGeom::filter_t> setup_filters(const std::vector<geom_filter>&, const std::string&);
 
-bool init_input_file(const std::string& filename, IfcParse::IfcFile& ifc_file, bool no_progress);
+bool init_input_file(const std::string& filename, IfcParse::IfcFile& ifc_file, bool no_progress, bool mmap);
 
 int main(int argc, char** argv)
 {
@@ -162,8 +162,12 @@ int main(int argc, char** argv)
 
     po::options_description fileio_options;
 	fileio_options.add_options()
+#ifdef USE_MMAP
+		("mmap", "use memory-mapped file for input")
+#endif
 		("input-file", po::value<std::string>(), "input IFC file")
 		("output-file", po::value<std::string>(), "output geometry file");
+		
 
     double deflection_tolerance;
     inclusion_filter include_filter;
@@ -337,6 +341,7 @@ int main(int argc, char** argv)
         print_usage();
         return EXIT_FAILURE;
     }
+	const bool mmap = vmap.count("mmap") != 0;
 	const bool verbose = vmap.count("verbose") != 0;
 	const bool no_progress = vmap.count("no-progress") != 0;
 	const bool weld_vertices = vmap.count("weld-vertices") != 0;
@@ -427,7 +432,7 @@ int main(int argc, char** argv)
     if (output_extension == ".xml") {
         int exit_code = EXIT_FAILURE;
         try {
-            if (init_input_file(input_filename, ifc_file, no_progress)) {
+            if (init_input_file(input_filename, ifc_file, no_progress, mmap)) {
                 XmlSerializer s(output_temp_filename);
                 s.setFile(&ifc_file);
                 Logger::Status("Writing XML output...");
@@ -564,7 +569,7 @@ int main(int argc, char** argv)
 	time_t start,end;
 	time(&start);
 	
-    if (!init_input_file(input_filename, ifc_file, no_progress)) {
+    if (!init_input_file(input_filename, ifc_file, no_progress, mmap)) {
         return EXIT_FAILURE;
     }
 
@@ -691,12 +696,16 @@ void write_log() {
 	}
 }
 
-bool init_input_file(const std::string &filename, IfcParse::IfcFile &ifc_file, bool no_progress)
+bool init_input_file(const std::string &filename, IfcParse::IfcFile &ifc_file, bool no_progress, bool mmap)
 {
     // Prevent IfcFile::Init() prints by setting output to null temporarily
     if (no_progress) { Logger::SetOutput(NULL, &log_stream); }
 
-    if (!ifc_file.Init(filename)) {
+#ifdef USE_MMAP
+	if (!ifc_file.Init(filename, mmap)) {
+#else
+	if (!ifc_file.Init(filename)) {
+#endif
         Logger::Error("Unable to parse input file '" + filename + "'");
         return false;
     }

src/ifcparse/IfcFile.h

@@ -107,7 +107,11 @@ public:
 	/// in the first function argument.
 	IfcEntityList::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level=-1);
 
+#ifdef USE_MMAP
+	bool Init(const std::string& fn, bool mmap=false);
+#else
 	bool Init(const std::string& fn);
+#endif
 	bool Init(std::istream& fn, int len);
 	bool Init(void* data, int len);
 	bool Init(IfcParse::IfcSpfStream* f);

src/ifcparse/IfcParse.cpp

@@ -46,6 +46,10 @@
 #include "../ifcparse/Ifc2x3-latebound.h"
 #endif
 
+#ifdef USE_MMAP
+#include <boost/filesystem/path.hpp>
+#endif
+
 #define PERMISSIVE_FLOAT
 
 using namespace IfcParse;
@@ -105,69 +109,99 @@ void init_locale() {
 #endif
 
 // 
-// Opens the file, gets the filesize and reads a chunk in memory
+// Opens the file and gets the filesize
 //
+#ifdef USE_MMAP
+IfcSpfStream::IfcSpfStream(const std::string& fn, bool mmap)
+#else
 IfcSpfStream::IfcSpfStream(const std::string& fn)
-		: stream(0)
+#endif
-		, buffer(0)
+	: stream(0)
+	, buffer(0)
+	, valid(false)
+	, eof(false)
 {
-	eof = false;
 #ifdef _MSC_VER
 	int fn_buffer_size = MultiByteToWideChar(CP_UTF8, 0, fn.c_str(), -1, 0, 0);
 	wchar_t* fn_wide = new wchar_t[fn_buffer_size];
 	MultiByteToWideChar(CP_UTF8, 0, fn.c_str(), -1, fn_wide, fn_buffer_size);
-	stream = _wfopen(fn_wide, L"rb");
+
+#ifdef USE_MMAP
+	if (mmap) {
+		mfs = boost::iostreams::mapped_file_source(boost::filesystem::wpath(fn_wide));
+	} else {
+#endif
+		stream = _wfopen(fn_wide, L"rb");
+#ifdef USE_MMAP	
+	}
+#endif
+
 	delete[] fn_wide;
 #else
-	stream = fopen(fn.c_str(), "rb");
+
+#ifdef USE_MMAP
+	if (mmap) {
+		mfs = boost::iostreams::mapped_file_source(fn);
+	} else {
 #endif
-	if (stream == NULL) {
+		stream = fopen(fn.c_str(), "rb");
-		valid = false;
+#ifdef USE_MMAP	
-		return;
+	}
+#endif
+
+#endif
+
+#ifdef USE_MMAP
+	if (mmap) {
+		if (!mfs.is_open()) {
+			return;
+		}
+
+		valid = true;
+		buffer = mfs.data();
+		ptr = 0;
+		len = mfs.size();
+	} else {
+#endif
+		if (stream == NULL) {
+			return;
+		}
+
+		valid = true;
+		fseek(stream, 0, SEEK_END);
+		size = (unsigned int)ftell(stream);
+		rewind(stream);
+		char* buffer_rw = new char[size];
+		len = (unsigned int)fread(buffer_rw, 1, size, stream);
+		buffer = buffer_rw;
+		eof = len == 0;
+		ptr = 0;
+		fclose(stream);
+#ifdef USE_MMAP	
 	}
-	valid = true;
-	fseek(stream, 0, SEEK_END);
-	size = (unsigned int) ftell(stream);
-	rewind(stream);
-#ifdef BUF_SIZE
-	offset = 0;
-	paging = size > BUF_SIZE;
-	buffer = new char[size < BUF_SIZE ? size : BUF_SIZE];
-#else
-	buffer = new char[size];
 #endif
-	ptr = 0;
-	len = 0;
-	ReadBuffer(false);
 }
 
 IfcSpfStream::IfcSpfStream(std::istream& f, int l)
-		: stream(0)
+	: stream(0)
-		, buffer(0)
+	, buffer(0)
 {
 	eof = false;
 	size = l;
-#ifdef BUF_SIZE
+	char* buffer_rw = new char[size];
-	paging = false;
+	f.read(buffer_rw,size);
-	offset = 0;
+	buffer = buffer_rw;
-#endif
-	buffer = new char[size];
-	f.read(buffer,size);
 	valid = f.gcount() == size;
 	ptr = 0;
 	len = l;	
 }
 
 IfcSpfStream::IfcSpfStream(void* data, int l)
-		: stream(0)
+	: stream(0)
-		, buffer(0)
+	, buffer(0)
 {
 	eof = false;
 	size = l;
-#ifdef BUF_SIZE
-	paging = false;
-	offset = 0;
-#endif
 	buffer = (char*) data;
 	valid = true;
 	ptr = 0;
@@ -180,60 +214,22 @@ IfcSpfStream::~IfcSpfStream()
 }
 
 void IfcSpfStream::Close() {
-#ifdef BUF_SIZE
+#ifdef USE_MMAP
-	if ( paging ) fclose(stream);
+	if (mfs.is_open()) {
+		mfs.close();
+		return;
+	}
 #endif
 	delete[] buffer;
 }
 
-//
-// Reads a chunk of BUF_SIZE in memory and increments cursor if requested
-//
-void IfcSpfStream::ReadBuffer(bool inc) {
-#ifdef BUF_SIZE
-	if ( inc ) {
-		offset += len;
-		fseek(stream, offset, SEEK_SET);
-	}
-#else
-	(void)inc;
-#endif
-	eof = feof(stream) != 0;
-	if ( eof ) return;
-#ifdef BUF_SIZE
-	len = (unsigned int) fread(buffer, 1, size < BUF_SIZE ? size : BUF_SIZE, stream);
-#else
-	len = (unsigned int) fread(buffer, 1, size, stream);
-#endif
-	eof = len == 0;
-	ptr = 0;
-#ifdef BUF_SIZE
-	if (!paging) fclose(stream);
-#else
-	fclose(stream);
-#endif
-}
-
 //
 // Seeks an arbitrary position in the file
 //
 void IfcSpfStream::Seek(unsigned int o) {
-#ifdef BUF_SIZE
+	ptr = o;
-	if ( !paging ) {
+	if (ptr >= len) throw IfcException("Reading outside of file limits");
-#endif
+	eof = false;
-		ptr = o;
-		if (ptr >= len) throw IfcException("Reading outside of file limits");
-		eof = false;
-#ifdef BUF_SIZE
-	} else if ( o >= offset && (o < (offset+len)) ) {
-		ptr = o - offset;
-	} else {
-		offset = o;
-		clearerr(stream);
-		fseek(stream, o, SEEK_SET);
-		ReadBuffer(false);
-	}
-#endif
 }
 
 //
@@ -247,30 +243,14 @@ char IfcSpfStream::Peek() {
 // Returns the character at specified offset
 //
 char IfcSpfStream::Read(unsigned int o) {
-#ifdef BUF_SIZE
+	return buffer[o];
-	if ( ! paging ) {
-#endif
-		return buffer[o];
-#ifdef BUF_SIZE
-	} else if ( o >= offset && (o < (offset+len)) ) {
-		return buffer[o-offset];
-	} else {
-		clearerr(stream);
-		fseek(stream, o, SEEK_SET);
-		return ungetc(getc(stream), stream);
-	}
-#endif
 }
 
 //
 // Returns the cursor position
 //
 unsigned int IfcSpfStream::Tell() {
-#ifdef BUF_SIZE
-	return offset + ptr;
-#else
 	return ptr;
-#endif
 }
 
 //
@@ -278,16 +258,10 @@ unsigned int IfcSpfStream::Tell() {
 //
 void IfcSpfStream::Inc() {
 	if ( ++ptr == len ) { 
-#ifdef BUF_SIZE
+		eof = true;
-		if ( paging ) ReadBuffer();
+		return;
-		else {
-#endif
-			eof = true;
-			return;
-#ifdef BUF_SIZE
-		}
-#endif
 	}
+	/// @todo: Shouldn't this be a loop of some kind
 	const char current = IfcSpfStream::Peek();
 	if ( current == '\n' || current == '\r' ) IfcSpfStream::Inc();
 }
@@ -1245,9 +1219,15 @@ void IfcEntityInstanceData::setArgument(unsigned int i, Argument* a, IfcUtil::Ar
 // Parses the IFC file in fn
 // Creates the maps
 //
+#ifdef USE_MMAP
+bool IfcFile::Init(const std::string& fn, bool mmap) {
+	return IfcFile::Init(new IfcSpfStream(fn, mmap));
+}
+#else
 bool IfcFile::Init(const std::string& fn) {
 	return IfcFile::Init(new IfcSpfStream(fn));
 }
+#endif
 
 bool IfcFile::Init(std::istream& f, int len) {
 	return IfcFile::Init(new IfcSpfStream(f,len));

src/ifcparse/IfcSpfStream.h

@@ -19,7 +19,7 @@
  
 /*********************************************************************************
  *                                                                               *
- * Reads a file in chunks of BUF_SIZE and provides functions to access its       *
+ * Reads a file and provides functions to access its                             *
  * contents randomly and character by character                                  *
  *                                                                               *
  ********************************************************************************/
@@ -30,40 +30,34 @@
 #include <fstream>
 #include <string>
 
-#include "ifc_parse_api.h"
+#ifdef USE_MMAP
+#include <boost/iostreams/device/mapped_file.hpp>
+#endif
 
-// As of IfcOpenShell version 0.3.0 the paging functionality, which
+#include "ifc_parse_api.h"
-// loads a file on disk into multiple chunks, has been disabled.
-// It proved to be an inefficient way of working with large files,
-// as often these did not facilitate to be parsed in a sequential 
-// manner efficiently. To enable the paging functionality uncomment
-// the following statement.
-// #define BUF_SIZE (8 * 1024 * 1024)
 
 namespace IfcParse {
 	/// The IfcSpfStream class represents a ISO 10303-21 IFC-SPF file in memory.
 	/// The file is interpreted as a sequence of tokens which are lazily
-	/// interpreted only when requested. If the size of the file is
+	/// interpreted only when requested.
-	/// larger than BUF_SIZE, the file is split into seperate pages, of
-	/// which only one is simultaneously kept in memory, for files
-	/// that define their entities not in a sequential nature, this is
-	/// detrimental for the performance of the parser.
 	class IFC_PARSE_API IfcSpfStream {
 	private:
+#ifdef USE_MMAP
+		boost::iostreams::mapped_file_source mfs;
+#endif
 		FILE* stream;
-		char* buffer;
+		const char* buffer;
 		unsigned int ptr;
 		unsigned int len;
-		void ReadBuffer(bool inc=true);
-#ifdef BUF_SIZE
-		unsigned int offset;
-		bool paging;
-#endif
 	public:
 		bool valid;
 		bool eof;
 		unsigned int size;
+#ifdef USE_MMAP
+		IfcSpfStream(const std::string& fn, bool mmap=false);
+#else
 		IfcSpfStream(const std::string& fn);
+#endif
 		IfcSpfStream(std::istream& f, int len);
 		IfcSpfStream(void* data, int len);
 		~IfcSpfStream();

win/build-deps.cmd

@@ -169,7 +169,7 @@ if not exist "%DEPS_DIR%\boost\project-config.jam". (
     IF NOT %ERRORLEVEL%==0 GOTO :Error
 )
 
-set BOOST_LIBS=--with-system --with-regex --with-thread --with-program_options --with-date_time
+set BOOST_LIBS=--with-system --with-regex --with-thread --with-program_options --with-date_time --with-iostreams --with-filesystem
 :: NOTE Boost is fast to build with limited set of libraries so build it always.
 cd "%DEPS_DIR%\boost"
 call cecho.cmd 0 13 "Building %DEPENDENCY_NAME% %BOOST_LIBS% Please be patient, this will take a while."