From ae0afe2c0a640c90722dfcd119614512b58d13fc Mon Sep 17 00:00:00 2001 From: Thomas Krijnen Date: Sun, 23 Jul 2017 14:52:04 +0200 Subject: [PATCH] mmap #22 --- cmake/CMakeLists.txt | 15 ++- nix/build-all.py | 3 +- src/ifcconvert/IfcConvert.cpp | 19 +++- src/ifcparse/IfcFile.h | 4 + src/ifcparse/IfcParse.cpp | 196 +++++++++++++++------------------- src/ifcparse/IfcSpfStream.h | 34 +++--- win/build-deps.cmd | 2 +- 7 files changed, 137 insertions(+), 136 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index cebc4ccde4..b2369f76f9 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -29,6 +29,7 @@ OPTION(USE_IFC4 "Use IFC 4 instead of IFC 2x3 (full rebuild recommended when swi OPTION(BUILD_IFCPYTHON "Build IfcPython." ON) OPTION(BUILD_EXAMPLES "Build example applications." ON) OPTION(USE_VLD "Use Visual Leak Detector for debugging memory leaks, MSVC-only." OFF) +OPTION(USE_MMAP "Adds a command line options to parse IFC files from memory mapped files using Boost.Iostreams" OFF) OPTION(BUILD_IFCMAX "Build IfcMax, a 3ds Max plug-in, Windows-only." OFF) OPTION(BUILD_SHARED_LIBS "Build IfcParse and IfcGeom as shared libs (SO/DLL)." OFF) # TODO QtViewer is deprecated ATM as it uses the 0.4 API @@ -112,7 +113,19 @@ IF(WIN32) SET(Boost_USE_STATIC_RUNTIME ON) SET(Boost_USE_MULTITHREADED ON) ENDIF() -FIND_PACKAGE(Boost REQUIRED COMPONENTS system program_options regex thread date_time) + +set(BOOST_COMPONENTS system program_options regex thread date_time) +if(USE_MMAP) + if(MSVC) + # filesystem is necessary for the utf-16 wpath + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} iostreams filesystem) + else() + set(BOOST_COMPONENTS ${BOOST_COMPONENTS} iostreams) + endif() + add_definitions(-DUSE_MMAP) +endif() + +FIND_PACKAGE(Boost REQUIRED COMPONENTS ${BOOST_COMPONENTS}) MESSAGE(STATUS "Boost include files found in ${Boost_INCLUDE_DIRS}") MESSAGE(STATUS "Boost libraries found in ${Boost_LIBRARY_DIRS}") diff --git a/nix/build-all.py b/nix/build-all.py index 7d28de71e3..b6c8b978a0 100644 --- a/nix/build-all.py +++ b/nix/build-all.py @@ -454,7 +454,7 @@ os.environ["CXXFLAGS"]=OLD_CXX_FLAGS os.environ["CFLAGS"]=OLD_C_FLAGS str_concat = lambda prefix: lambda postfix: "=".join((prefix, postfix.strip())) -build_dependency("boost-%s" % (BOOST_VERSION,), mode="bjam", build_tool_args=["--stagedir=%s/install/boost-%s" % (DEPS_DIR, BOOST_VERSION), "--with-system", "--with-program_options", "--with-regex", "--with-thread", "--with-date_time", "link=static"]+BOOST_ADDRESS_MODEL+list(map(str_concat("cxxflags"), CXXFLAGS.strip().split(' '))) + list(map(str_concat("linkflags"), LDFLAGS.strip().split(' '))) + ["stage"], download_url="http://downloads.sourceforge.net/project/boost/boost/%s/" % (BOOST_VERSION,), download_name="boost_%s.tar.bz2" % (BOOST_VERSION_UNDERSCORE,)) +build_dependency("boost-%s" % (BOOST_VERSION,), mode="bjam", build_tool_args=["--stagedir=%s/install/boost-%s" % (DEPS_DIR, BOOST_VERSION), "--with-system", "--with-program_options", "--with-regex", "--with-thread", "--with-date_time", "--with-iostreams", "link=static"]+BOOST_ADDRESS_MODEL+list(map(str_concat("cxxflags"), CXXFLAGS.strip().split(' '))) + list(map(str_concat("linkflags"), LDFLAGS.strip().split(' '))) + ["stage"], download_url="http://downloads.sourceforge.net/project/boost/boost/%s/" % (BOOST_VERSION,), download_name="boost_%s.tar.bz2" % (BOOST_VERSION_UNDERSCORE,)) build_dependency(name="icu-%s" % (ICU_VERSION,), mode="icu", build_tool_args=["--enable-static", "--disable-shared"], download_url="http://download.icu-project.org/files/icu4c/%s/" % (ICU_VERSION,), download_name="icu4c-%s-src.tgz" % (ICU_VERSION_UNDERSCORE,)) @@ -481,6 +481,7 @@ run_cmake("", cmake_args=[ "-DICU_LIBRARY_DIR=" "%s/install/icu-%s/lib" % (DEPS_DIR, ICU_VERSION), "-DPCRE_LIBRARY_DIR=" "%s/install/pcre-%s/lib" % (DEPS_DIR, PCRE_VERSION), "-DBUILD_IFCPYTHON=" "OFF", + "-DUSE_MMAP=" "ON", "-DCMAKE_INSTALL_PREFIX=" "%s/install/ifcopenshell" % (DEPS_DIR,)], cmake_dir=CMAKE_DIR, cwd=executables_dir) logger.info("\rBuilding executables... ") diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 3f002c552d..3b97df7abc 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -148,7 +148,7 @@ size_t read_filters_from_file(const std::string&, inclusion_filter&, inclusion_t void parse_filter(geom_filter &, const std::vector&); std::vector setup_filters(const std::vector&, const std::string&); -bool init_input_file(const std::string& filename, IfcParse::IfcFile& ifc_file, bool no_progress); +bool init_input_file(const std::string& filename, IfcParse::IfcFile& ifc_file, bool no_progress, bool mmap); int main(int argc, char** argv) { @@ -162,8 +162,12 @@ int main(int argc, char** argv) po::options_description fileio_options; fileio_options.add_options() +#ifdef USE_MMAP + ("mmap", "use memory-mapped file for input") +#endif ("input-file", po::value(), "input IFC file") ("output-file", po::value(), "output geometry file"); + double deflection_tolerance; inclusion_filter include_filter; @@ -337,6 +341,7 @@ int main(int argc, char** argv) print_usage(); return EXIT_FAILURE; } + const bool mmap = vmap.count("mmap") != 0; const bool verbose = vmap.count("verbose") != 0; const bool no_progress = vmap.count("no-progress") != 0; const bool weld_vertices = vmap.count("weld-vertices") != 0; @@ -427,7 +432,7 @@ int main(int argc, char** argv) if (output_extension == ".xml") { int exit_code = EXIT_FAILURE; try { - if (init_input_file(input_filename, ifc_file, no_progress)) { + if (init_input_file(input_filename, ifc_file, no_progress, mmap)) { XmlSerializer s(output_temp_filename); s.setFile(&ifc_file); Logger::Status("Writing XML output..."); @@ -564,7 +569,7 @@ int main(int argc, char** argv) time_t start,end; time(&start); - if (!init_input_file(input_filename, ifc_file, no_progress)) { + if (!init_input_file(input_filename, ifc_file, no_progress, mmap)) { return EXIT_FAILURE; } @@ -691,12 +696,16 @@ void write_log() { } } -bool init_input_file(const std::string &filename, IfcParse::IfcFile &ifc_file, bool no_progress) +bool init_input_file(const std::string &filename, IfcParse::IfcFile &ifc_file, bool no_progress, bool mmap) { // Prevent IfcFile::Init() prints by setting output to null temporarily if (no_progress) { Logger::SetOutput(NULL, &log_stream); } - if (!ifc_file.Init(filename)) { +#ifdef USE_MMAP + if (!ifc_file.Init(filename, mmap)) { +#else + if (!ifc_file.Init(filename)) { +#endif Logger::Error("Unable to parse input file '" + filename + "'"); return false; } diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index 2ae5dca8c8..fe927eb9f6 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -107,7 +107,11 @@ public: /// in the first function argument. IfcEntityList::ptr traverse(IfcUtil::IfcBaseClass* instance, int max_level=-1); +#ifdef USE_MMAP + bool Init(const std::string& fn, bool mmap=false); +#else bool Init(const std::string& fn); +#endif bool Init(std::istream& fn, int len); bool Init(void* data, int len); bool Init(IfcParse::IfcSpfStream* f); diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index 947c3d92d1..e5f3a6ebce 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -46,6 +46,10 @@ #include "../ifcparse/Ifc2x3-latebound.h" #endif +#ifdef USE_MMAP +#include +#endif + #define PERMISSIVE_FLOAT using namespace IfcParse; @@ -105,69 +109,99 @@ void init_locale() { #endif // -// Opens the file, gets the filesize and reads a chunk in memory +// Opens the file and gets the filesize // +#ifdef USE_MMAP +IfcSpfStream::IfcSpfStream(const std::string& fn, bool mmap) +#else IfcSpfStream::IfcSpfStream(const std::string& fn) - : stream(0) - , buffer(0) +#endif + : stream(0) + , buffer(0) + , valid(false) + , eof(false) { - eof = false; #ifdef _MSC_VER int fn_buffer_size = MultiByteToWideChar(CP_UTF8, 0, fn.c_str(), -1, 0, 0); wchar_t* fn_wide = new wchar_t[fn_buffer_size]; MultiByteToWideChar(CP_UTF8, 0, fn.c_str(), -1, fn_wide, fn_buffer_size); - stream = _wfopen(fn_wide, L"rb"); + +#ifdef USE_MMAP + if (mmap) { + mfs = boost::iostreams::mapped_file_source(boost::filesystem::wpath(fn_wide)); + } else { +#endif + stream = _wfopen(fn_wide, L"rb"); +#ifdef USE_MMAP + } +#endif + delete[] fn_wide; #else - stream = fopen(fn.c_str(), "rb"); + +#ifdef USE_MMAP + if (mmap) { + mfs = boost::iostreams::mapped_file_source(fn); + } else { #endif - if (stream == NULL) { - valid = false; - return; + stream = fopen(fn.c_str(), "rb"); +#ifdef USE_MMAP + } +#endif + +#endif + +#ifdef USE_MMAP + if (mmap) { + if (!mfs.is_open()) { + return; + } + + valid = true; + buffer = mfs.data(); + ptr = 0; + len = mfs.size(); + } else { +#endif + if (stream == NULL) { + return; + } + + valid = true; + fseek(stream, 0, SEEK_END); + size = (unsigned int)ftell(stream); + rewind(stream); + char* buffer_rw = new char[size]; + len = (unsigned int)fread(buffer_rw, 1, size, stream); + buffer = buffer_rw; + eof = len == 0; + ptr = 0; + fclose(stream); +#ifdef USE_MMAP } - valid = true; - fseek(stream, 0, SEEK_END); - size = (unsigned int) ftell(stream); - rewind(stream); -#ifdef BUF_SIZE - offset = 0; - paging = size > BUF_SIZE; - buffer = new char[size < BUF_SIZE ? size : BUF_SIZE]; -#else - buffer = new char[size]; #endif - ptr = 0; - len = 0; - ReadBuffer(false); } IfcSpfStream::IfcSpfStream(std::istream& f, int l) - : stream(0) - , buffer(0) + : stream(0) + , buffer(0) { eof = false; size = l; -#ifdef BUF_SIZE - paging = false; - offset = 0; -#endif - buffer = new char[size]; - f.read(buffer,size); + char* buffer_rw = new char[size]; + f.read(buffer_rw,size); + buffer = buffer_rw; valid = f.gcount() == size; ptr = 0; len = l; } IfcSpfStream::IfcSpfStream(void* data, int l) - : stream(0) - , buffer(0) + : stream(0) + , buffer(0) { eof = false; size = l; -#ifdef BUF_SIZE - paging = false; - offset = 0; -#endif buffer = (char*) data; valid = true; ptr = 0; @@ -180,60 +214,22 @@ IfcSpfStream::~IfcSpfStream() } void IfcSpfStream::Close() { -#ifdef BUF_SIZE - if ( paging ) fclose(stream); +#ifdef USE_MMAP + if (mfs.is_open()) { + mfs.close(); + return; + } #endif delete[] buffer; } -// -// Reads a chunk of BUF_SIZE in memory and increments cursor if requested -// -void IfcSpfStream::ReadBuffer(bool inc) { -#ifdef BUF_SIZE - if ( inc ) { - offset += len; - fseek(stream, offset, SEEK_SET); - } -#else - (void)inc; -#endif - eof = feof(stream) != 0; - if ( eof ) return; -#ifdef BUF_SIZE - len = (unsigned int) fread(buffer, 1, size < BUF_SIZE ? size : BUF_SIZE, stream); -#else - len = (unsigned int) fread(buffer, 1, size, stream); -#endif - eof = len == 0; - ptr = 0; -#ifdef BUF_SIZE - if (!paging) fclose(stream); -#else - fclose(stream); -#endif -} - // // Seeks an arbitrary position in the file // void IfcSpfStream::Seek(unsigned int o) { -#ifdef BUF_SIZE - if ( !paging ) { -#endif - ptr = o; - if (ptr >= len) throw IfcException("Reading outside of file limits"); - eof = false; -#ifdef BUF_SIZE - } else if ( o >= offset && (o < (offset+len)) ) { - ptr = o - offset; - } else { - offset = o; - clearerr(stream); - fseek(stream, o, SEEK_SET); - ReadBuffer(false); - } -#endif + ptr = o; + if (ptr >= len) throw IfcException("Reading outside of file limits"); + eof = false; } // @@ -247,30 +243,14 @@ char IfcSpfStream::Peek() { // Returns the character at specified offset // char IfcSpfStream::Read(unsigned int o) { -#ifdef BUF_SIZE - if ( ! paging ) { -#endif - return buffer[o]; -#ifdef BUF_SIZE - } else if ( o >= offset && (o < (offset+len)) ) { - return buffer[o-offset]; - } else { - clearerr(stream); - fseek(stream, o, SEEK_SET); - return ungetc(getc(stream), stream); - } -#endif + return buffer[o]; } // // Returns the cursor position // unsigned int IfcSpfStream::Tell() { -#ifdef BUF_SIZE - return offset + ptr; -#else return ptr; -#endif } // @@ -278,16 +258,10 @@ unsigned int IfcSpfStream::Tell() { // void IfcSpfStream::Inc() { if ( ++ptr == len ) { -#ifdef BUF_SIZE - if ( paging ) ReadBuffer(); - else { -#endif - eof = true; - return; -#ifdef BUF_SIZE - } -#endif + eof = true; + return; } + /// @todo: Shouldn't this be a loop of some kind const char current = IfcSpfStream::Peek(); if ( current == '\n' || current == '\r' ) IfcSpfStream::Inc(); } @@ -1245,9 +1219,15 @@ void IfcEntityInstanceData::setArgument(unsigned int i, Argument* a, IfcUtil::Ar // Parses the IFC file in fn // Creates the maps // +#ifdef USE_MMAP +bool IfcFile::Init(const std::string& fn, bool mmap) { + return IfcFile::Init(new IfcSpfStream(fn, mmap)); +} +#else bool IfcFile::Init(const std::string& fn) { return IfcFile::Init(new IfcSpfStream(fn)); } +#endif bool IfcFile::Init(std::istream& f, int len) { return IfcFile::Init(new IfcSpfStream(f,len)); diff --git a/src/ifcparse/IfcSpfStream.h b/src/ifcparse/IfcSpfStream.h index 4e5f9dbf2f..820b69d275 100644 --- a/src/ifcparse/IfcSpfStream.h +++ b/src/ifcparse/IfcSpfStream.h @@ -19,7 +19,7 @@ /********************************************************************************* * * - * Reads a file in chunks of BUF_SIZE and provides functions to access its * + * Reads a file and provides functions to access its * * contents randomly and character by character * * * ********************************************************************************/ @@ -30,40 +30,34 @@ #include #include -#include "ifc_parse_api.h" +#ifdef USE_MMAP +#include +#endif -// As of IfcOpenShell version 0.3.0 the paging functionality, which -// loads a file on disk into multiple chunks, has been disabled. -// It proved to be an inefficient way of working with large files, -// as often these did not facilitate to be parsed in a sequential -// manner efficiently. To enable the paging functionality uncomment -// the following statement. -// #define BUF_SIZE (8 * 1024 * 1024) +#include "ifc_parse_api.h" namespace IfcParse { /// The IfcSpfStream class represents a ISO 10303-21 IFC-SPF file in memory. /// The file is interpreted as a sequence of tokens which are lazily - /// interpreted only when requested. If the size of the file is - /// larger than BUF_SIZE, the file is split into seperate pages, of - /// which only one is simultaneously kept in memory, for files - /// that define their entities not in a sequential nature, this is - /// detrimental for the performance of the parser. + /// interpreted only when requested. class IFC_PARSE_API IfcSpfStream { private: +#ifdef USE_MMAP + boost::iostreams::mapped_file_source mfs; +#endif FILE* stream; - char* buffer; + const char* buffer; unsigned int ptr; unsigned int len; - void ReadBuffer(bool inc=true); -#ifdef BUF_SIZE - unsigned int offset; - bool paging; -#endif public: bool valid; bool eof; unsigned int size; +#ifdef USE_MMAP + IfcSpfStream(const std::string& fn, bool mmap=false); +#else IfcSpfStream(const std::string& fn); +#endif IfcSpfStream(std::istream& f, int len); IfcSpfStream(void* data, int len); ~IfcSpfStream(); diff --git a/win/build-deps.cmd b/win/build-deps.cmd index 8f1de0fc1b..b32f031095 100644 --- a/win/build-deps.cmd +++ b/win/build-deps.cmd @@ -169,7 +169,7 @@ if not exist "%DEPS_DIR%\boost\project-config.jam". ( IF NOT %ERRORLEVEL%==0 GOTO :Error ) -set BOOST_LIBS=--with-system --with-regex --with-thread --with-program_options --with-date_time +set BOOST_LIBS=--with-system --with-regex --with-thread --with-program_options --with-date_time --with-iostreams --with-filesystem :: NOTE Boost is fast to build with limited set of libraries so build it always. cd "%DEPS_DIR%\boost" call cecho.cmd 0 13 "Building %DEPENDENCY_NAME% %BOOST_LIBS% Please be patient, this will take a while."