mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-09-27 10:51:13 +00:00
ifcviewer: v16 zstd-compressed sidecars (~10x smaller over the wire)
The .ifcview data is hugely redundant (repeated double instance matrices, patterned indices) — measured 12x zstd whole-file. Server Content-Encoding can't be used (it breaks HTTP Range), so compress PER-CHUNK into the format. Format (v16): geometry becomes per-chunk zstd(vertices)+zstd(indices) frames — each independently Range-fetchable, so streaming is intact — and the critical + deferred metadata blocks are single zstd frames. SidecarChunk carries the compressed blob offsets/sizes; applyStreamedChunk (render/upload) is UNCHANGED — decompression slots into the fetch. Full readSidecar (test/tooling) reconstructs by decompress+scatter. zstd: desktop links libzstd (also compresses at bake); the web build (Emscripten has no zstd port) FetchContent's the pinned zstd source and compiles its decompress-only subset for wasm — no vendored blob, same version as desktop. New SidecarCompress wraps it (compress guarded off under Emscripten). Both stream paths — desktop StreamingThread worker + sync fallback (readChunkGeometryCompressed) and web beginWebChunkLoad — decompress; readSidecarMetadataOnly / the web bootstrap / loadDeferredMetadataWeb decompress the metadata blocks. streamingByteProgress reports COMPRESSED bytes. MEASURED: a 752 MB v15 federation → 75 MB v16 (10x; per-file 6.7-15.3x); PP-PLP 118→15 MB, loads 13/13 chunks on web, 0 errors. Three fixes found while testing big federations on a real server: - Web-streamed race: streaming_from_web was set in the deferred-header callback (a round-trip after the model+chunks exist), so driveStreamingLoads could take the sync fopen path meanwhile → "failed to read/decompress chunk 0". Now set immediately after applyCachedModel. - OOM abort on 18 models: the pool grew unbounded until an alloc failed, but on web that's an uncatchable bad_alloc abort. Cap total pool capacity (setMaxTotalCapacity, 3 GB) so it stops before the heap ceiling, and raise MAXIMUM_MEMORY 2→4 GB (wasm32 max) for headroom. - Web never evicted (grow-or-block only). At the hard budget, fall through to the LRU/priority evictor so a big federation stays navigable (highest-contribution chunks win) instead of freezing with holes. 113/113 desktop + 6/6 web smoke pass. No back-compat: regenerate sidecars (desktop bakes v16; scratch conv tool migrates v15→v16). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -143,10 +143,46 @@ set(IFCVIEWER_CORE_SOURCES
|
||||
InstanceCompose.cpp
|
||||
LodBuilder.cpp
|
||||
SidecarCache.cpp
|
||||
SidecarCompress.cpp
|
||||
StreamingLoader.cpp
|
||||
StreamingThread.cpp
|
||||
ViewportCore.cpp
|
||||
)
|
||||
# Web needs a zstd DECODER (Emscripten has no zstd port; the desktop links the
|
||||
# full libzstd below). Rather than vendor a generated blob, fetch the pinned
|
||||
# zstd source and compile its decompress-only subset — the exact set zstd's
|
||||
# own single-file decoder inlines — straight into the wasm.
|
||||
if(EMSCRIPTEN)
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(zstd_dec
|
||||
GIT_REPOSITORY https://github.com/facebook/zstd.git
|
||||
GIT_TAG v1.5.7
|
||||
GIT_SHALLOW TRUE
|
||||
# zstd's CMake lives in build/cmake, not the root — point SOURCE_SUBDIR
|
||||
# at a non-existent dir so MakeAvailable only POPULATES the source and
|
||||
# never tries to configure zstd's own (non-emscripten) build.
|
||||
SOURCE_SUBDIR does-not-exist)
|
||||
FetchContent_MakeAvailable(zstd_dec)
|
||||
set(ZSTD_DEC_DIR "${zstd_dec_SOURCE_DIR}/lib")
|
||||
set(ZSTD_DEC_SRC
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_decompress.c
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_decompress_block.c
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_ddict.c
|
||||
${ZSTD_DEC_DIR}/decompress/huf_decompress.c
|
||||
${ZSTD_DEC_DIR}/common/entropy_common.c
|
||||
${ZSTD_DEC_DIR}/common/error_private.c
|
||||
${ZSTD_DEC_DIR}/common/fse_decompress.c
|
||||
${ZSTD_DEC_DIR}/common/pool.c
|
||||
${ZSTD_DEC_DIR}/common/threading.c
|
||||
${ZSTD_DEC_DIR}/common/xxhash.c
|
||||
${ZSTD_DEC_DIR}/common/zstd_common.c
|
||||
${ZSTD_DEC_DIR}/common/debug.c)
|
||||
# wasm isn't x86-64 → no BMI2 asm path (belt-and-braces disable).
|
||||
set_source_files_properties(${ZSTD_DEC_SRC} PROPERTIES
|
||||
COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1")
|
||||
# NOT appended to IFCVIEWER_CORE_SOURCES — those get a source-dir PREPEND
|
||||
# below which would mangle these absolute paths; added via target_sources.
|
||||
endif()
|
||||
set(IFCVIEWER_CORE_HEADERS
|
||||
BufferPool.h
|
||||
CameraMath.h
|
||||
@@ -162,6 +198,7 @@ set(IFCVIEWER_CORE_HEADERS
|
||||
SectionPlane.h
|
||||
SelectionState.h
|
||||
SidecarCache.h
|
||||
SidecarCompress.h
|
||||
StreamingLoader.h
|
||||
StreamingThread.h
|
||||
VertexQuantization.h
|
||||
@@ -183,6 +220,19 @@ if(UNIX AND NOT APPLE)
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(IfcViewerCore PUBLIC Threads::Threads)
|
||||
endif()
|
||||
# Sidecar (de)compression. Web compiles the vendored single-file decoder (added
|
||||
# to the sources above) + uses the vendored zstd.h; desktop links the full
|
||||
# libzstd (compress + decompress) and takes zstd.h from the system.
|
||||
if(EMSCRIPTEN)
|
||||
target_sources(IfcViewerCore PRIVATE ${ZSTD_DEC_SRC})
|
||||
target_include_directories(IfcViewerCore PRIVATE ${ZSTD_DEC_DIR})
|
||||
else()
|
||||
find_library(ZSTD_LIBRARY NAMES zstd libzstd)
|
||||
if(NOT ZSTD_LIBRARY)
|
||||
message(FATAL_ERROR "libzstd not found (needed for .ifcview compression)")
|
||||
endif()
|
||||
target_link_libraries(IfcViewerCore PUBLIC ${ZSTD_LIBRARY})
|
||||
endif()
|
||||
install(TARGETS IfcViewerCore EXPORT ${IFCOPENSHELL_EXPORT_TARGETS})
|
||||
|
||||
# IfcViewer: the Qt + IfcGeom + OpenCASCADE shell — everything in this
|
||||
|
||||
Reference in New Issue
Block a user