From 0b8c787ac02ccaef9bd21d12ff8eb1c3ed215260 Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Thu, 2 Jul 2026 10:24:59 +1000 Subject: [PATCH] ifcviewer: v16 zstd-compressed sidecars (~10x smaller over the wire) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The .ifcview data is hugely redundant (repeated double instance matrices, patterned indices) — measured 12x zstd whole-file. Server Content-Encoding can't be used (it breaks HTTP Range), so compress PER-CHUNK into the format. Format (v16): geometry becomes per-chunk zstd(vertices)+zstd(indices) frames — each independently Range-fetchable, so streaming is intact — and the critical + deferred metadata blocks are single zstd frames. SidecarChunk carries the compressed blob offsets/sizes; applyStreamedChunk (render/upload) is UNCHANGED — decompression slots into the fetch. Full readSidecar (test/tooling) reconstructs by decompress+scatter. zstd: desktop links libzstd (also compresses at bake); the web build (Emscripten has no zstd port) FetchContent's the pinned zstd source and compiles its decompress-only subset for wasm — no vendored blob, same version as desktop. New SidecarCompress wraps it (compress guarded off under Emscripten). Both stream paths — desktop StreamingThread worker + sync fallback (readChunkGeometryCompressed) and web beginWebChunkLoad — decompress; readSidecarMetadataOnly / the web bootstrap / loadDeferredMetadataWeb decompress the metadata blocks. streamingByteProgress reports COMPRESSED bytes. MEASURED: a 752 MB v15 federation → 75 MB v16 (10x; per-file 6.7-15.3x); PP-PLP 118→15 MB, loads 13/13 chunks on web, 0 errors. Three fixes found while testing big federations on a real server: - Web-streamed race: streaming_from_web was set in the deferred-header callback (a round-trip after the model+chunks exist), so driveStreamingLoads could take the sync fopen path meanwhile → "failed to read/decompress chunk 0". Now set immediately after applyCachedModel. - OOM abort on 18 models: the pool grew unbounded until an alloc failed, but on web that's an uncatchable bad_alloc abort. Cap total pool capacity (setMaxTotalCapacity, 3 GB) so it stops before the heap ceiling, and raise MAXIMUM_MEMORY 2→4 GB (wasm32 max) for headroom. - Web never evicted (grow-or-block only). At the hard budget, fall through to the LRU/priority evictor so a big federation stays navigable (highest-contribution chunks win) instead of freezing with holes. 113/113 desktop + 6/6 web smoke pass. No back-compat: regenerate sidecars (desktop bakes v16; scratch conv tool migrates v15→v16). Co-Authored-By: Claude Opus 4.8 --- src/ifcviewer-web/CMakeLists.txt | 4 +- src/ifcviewer-web/sample.ifcview | Bin 2583 -> 541 bytes src/ifcviewer/BufferPool.h | 14 +- src/ifcviewer/CMakeLists.txt | 50 +++ src/ifcviewer/ModelGpuData.h | 20 +- src/ifcviewer/SceneLoader.cpp | 7 +- src/ifcviewer/SidecarCache.cpp | 293 ++++++++++---- src/ifcviewer/SidecarCache.h | 26 +- src/ifcviewer/SidecarCompress.cpp | 46 +++ src/ifcviewer/SidecarCompress.h | 51 +++ src/ifcviewer/StreamingLoader.cpp | 108 +++-- src/ifcviewer/StreamingLoader.h | 55 +-- src/ifcviewer/StreamingThread.cpp | 20 +- src/ifcviewer/StreamingThread.h | 18 +- src/ifcviewer/ViewportCore.cpp | 373 ++++++++---------- src/ifcviewer/tests/CMakeLists.txt | 16 +- src/ifcviewer/tests/test_sidecar_cache.cpp | 7 +- src/ifcviewer/tests/test_sidecar_compress.cpp | 68 ++++ src/ifcviewer/tests/test_streaming_loader.cpp | 128 +++--- 19 files changed, 862 insertions(+), 442 deletions(-) create mode 100644 src/ifcviewer/SidecarCompress.cpp create mode 100644 src/ifcviewer/SidecarCompress.h create mode 100644 src/ifcviewer/tests/test_sidecar_compress.cpp diff --git a/src/ifcviewer-web/CMakeLists.txt b/src/ifcviewer-web/CMakeLists.txt index f4443f1ad4..4a342889d8 100644 --- a/src/ifcviewer-web/CMakeLists.txt +++ b/src/ifcviewer-web/CMakeLists.txt @@ -38,7 +38,7 @@ # Better to keep the web path's machinery local to this directory. cmake_minimum_required(VERSION 3.21) -project(IfcViewerWeb LANGUAGES CXX) +project(IfcViewerWeb LANGUAGES C CXX) # C for the vendored zstd decoder if(NOT EMSCRIPTEN) message(FATAL_ERROR @@ -114,7 +114,7 @@ target_link_options(IfcViewerWeb PRIVATE # cap; --shared64 / MEMORY64 would lift this later if we need it). "-sALLOW_MEMORY_GROWTH=1" "-sINITIAL_MEMORY=268435456" # 256 MB - "-sMAXIMUM_MEMORY=2147483648" # 2 GB + "-sMAXIMUM_MEMORY=4294967296" # 4 GB (wasm32 max) — big federations need the headroom # FETCH lets emscripten_fetch issue HTTP Range requests. The local # file path (#88) reads byte ranges via Blob.slice and does NOT need # this; it's retained for the remote-URL Range backend (follow-up). diff --git a/src/ifcviewer-web/sample.ifcview b/src/ifcviewer-web/sample.ifcview index 2ea0b91f48e864f634bdc018fd382dc55f3a779a..e91c0a92c83629ac05fb643f679e52eba12d6feb 100644 GIT binary patch literal 541 zcmWF!bMq8nU|?WjW@5Yoq(MMqtN!1F1g5pj3`v|kybmOgF`Q=MNf2i^puqYyJ~aO1 zSJr>Q;i;jqUwIXq_G_)re)Y?1pVkki_4+LZp{)8VCbB#@z{Fa9ZGogw52IiJ%LE6J z1CDIp|NrMb`#5aI^@9igt&#foYNf-*%C{0HKlZ+zx@mrmm+|pdUM1~>J+obS!PakJ zh-G33V&~=*STK)SOq?;DEB~IAvw!?OgZOoK{;HO{*KLv3zjy9R=la#lKRpTdLdm9rm6{i~dY;nOpI={jHoJ1>PyGUj*BC5VAPlf27R;v13{34jf)8}r z8(6p#dtbfeXIcE~t~INDM!m|V_)dEbxu(do(tlhTRAdg=u|4^p@c5ek|BvE~!U`L* zvb0K!J1zF@Z9X7gdT@crp17|3mW}Kpj~439VM_W^=3Xvt(f0aAuJApkV^YjZq4qz9 z(jfa49zSJa_;f;ECgC=tfh3D#hdRU6ru}mZ+?WE=ov-wNozP%AZONOAZ=crOxcElw z)XHT_l0xrJY>JJ!+4pGY>CI8sFTTE}XMV9PUH$f-xj)ua7~X#xIq~tTjklS^1FxFA zoG3dpN!DoZl)wK|e|-JF{QrJ2g}Oigzx*%w@c+nv>Hqd3^Z(eh{n;{*9L*>Qab`#a?NN|8kkuq_;2XG@ z7w`>c>Ff9g!3p{kHtn2edD0Wkd z(~%SP^xUj)LO2;Y@suzpoEFYRPCP4|6V3}4A}8{eFA0~0E0JG~^J|LNg&P9=W}GYh zll#FN{F?L!Z}97|{@@cHP5OgRcr?esXmCc1TOqi4ANNMil~@*!2%bi%yQ2fA?yGQ6 z9rps*c%;ig9rXv*afhmPwpYtpbnd72cbIV$I=q=2f0HYH9wndmmiab&OUL;;?LUEo z$N4+c4LOQCn7aR&*}5s3-@Gfp96S^52@L)hfZxS!f&Y`e%N_9*;jVxt7tlQv&{@y# z;@^b2fbNxm?u~%1D`X4n?>n{LYQNTh@TA%Nu(|kpE$cOlcBir3C1m-R*1~SS@;T3* v*LFW;kC*!OVx!eA3JD0=_GbHI_vQMF#rgV9=YFTLw3@$Lt8cY?5)gg?oM1)f diff --git a/src/ifcviewer/BufferPool.h b/src/ifcviewer/BufferPool.h index 5803665581..b072d5b7f9 100644 --- a/src/ifcviewer/BufferPool.h +++ b/src/ifcviewer/BufferPool.h @@ -109,7 +109,18 @@ public: // false the first time addSubBuffer is refused even at the floor // size — eviction callers need this to know whether a future alloc // could rescue them, or whether eviction is the only path. - bool can_grow() const { return !growth_disabled_ && per_sub_buffer_capacity_ > 0; } + bool can_grow() const { + return !growth_disabled_ && per_sub_buffer_capacity_ > 0 + && (max_total_capacity_bytes_ == 0 + || total_capacity_bytes() < max_total_capacity_bytes_); + } + + // Hard ceiling on total pool capacity (0 = unlimited). Once total capacity + // reaches this, can_grow() returns false so the streaming driver EVICTS + // instead of growing. Critical on web: a growth past the wasm heap ceiling + // is a bad_alloc that -fno-exceptions turns into an uncatchable abort, so + // the async grow-OOM detection can't save us — we must stop first. + void setMaxTotalCapacity(uint64_t max_bytes) { max_total_capacity_bytes_ = max_bytes; } // Proactively add a sub-buffer (no allocation). On web this kicks off the // async provisional-validation cycle so validated free space appears a @@ -175,6 +186,7 @@ private: WGPUDevice device_ = nullptr; WGPUBufferUsage usage_ = 0; uint64_t per_sub_buffer_capacity_ = 0; + uint64_t max_total_capacity_bytes_ = 0; // 0 = unlimited (see can_grow) // The largest size addSubBuffer last *succeeded* at, in bytes. // Starts at per_sub_buffer_capacity_ (the probe's discovered max) // and decays as the driver refuses larger allocations. Future grow diff --git a/src/ifcviewer/CMakeLists.txt b/src/ifcviewer/CMakeLists.txt index 510f8560e2..127cd67cf8 100644 --- a/src/ifcviewer/CMakeLists.txt +++ b/src/ifcviewer/CMakeLists.txt @@ -143,10 +143,46 @@ set(IFCVIEWER_CORE_SOURCES InstanceCompose.cpp LodBuilder.cpp SidecarCache.cpp + SidecarCompress.cpp StreamingLoader.cpp StreamingThread.cpp ViewportCore.cpp ) +# Web needs a zstd DECODER (Emscripten has no zstd port; the desktop links the +# full libzstd below). Rather than vendor a generated blob, fetch the pinned +# zstd source and compile its decompress-only subset — the exact set zstd's +# own single-file decoder inlines — straight into the wasm. +if(EMSCRIPTEN) + include(FetchContent) + FetchContent_Declare(zstd_dec + GIT_REPOSITORY https://github.com/facebook/zstd.git + GIT_TAG v1.5.7 + GIT_SHALLOW TRUE + # zstd's CMake lives in build/cmake, not the root — point SOURCE_SUBDIR + # at a non-existent dir so MakeAvailable only POPULATES the source and + # never tries to configure zstd's own (non-emscripten) build. + SOURCE_SUBDIR does-not-exist) + FetchContent_MakeAvailable(zstd_dec) + set(ZSTD_DEC_DIR "${zstd_dec_SOURCE_DIR}/lib") + set(ZSTD_DEC_SRC + ${ZSTD_DEC_DIR}/decompress/zstd_decompress.c + ${ZSTD_DEC_DIR}/decompress/zstd_decompress_block.c + ${ZSTD_DEC_DIR}/decompress/zstd_ddict.c + ${ZSTD_DEC_DIR}/decompress/huf_decompress.c + ${ZSTD_DEC_DIR}/common/entropy_common.c + ${ZSTD_DEC_DIR}/common/error_private.c + ${ZSTD_DEC_DIR}/common/fse_decompress.c + ${ZSTD_DEC_DIR}/common/pool.c + ${ZSTD_DEC_DIR}/common/threading.c + ${ZSTD_DEC_DIR}/common/xxhash.c + ${ZSTD_DEC_DIR}/common/zstd_common.c + ${ZSTD_DEC_DIR}/common/debug.c) + # wasm isn't x86-64 → no BMI2 asm path (belt-and-braces disable). + set_source_files_properties(${ZSTD_DEC_SRC} PROPERTIES + COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1") + # NOT appended to IFCVIEWER_CORE_SOURCES — those get a source-dir PREPEND + # below which would mangle these absolute paths; added via target_sources. +endif() set(IFCVIEWER_CORE_HEADERS BufferPool.h CameraMath.h @@ -162,6 +198,7 @@ set(IFCVIEWER_CORE_HEADERS SectionPlane.h SelectionState.h SidecarCache.h + SidecarCompress.h StreamingLoader.h StreamingThread.h VertexQuantization.h @@ -183,6 +220,19 @@ if(UNIX AND NOT APPLE) find_package(Threads REQUIRED) target_link_libraries(IfcViewerCore PUBLIC Threads::Threads) endif() +# Sidecar (de)compression. Web compiles the vendored single-file decoder (added +# to the sources above) + uses the vendored zstd.h; desktop links the full +# libzstd (compress + decompress) and takes zstd.h from the system. +if(EMSCRIPTEN) + target_sources(IfcViewerCore PRIVATE ${ZSTD_DEC_SRC}) + target_include_directories(IfcViewerCore PRIVATE ${ZSTD_DEC_DIR}) +else() + find_library(ZSTD_LIBRARY NAMES zstd libzstd) + if(NOT ZSTD_LIBRARY) + message(FATAL_ERROR "libzstd not found (needed for .ifcview compression)") + endif() + target_link_libraries(IfcViewerCore PUBLIC ${ZSTD_LIBRARY}) +endif() install(TARGETS IfcViewerCore EXPORT ${IFCOPENSHELL_EXPORT_TARGETS}) # IfcViewer: the Qt + IfcGeom + OpenCASCADE shell — everything in this diff --git a/src/ifcviewer/ModelGpuData.h b/src/ifcviewer/ModelGpuData.h index 17ecb8a894..8971726151 100644 --- a/src/ifcviewer/ModelGpuData.h +++ b/src/ifcviewer/ModelGpuData.h @@ -177,6 +177,14 @@ struct ModelGpuData { // is recovered by walking mesh_ids and the model's MeshInfo[]. uint64_t vertex_byte_size = 0; uint64_t index_count = 0; + // v16: where this chunk's two zstd frames live in the file's geometry + // section (offsets relative to model.geometry_section_offset) and their + // compressed sizes. The raw sizes are vertex_byte_size / index_count*4. + // A per-chunk load fetches [off, +comp) and decompresses. + uint64_t v_comp_off = 0; + uint64_t v_comp_size = 0; + uint64_t i_comp_off = 0; + uint64_t i_comp_size = 0; // Of `index_count`, how many are LOD1 indices. LOD0 indices occupy // chunk-local u32 offsets [0, index_count - lod1_index_count); LOD1 // indices occupy [index_count - lod1_index_count, index_count). 0 @@ -286,8 +294,9 @@ struct ModelGpuData { // streaming path: chunks may be non-resident and need byte-range reads // from this file. Empty path = legacy non-streaming load. std::string streaming_file_path; - uint64_t streaming_vertex_section_offset = 0; - uint64_t streaming_index_section_offset = 0; + // v16: file offset of the compressed geometry section. A chunk's blobs are + // at geometry_section_offset + chunk.{v_comp_off,i_comp_off}. + uint64_t geometry_section_offset = 0; // Web only: chunk byte ranges come from the JS-side source — a picked File // (Blob.slice) or a remote URL (HTTP Range) — read asynchronously, not via // a synchronous fopen on streaming_file_path. Set by loadSidecarMetadataWeb @@ -308,8 +317,11 @@ struct ModelGpuData { // it; deferred_meta_loaded latches so it fetches at most once. std::vector elements; std::string string_table; - uint64_t deferred_meta_offset = 0; - uint64_t deferred_meta_bytes = 0; + // v16: the deferred block is a single zstd frame at deferred_comp_offset of + // deferred_comp_size bytes, expanding to deferred_raw_size. + uint64_t deferred_comp_offset = 0; + uint64_t deferred_comp_size = 0; + uint64_t deferred_raw_size = 0; bool deferred_meta_loaded = false; // applyCachedModel rebases instance object_ids by this base to keep them // globally unique across models; deferred elements carry the sidecar's diff --git a/src/ifcviewer/SceneLoader.cpp b/src/ifcviewer/SceneLoader.cpp index 9a4b80195c..fa5beb15e3 100644 --- a/src/ifcviewer/SceneLoader.cpp +++ b/src/ifcviewer/SceneLoader.cpp @@ -189,6 +189,7 @@ void SceneLoader::startNextLoad() { (long long)rt.elapsed(), ifc_path.c_str()); auto result = std::make_shared>(std::move(cached)); QMetaObject::invokeMethod(this, [this, mid, result, is_sidecar_source]() { + auto it = models_.find(mid); if (*result && !(*result)->meta.instances.empty()) { applySidecarData(mid, std::move(**result)); if (!is_sidecar_source) { @@ -197,7 +198,6 @@ void SceneLoader::startNextLoad() { return; } - auto it = models_.find(mid); if (it == models_.end()) return; if (is_sidecar_source) { @@ -236,10 +236,9 @@ void SceneLoader::applySidecarData(uint32_t mid, StreamingSidecar metadata) { SidecarData& d = metadata.meta; std::fprintf(stderr, - "[info] Sidecar hit: %s (%zu metadata bytes, %zu indices, %zu meshes, %zu instances, %zu elements)\n", + "[info] Sidecar hit: %s (%zu chunks, %zu meshes, %zu instances, %zu elements)\n", model.file_path.toStdString().c_str(), - size_t(metadata.vertex_total_bytes), - size_t(metadata.index_total_count), + d.chunks.size(), d.meshes.size(), d.instances.size(), d.elements.size()); diff --git a/src/ifcviewer/SidecarCache.cpp b/src/ifcviewer/SidecarCache.cpp index 2a3f2c2ed5..666097bdc3 100644 --- a/src/ifcviewer/SidecarCache.cpp +++ b/src/ifcviewer/SidecarCache.cpp @@ -43,10 +43,70 @@ // char[string_table_bytes] #include "SidecarCache.h" +#include "SidecarCompress.h" #include #include +// The baker (writeSidecar) compresses — desktop only; the web build never bakes +// and links a decompress-only zstd. Everything from here to writeSidecar's end +// is guarded off under Emscripten. +#if !defined(__EMSCRIPTEN__) + +// zstd level for baking. 19 is near-max ratio; decode speed is level- +// independent and the bake is offline, so favour ratio. +static constexpr int kSidecarZstdLevel = 19; + +// --- In-memory serialisation (a block is built in RAM, then compressed) ------ +template +static void appendVec(std::vector& b, const std::vector& v) { + std::uint32_t n = static_cast(v.size()); + const auto* np = reinterpret_cast(&n); + b.insert(b.end(), np, np + 4); + if (n > 0) { + const auto* p = reinterpret_cast(v.data()); + b.insert(b.end(), p, p + std::size_t(sizeof(T)) * n); + } +} +static void appendBytes(std::vector& b, const void* p, std::size_t n) { + const auto* c = static_cast(p); + b.insert(b.end(), c, c + n); +} + +// Pull one chunk's geometry out of the whole-model vertex/index arrays into the +// chunk-LOCAL layout applyStreamedChunk expects: vertices of its meshes in chunk +// order, then indices as LOD0 (per mesh) followed by LOD1 (per mesh). +static void extractChunkGeometry(const SidecarData& d, const SidecarChunk& c, + std::vector& vbytes, + std::vector& ibytes) { + vbytes.clear(); + ibytes.clear(); + const std::uint32_t end = c.first_mesh + c.mesh_count; + for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) { + const MeshInfo& m = d.meshes[mi]; + const std::size_t voff = m.vbo_byte_offset; + const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES; + if (voff + vn <= d.vertices.size()) + vbytes.insert(vbytes.end(), d.vertices.begin() + voff, + d.vertices.begin() + voff + vn); + } + auto appendIdx = [&](std::size_t first_u32, std::size_t count) { + if (first_u32 + count > d.indices.size()) return; + const auto* p = reinterpret_cast(d.indices.data() + first_u32); + ibytes.insert(ibytes.end(), p, p + count * sizeof(std::uint32_t)); + }; + for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) { + const MeshInfo& m = d.meshes[mi]; + if (m.index_count) appendIdx(m.ebo_byte_offset / sizeof(std::uint32_t), m.index_count); + } + for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) { + const MeshInfo& m = d.meshes[mi]; + if (m.lod1_index_count) + appendIdx(m.lod1_ebo_byte_offset / sizeof(std::uint32_t), m.lod1_index_count); + } +} +#endif // !__EMSCRIPTEN__ (bake-only serialisation helpers) + struct SidecarHeader { uint32_t magic; uint32_t version; @@ -86,100 +146,197 @@ static bool readVec(FILE* f, std::vector& v) { return true; } +#if !defined(__EMSCRIPTEN__) // bake path — compresses, desktop only bool writeSidecar(const std::string& ifc_path, const SidecarData& data) { std::string path = sidecarPath(ifc_path); FILE* f = fopen(path.c_str(), "wb"); if (!f) return false; + auto wr = [&](const void* p, std::size_t n) { + return fwrite(p, 1, n, f) == n; + }; + auto wrU64 = [&](std::uint64_t v) { return wr(&v, sizeof(v)); }; + auto wrBlock = [&](const std::vector& raw) -> bool { + auto z = SidecarCompress::compress(raw.data(), raw.size(), kSidecarZstdLevel); + if (raw.size() > 0 && z.empty()) return false; // compress failed + return wrU64(z.size()) && wrU64(raw.size()) && (z.empty() || wr(z.data(), z.size())); + }; + SidecarHeader hdr = { SIDECAR_MAGIC, SIDECAR_VERSION, SIDECAR_ENDIAN }; - if (fwrite(&hdr, sizeof(hdr), 1, f) != 1) { fclose(f); return false; } + if (!wr(&hdr, sizeof(hdr))) { fclose(f); return false; } - if (!writeVec(f, data.vertices)) { fclose(f); return false; } - if (!writeVec(f, data.indices)) { fclose(f); return false; } + // --- Geometry section: per-chunk zstd(vertex) + zstd(index) frames ------- + // Offsets in the chunk TOC are relative to the geometry section start, so + // the loader range-fetches exactly one chunk without reading anything else. + const long geom_len_pos = ftell(f); + if (!wrU64(0)) { fclose(f); return false; } // geom_bytes placeholder + const long geom_start = ftell(f); - // v15: a render-critical metadata block (meshes, instances, georef, chunk - // TOC) preceded by its byte length, then a deferred block (elements + - // string_table). The length lets the web loader read just the critical - // block before painting and fetch the property data lazily / not at all. - const long crit_len_pos = ftell(f); - uint64_t crit_bytes = 0; - if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; } - const long crit_start = ftell(f); - - if (!writeVec(f, data.meshes)) { fclose(f); return false; } - if (!writeVec(f, data.instances)) { fclose(f); return false; } - // v11 georef block (148 B). - if (fwrite(&data.has_coordinate_operation, 4, 1, f) != 1) { fclose(f); return false; } - if (fwrite(data.coordinate_operation_meters, - sizeof(double), 16, f) != 16) { fclose(f); return false; } - if (fwrite(&data.project_length_to_meters, - sizeof(double), 1, f) != 1) { fclose(f); return false; } - if (fwrite(&data.map_unit_to_meters, - sizeof(double), 1, f) != 1) { fclose(f); return false; } - if (!writeVec(f, data.chunks)) { fclose(f); return false; } - - // Backpatch the critical-block length. - const long crit_end = ftell(f); - if (crit_start < 0 || crit_end < 0) { fclose(f); return false; } - crit_bytes = uint64_t(crit_end - crit_start); - if (fseek(f, crit_len_pos, SEEK_SET) != 0) { fclose(f); return false; } - if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; } - if (fseek(f, crit_end, SEEK_SET) != 0) { fclose(f); return false; } - - // Deferred block: element tree + string table (UI/picking, never rendered). - if (!writeVec(f, data.elements)) { fclose(f); return false; } - uint32_t stbl_len = static_cast(data.string_table.size()); - if (fwrite(&stbl_len, 4, 1, f) != 1) { fclose(f); return false; } - if (stbl_len > 0 && fwrite(data.string_table.data(), 1, stbl_len, f) != stbl_len) { - fclose(f); return false; + std::vector chunks = data.chunks; // fill blob offsets below + std::vector vraw, iraw; + for (auto& c : chunks) { + extractChunkGeometry(data, c, vraw, iraw); + auto vz = SidecarCompress::compress(vraw.data(), vraw.size(), kSidecarZstdLevel); + auto iz = SidecarCompress::compress(iraw.data(), iraw.size(), kSidecarZstdLevel); + if ((vraw.size() && vz.empty()) || (iraw.size() && iz.empty())) { fclose(f); return false; } + c.v_comp_off = std::uint64_t(ftell(f) - geom_start); + c.v_comp_size = vz.size(); + c.v_raw_size = vraw.size(); + if (!vz.empty() && !wr(vz.data(), vz.size())) { fclose(f); return false; } + c.i_comp_off = std::uint64_t(ftell(f) - geom_start); + c.i_comp_size = iz.size(); + c.i_raw_size = iraw.size(); + if (!iz.empty() && !wr(iz.data(), iz.size())) { fclose(f); return false; } } + const long geom_end = ftell(f); + if (geom_start < 0 || geom_end < 0) { fclose(f); return false; } + if (fseek(f, geom_len_pos, SEEK_SET) != 0) { fclose(f); return false; } + if (!wrU64(std::uint64_t(geom_end - geom_start))) { fclose(f); return false; } + if (fseek(f, geom_end, SEEK_SET) != 0) { fclose(f); return false; } + + // --- Critical metadata block (zstd): meshes, instances, georef, chunk TOC + std::vector crit; + appendVec(crit, data.meshes); + appendVec(crit, data.instances); + appendBytes(crit, &data.has_coordinate_operation, 4); + appendBytes(crit, data.coordinate_operation_meters, sizeof(double) * 16); + appendBytes(crit, &data.project_length_to_meters, sizeof(double)); + appendBytes(crit, &data.map_unit_to_meters, sizeof(double)); + appendVec(crit, chunks); + if (!wrBlock(crit)) { fclose(f); return false; } + + // --- Deferred metadata block (zstd): element tree + string table --------- + std::vector def; + appendVec(def, data.elements); + std::uint32_t stbl_len = static_cast(data.string_table.size()); + appendBytes(def, &stbl_len, 4); + appendBytes(def, data.string_table.data(), stbl_len); + if (!wrBlock(def)) { fclose(f); return false; } fclose(f); return true; } +#endif // !__EMSCRIPTEN__ +// Cursor over an in-memory (decompressed) metadata block. +namespace { +struct BufReader { + const std::uint8_t* p; + std::size_t n; + std::size_t pos = 0; + bool take(void* dst, std::size_t k) { + if (pos + k > n) return false; + std::memcpy(dst, p + pos, k); + pos += k; + return true; + } + template + bool takeVec(std::vector& v) { + std::uint32_t c = 0; + if (!take(&c, 4)) return false; + if (pos + std::size_t(c) * sizeof(T) > n) return false; + v.resize(c); + if (c) { std::memcpy(v.data(), p + pos, std::size_t(c) * sizeof(T)); pos += std::size_t(c) * sizeof(T); } + return true; + } +}; +} // namespace + +// Full read: reconstruct the whole SidecarData (test/tooling path — the runtime +// streams via readSidecarMetadataOnly + per-chunk loads and never calls this). +// Decompresses the metadata blocks, then scatters each chunk's decompressed +// geometry back into the whole-model vertex/index arrays using the mesh offsets. std::optional readSidecar(const std::string& ifc_path) { std::string path = sidecarPath(ifc_path); FILE* f = fopen(path.c_str(), "rb"); if (!f) return std::nullopt; - auto fail = [&]() -> std::optional { fclose(f); return std::nullopt; }; SidecarHeader hdr; if (fread(&hdr, sizeof(hdr), 1, f) != 1) return fail(); - if (hdr.magic != SIDECAR_MAGIC || - hdr.version != SIDECAR_VERSION || + if (hdr.magic != SIDECAR_MAGIC || hdr.version != SIDECAR_VERSION || hdr.endian != SIDECAR_ENDIAN) return fail(); - SidecarData data; - if (!readVec(f, data.vertices)) return fail(); - if (!readVec(f, data.indices)) return fail(); + auto rd = [&](void* p, std::size_t k) { return fread(p, 1, k, f) == k; }; + auto rdU64 = [&](std::uint64_t& v) { return rd(&v, sizeof(v)); }; - // v15 critical-block length (consumed; only the streaming/web readers need - // it for a one-shot range read — here we read sequentially). - uint64_t crit_bytes = 0; - if (fread(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) return fail(); - - // Critical block: meshes, instances, georef, chunk TOC. - if (!readVec(f, data.meshes)) return fail(); - if (!readVec(f, data.instances)) return fail(); - if (fread(&data.has_coordinate_operation, 4, 1, f) != 1) return fail(); - if (fread(data.coordinate_operation_meters, - sizeof(double), 16, f) != 16) return fail(); - if (fread(&data.project_length_to_meters, - sizeof(double), 1, f) != 1) return fail(); - if (fread(&data.map_unit_to_meters, - sizeof(double), 1, f) != 1) return fail(); - if (!readVec(f, data.chunks)) return fail(); - - // Deferred block: element tree + string table. - if (!readVec(f, data.elements)) return fail(); - uint32_t stbl_len; - if (fread(&stbl_len, 4, 1, f) != 1) return fail(); - data.string_table.resize(stbl_len); - if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len) - return fail(); + std::uint64_t geom_bytes = 0; + if (!rdU64(geom_bytes)) return fail(); + std::vector geom(static_cast(geom_bytes)); + if (geom_bytes && !rd(geom.data(), geom.size())) return fail(); + auto readBlock = [&](std::vector& out) -> bool { + std::uint64_t comp = 0, raw = 0; + if (!rdU64(comp) || !rdU64(raw)) return false; + std::vector z(static_cast(comp)); + if (comp && !rd(z.data(), z.size())) return false; + out.assign(std::size_t(raw), 0); + return SidecarCompress::decompress(z.data(), z.size(), out.data(), out.size()); + }; + std::vector crit, def; + if (!readBlock(crit) || !readBlock(def)) return fail(); fclose(f); + + SidecarData data; + BufReader cr{ crit.data(), crit.size() }; + if (!cr.takeVec(data.meshes)) return std::nullopt; + if (!cr.takeVec(data.instances)) return std::nullopt; + if (!cr.take(&data.has_coordinate_operation, 4)) return std::nullopt; + if (!cr.take(data.coordinate_operation_meters, sizeof(double) * 16)) return std::nullopt; + if (!cr.take(&data.project_length_to_meters, sizeof(double))) return std::nullopt; + if (!cr.take(&data.map_unit_to_meters, sizeof(double))) return std::nullopt; + if (!cr.takeVec(data.chunks)) return std::nullopt; + + BufReader dr{ def.data(), def.size() }; + if (!dr.takeVec(data.elements)) return std::nullopt; + std::uint32_t stbl_len = 0; + if (!dr.take(&stbl_len, 4)) return std::nullopt; + data.string_table.resize(stbl_len); + if (stbl_len && !dr.take(data.string_table.data(), stbl_len)) return std::nullopt; + + // Reconstruct the whole-model vertex/index arrays from the per-chunk blobs. + std::size_t vsize = 0, isize = 0; + for (const auto& m : data.meshes) { + vsize = std::max(vsize, + std::size_t(m.vbo_byte_offset) + std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES); + isize = std::max(isize, m.ebo_byte_offset / sizeof(std::uint32_t) + m.index_count); + if (m.lod1_index_count) + isize = std::max(isize, m.lod1_ebo_byte_offset / sizeof(std::uint32_t) + m.lod1_index_count); + } + data.vertices.assign(vsize, 0); + data.indices.assign(isize, 0); + for (const auto& c : data.chunks) { + if (c.v_comp_off + c.v_comp_size > geom.size() || + c.i_comp_off + c.i_comp_size > geom.size()) return std::nullopt; + std::vector vraw(static_cast(c.v_raw_size)); + std::vector iraw(static_cast(c.i_raw_size)); + if (!SidecarCompress::decompress(geom.data() + c.v_comp_off, c.v_comp_size, vraw.data(), vraw.size()) || + !SidecarCompress::decompress(geom.data() + c.i_comp_off, c.i_comp_size, iraw.data(), iraw.size())) + return std::nullopt; + const auto* iu = reinterpret_cast(iraw.data()); + std::size_t vcur = 0, icur = 0; + const std::uint32_t end = c.first_mesh + c.mesh_count; + for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) { + const MeshInfo& m = data.meshes[mi]; + const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES; + if (vcur + vn <= vraw.size() && m.vbo_byte_offset + vn <= data.vertices.size()) + std::memcpy(&data.vertices[m.vbo_byte_offset], vraw.data() + vcur, vn); + vcur += vn; + } + for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) { + const MeshInfo& m = data.meshes[mi]; + if (!m.index_count) continue; + if (icur + m.index_count <= iraw.size() / 4) + std::memcpy(&data.indices[m.ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.index_count * 4); + icur += m.index_count; + } + for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) { + const MeshInfo& m = data.meshes[mi]; + if (!m.lod1_index_count) continue; + if (icur + m.lod1_index_count <= iraw.size() / 4) + std::memcpy(&data.indices[m.lod1_ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.lod1_index_count * 4); + icur += m.lod1_index_count; + } + } return data; } diff --git a/src/ifcviewer/SidecarCache.h b/src/ifcviewer/SidecarCache.h index f8b6d95341..e1b18e715a 100644 --- a/src/ifcviewer/SidecarCache.h +++ b/src/ifcviewer/SidecarCache.h @@ -81,16 +81,32 @@ static constexpr uint32_t SIDECAR_MAGIC = 0x49465657; // "IFVW" // painting, so first geometry no longer waits on the property data; the // deferred block is fetched lazily (or skipped where unused). Desktop // reads both. No back-compat: regenerate sidecars. -static constexpr uint32_t SIDECAR_VERSION = 15; +// v16 = Geometry + metadata are zstd-COMPRESSED. Each chunk's vertex bytes and +// index bytes are stored as two independent zstd frames (so per-chunk +// Range streaming still works — you fetch + decompress just one chunk), +// and the critical + deferred metadata blocks are single zstd frames. +// The chunk TOC records each chunk's compressed blob offsets/sizes plus +// the raw (decompressed) sizes. ~3-5x fewer bytes over the wire while +// keeping HTTP Range intact (unlike server Content-Encoding). No +// back-compat: regenerate sidecars. +static constexpr uint32_t SIDECAR_VERSION = 16; static constexpr uint32_t SIDECAR_ENDIAN = 0x01020304; -// Chunk table-of-contents entry (v14+). A chunk is a CONTIGUOUS range of -// meshes in the (reordered) meshes array — and therefore a contiguous span of -// vertex + index bytes, since the geometry is laid out in chunk order. The -// loader builds chunk `i` from meshes [first_mesh, first_mesh + mesh_count). +// Chunk table-of-contents entry (v16). A chunk is a CONTIGUOUS range of meshes +// [first_mesh, first_mesh + mesh_count). Its vertex + index bytes are stored as +// two zstd frames in the geometry section; the loader fetches [v_comp_off, +// +v_comp_size) / [i_comp_off, +i_comp_size) (offsets relative to the geometry +// section start) and decompresses them to v_raw_size / i_raw_size bytes — the +// chunk-local (vbytes, idx) applyStreamedChunk consumes. struct SidecarChunk { uint32_t first_mesh; uint32_t mesh_count; + uint64_t v_comp_off; + uint64_t v_comp_size; + uint64_t v_raw_size; + uint64_t i_comp_off; + uint64_t i_comp_size; + uint64_t i_raw_size; }; // Fixed-size element record. Strings are stored as (offset, length) pairs diff --git a/src/ifcviewer/SidecarCompress.cpp b/src/ifcviewer/SidecarCompress.cpp new file mode 100644 index 0000000000..1e49e81bdb --- /dev/null +++ b/src/ifcviewer/SidecarCompress.cpp @@ -0,0 +1,46 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#include "SidecarCompress.h" + +#include + +namespace SidecarCompress { + +bool decompress(const std::uint8_t* src, std::size_t src_size, + std::uint8_t* dst, std::size_t raw_size) { + if (raw_size == 0) return src_size == 0; // empty in ↔ empty out + if (!src || !dst || src_size == 0) return false; + const size_t got = ZSTD_decompress(dst, raw_size, src, src_size); + return !ZSTD_isError(got) && got == raw_size; +} + +#if !defined(__EMSCRIPTEN__) +std::vector compress(const std::uint8_t* src, std::size_t n, + int level) { + if (n == 0) return {}; + std::vector out(ZSTD_compressBound(n)); + const size_t got = ZSTD_compress(out.data(), out.size(), src, n, level); + if (ZSTD_isError(got)) return {}; + out.resize(got); + return out; +} +#endif + +} // namespace SidecarCompress diff --git a/src/ifcviewer/SidecarCompress.h b/src/ifcviewer/SidecarCompress.h new file mode 100644 index 0000000000..135bf5d58b --- /dev/null +++ b/src/ifcviewer/SidecarCompress.h @@ -0,0 +1,51 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#ifndef SIDECARCOMPRESS_H +#define SIDECARCOMPRESS_H + +// zstd wrappers for the .ifcview format (v16+). Geometry chunks and metadata +// blocks are stored zstd-compressed so the network pulls far fewer bytes while +// keeping HTTP Range streaming intact (unlike server Content-Encoding, which +// can't be byte-ranged). The baker compresses (desktop only); every loader — +// desktop and web — decompresses. The web build links the vendored single-file +// zstd DECODER (third_party/zstddeclib.c); desktop links the full libzstd. + +#include +#include +#include + +namespace SidecarCompress { + +// Decompress a zstd frame in [src, src+src_size) into dst, which must have room +// for exactly raw_size bytes. Returns false on any zstd error or if the frame +// doesn't expand to exactly raw_size. Available on all platforms. +bool decompress(const std::uint8_t* src, std::size_t src_size, + std::uint8_t* dst, std::size_t raw_size); + +#if !defined(__EMSCRIPTEN__) +// Compress [src, src+n) with zstd at `level`. Returns the compressed frame, or +// an empty vector on error. Bake/desktop only — the web build never compresses. +std::vector compress(const std::uint8_t* src, std::size_t n, + int level); +#endif + +} // namespace SidecarCompress + +#endif // SIDECARCOMPRESS_H diff --git a/src/ifcviewer/StreamingLoader.cpp b/src/ifcviewer/StreamingLoader.cpp index d5de5b0b10..3238e5f321 100644 --- a/src/ifcviewer/StreamingLoader.cpp +++ b/src/ifcviewer/StreamingLoader.cpp @@ -34,6 +34,7 @@ // can be range-read on demand. File handle is closed before return. #include "StreamingLoader.h" +#include "SidecarCompress.h" #include #include @@ -88,14 +89,14 @@ std::string sidecarPath(const std::string& ifc_path) { } // namespace -bool parseSidecarHead(const uint8_t* data, size_t n, uint32_t& out_num_vertex_bytes) { +bool parseSidecarHead(const uint8_t* data, size_t n, uint64_t& out_geom_bytes) { if (n < SIDECAR_HEAD_BYTES) return false; SidecarHeaderRaw hdr; std::memcpy(&hdr, data, sizeof(hdr)); if (hdr.magic != SIDECAR_MAGIC) return false; if (hdr.version != SIDECAR_VERSION) return false; if (hdr.endian != SIDECAR_ENDIAN) return false; - std::memcpy(&out_num_vertex_bytes, data + sizeof(hdr), 4); + std::memcpy(&out_geom_bytes, data + sizeof(hdr), 8); return true; } @@ -134,58 +135,79 @@ std::optional readSidecarMetadataOnly(const std::string& ifc_p return std::nullopt; }; - // Head: 12-byte header + the vertex-byte count. The vertex section starts - // immediately after, at SIDECAR_HEAD_BYTES. + // Head (v16): 12-byte header + the compressed-geometry-section length. The + // metadata blocks follow the geometry at SIDECAR_HEAD_BYTES + geom_bytes. uint8_t head[SIDECAR_HEAD_BYTES]; if (std::fread(head, 1, SIDECAR_HEAD_BYTES, f) != SIDECAR_HEAD_BYTES) return fail(); - uint32_t num_vertex_bytes = 0; - if (!parseSidecarHead(head, SIDECAR_HEAD_BYTES, num_vertex_bytes)) return fail(); + uint64_t geom_bytes = 0; + if (!parseSidecarHead(head, SIDECAR_HEAD_BYTES, geom_bytes)) return fail(); StreamingSidecar out; - out.file_path = path; - out.vertex_section_offset = SIDECAR_HEAD_BYTES; - out.vertex_total_bytes = num_vertex_bytes; + out.file_path = path; + out.geometry_section_offset = SIDECAR_HEAD_BYTES; - // Skip the vertex section; read the index count that follows it. - if (std::fseek(f, long(num_vertex_bytes), SEEK_CUR) != 0) return fail(); - uint32_t num_indices = 0; - if (std::fread(&num_indices, 4, 1, f) != 1) return fail(); - out.index_section_offset = uint64_t(std::ftell(f)); - out.index_total_count = num_indices; - - // Skip the index section; the metadata tail runs from there to EOF. - if (std::fseek(f, long(num_indices) * 4, SEEK_CUR) != 0) return fail(); - const long tail_off = std::ftell(f); - if (tail_off < 0) return fail(); - if (std::fseek(f, 0, SEEK_END) != 0) return fail(); - const long file_end = std::ftell(f); - if (file_end < tail_off) return fail(); - if (std::fseek(f, tail_off, SEEK_SET) != 0) return fail(); - - std::vector tail(size_t(file_end - tail_off)); - if (!tail.empty() && std::fread(tail.data(), 1, tail.size(), f) != tail.size()) + // Skip the geometry section; the two compressed metadata blocks follow. + if (std::fseek(f, long(SIDECAR_HEAD_BYTES) + long(geom_bytes), SEEK_SET) != 0) return fail(); + + // Each metadata block on disk is [comp u64][raw u64][zstd frame]. + auto readBlock = [&](std::vector& raw, + uint64_t* comp_off = nullptr, uint64_t* comp_sz = nullptr, + uint64_t* raw_sz = nullptr) -> bool { + uint64_t comp = 0, rawn = 0; + if (std::fread(&comp, 8, 1, f) != 1 || std::fread(&rawn, 8, 1, f) != 1) return false; + const long here = std::ftell(f); + std::vector z(static_cast(comp)); + if (comp && std::fread(z.data(), 1, z.size(), f) != z.size()) return false; + raw.assign(size_t(rawn), 0); + if (comp_off) *comp_off = uint64_t(here); + if (comp_sz) *comp_sz = comp; + if (raw_sz) *raw_sz = rawn; + return SidecarCompress::decompress(z.data(), z.size(), raw.data(), raw.size()); + }; + + std::vector crit, def; + if (!readBlock(crit)) return fail(); + if (!readBlock(def, &out.deferred_comp_offset, &out.deferred_comp_size, + &out.deferred_raw_size)) return fail(); std::fclose(f); - // Tail (v15) = [critical_meta_bytes (8)][critical block][deferred block]. - // Desktop is local, so read both; the web path reads only the critical - // block before painting and the deferred block on demand. - if (tail.size() < sizeof(uint64_t)) return std::nullopt; - uint64_t crit_bytes = 0; - std::memcpy(&crit_bytes, tail.data(), sizeof(crit_bytes)); - const size_t crit_off = sizeof(crit_bytes); - if (crit_off + crit_bytes > tail.size()) return std::nullopt; - out.critical_meta_offset = out.index_section_offset - + uint64_t(num_indices) * 4u + crit_off; - out.critical_meta_bytes = crit_bytes; - if (!parseSidecarCritical(tail.data() + crit_off, size_t(crit_bytes), out.meta)) - return std::nullopt; - if (!parseSidecarDeferred(tail.data() + crit_off + crit_bytes, - tail.size() - crit_off - size_t(crit_bytes), out.meta)) - return std::nullopt; + // Desktop reads both blocks up front; the web path reads only critical + // before painting and fetches the deferred block on demand. + if (!parseSidecarCritical(crit.data(), crit.size(), out.meta)) return std::nullopt; + if (!parseSidecarDeferred(def.data(), def.size(), out.meta)) return std::nullopt; return out; } +bool readChunkGeometryCompressed(const std::string& ifc_path, + std::uint64_t geometry_section_offset, + std::uint64_t v_comp_off, std::uint64_t v_comp_size, + std::uint64_t v_raw_size, + std::uint64_t i_comp_off, std::uint64_t i_comp_size, + std::uint64_t i_raw_size, + std::vector& out_vbytes, + std::vector& out_idx) { + const std::string path = sidecarPath(ifc_path); + FILE* f = std::fopen(path.c_str(), "rb"); + if (!f) return false; + auto readFrame = [&](std::uint64_t off, std::uint64_t comp, std::uint64_t raw, + std::uint8_t* dst) -> bool { + if (raw == 0) return comp == 0; + std::vector z(static_cast(comp)); + if (std::fseek(f, long(geometry_section_offset + off), SEEK_SET) != 0) return false; + if (comp && std::fread(z.data(), 1, z.size(), f) != z.size()) return false; + return SidecarCompress::decompress(z.data(), z.size(), dst, size_t(raw)); + }; + out_vbytes.assign(size_t(v_raw_size), 0); + out_idx.assign(size_t(i_raw_size / sizeof(std::uint32_t)), 0); + const bool ok = + readFrame(v_comp_off, v_comp_size, v_raw_size, out_vbytes.data()) && + readFrame(i_comp_off, i_comp_size, i_raw_size, + reinterpret_cast(out_idx.data())); + std::fclose(f); + return ok; +} + bool readSidecarVertexChunk(const std::string& ifc_path, uint64_t vertex_section_offset, uint64_t chunk_byte_offset, diff --git a/src/ifcviewer/StreamingLoader.h b/src/ifcviewer/StreamingLoader.h index ff53792f31..7f1d5c2810 100644 --- a/src/ifcviewer/StreamingLoader.h +++ b/src/ifcviewer/StreamingLoader.h @@ -46,25 +46,20 @@ struct StreamingSidecar { // Everything except vertices + indices — same shape as SidecarData but // with empty vertices / indices vectors. The renderer uses meshes / - // instances / georef / elements immediately. + // instances / georef / chunks immediately (elements/strings deferred). SidecarData meta; - // Byte offsets in the on-disk file where the vertex and index sections - // start (after their 4-byte count headers). Pair with vertex_total_bytes - // / index_total_bytes for the section length; per-chunk reads slice - // arbitrary ranges within these. - uint64_t vertex_section_offset = 0; - uint64_t vertex_total_bytes = 0; - uint64_t index_section_offset = 0; - uint64_t index_total_count = 0; // u32 indices, NOT bytes + // v16: the compressed geometry section starts here. Each chunk's two zstd + // blobs live at geometry_section_offset + SidecarChunk.{v_comp_off,i_comp_off}; + // a per-chunk load fetches [that, +*_comp_size) and decompresses to *_raw_size. + uint64_t geometry_section_offset = 0; - // v15 deferred-metadata locator. The render-critical metadata block starts - // at critical_meta_offset and is critical_meta_bytes long; the deferred - // block (elements + string_table) runs from there to EOF. The web loader - // reads only the critical block before painting and fetches the deferred - // block on demand from [critical_meta_offset + critical_meta_bytes, EOF). - uint64_t critical_meta_offset = 0; - uint64_t critical_meta_bytes = 0; + // v16 deferred (property) block locator: a single zstd frame at + // deferred_comp_offset of deferred_comp_size bytes → deferred_raw_size. The + // web loader fetches it on demand (elements/strings); desktop reads it up front. + uint64_t deferred_comp_offset = 0; + uint64_t deferred_comp_size = 0; + uint64_t deferred_raw_size = 0; // Resolved on-disk path so subsequent chunk reads can re-open / seek. std::string file_path; @@ -75,6 +70,20 @@ struct StreamingSidecar { // before return — callers re-open for per-chunk reads. std::optional readSidecarMetadataOnly(const std::string& ifc_path); +// Read + decompress one chunk's geometry (v16) from disk: the vertex zstd frame +// at [geometry_section_offset + v_comp_off, +v_comp_size) → out_vbytes (v_raw +// bytes) and the index frame → out_idx (i_raw/4 u32s). Returns false on I/O or +// decompress failure. Used by the desktop StreamingThread worker + the sync +// first-frame fallback; the web path decompresses in beginWebChunkLoad instead. +bool readChunkGeometryCompressed(const std::string& ifc_path, + std::uint64_t geometry_section_offset, + std::uint64_t v_comp_off, std::uint64_t v_comp_size, + std::uint64_t v_raw_size, + std::uint64_t i_comp_off, std::uint64_t i_comp_size, + std::uint64_t i_raw_size, + std::vector& out_vbytes, + std::vector& out_idx); + // --- Pure, buffer-based building blocks ------------------------------------ // // The metadata lives in two disjoint regions of the file: a small fixed @@ -85,15 +94,15 @@ std::optional readSidecarMetadataOnly(const std::string& ifc_p // source and hand the bytes to these parsers, so the wire-format knowledge // lives in exactly one place and is unit-testable without touching a file. -// Bytes the head spans: SidecarHeader (12) + uint32 num_vertex_bytes (4). -inline constexpr std::size_t SIDECAR_HEAD_BYTES = 16; +// Bytes the head spans: SidecarHeader (12) + uint64 geometry-section length (8). +inline constexpr std::size_t SIDECAR_HEAD_BYTES = 20; -// Parse the 16-byte head. Validates magic / version / endian and, on success, -// writes the vertex-section byte count (which locates the index-count field at -// SIDECAR_HEAD_BYTES + out_num_vertex_bytes). Returns false if `n` is short or -// the header is wrong. `data` must point at the start of the file. +// Parse the 20-byte head (v16). Validates magic / version / endian and, on +// success, writes the compressed-geometry-section byte length (the metadata +// blocks follow at SIDECAR_HEAD_BYTES + out_geom_bytes). Returns false if `n` +// is short or the header is wrong. `data` must point at the start of the file. bool parseSidecarHead(const std::uint8_t* data, std::size_t n, - std::uint32_t& out_num_vertex_bytes); + std::uint64_t& out_geom_bytes); // Parse the v15 render-CRITICAL metadata block (mesh dict, instance dict, // georef, chunk TOC) — everything needed to set up + draw the scene. `data` diff --git a/src/ifcviewer/StreamingThread.cpp b/src/ifcviewer/StreamingThread.cpp index 309181028d..15ef91d919 100644 --- a/src/ifcviewer/StreamingThread.cpp +++ b/src/ifcviewer/StreamingThread.cpp @@ -97,21 +97,11 @@ void StreamingThread::workerLoop() { Result res; res.model_id = req.model_id; res.chunk_idx = req.chunk_idx; - res.success = true; - if (!req.v_ranges.empty()) { - if (!readSidecarVertexRanges(req.file_path, - req.vertex_section_offset, - req.v_ranges, res.vbytes)) { - res.success = false; - } - } - if (res.success && !req.i_ranges.empty()) { - if (!readSidecarIndexRanges(req.file_path, - req.index_section_offset, - req.i_ranges, res.idx)) { - res.success = false; - } - } + res.success = readChunkGeometryCompressed( + req.file_path, req.geometry_section_offset, + req.v_comp_off, req.v_comp_size, req.v_raw_size, + req.i_comp_off, req.i_comp_size, req.i_raw_size, + res.vbytes, res.idx); { std::unique_lock lk(mu_); diff --git a/src/ifcviewer/StreamingThread.h b/src/ifcviewer/StreamingThread.h index b1e065477a..d019ccf70b 100644 --- a/src/ifcviewer/StreamingThread.h +++ b/src/ifcviewer/StreamingThread.h @@ -44,15 +44,15 @@ class StreamingThread { public: struct Request { - uint32_t model_id; - std::size_t chunk_idx; - std::string file_path; - uint64_t vertex_section_offset; - uint64_t index_section_offset; - // (section-relative byte_offset, byte_size) - std::vector> v_ranges; - // (first_u32, count_u32) - std::vector> i_ranges; + uint32_t model_id; + std::size_t chunk_idx; + std::string file_path; + // v16: the chunk's two zstd frames in the geometry section. The reader + // fetches [geometry_section_offset + *_comp_off, +*_comp_size) and + // decompresses to *_raw_size. + uint64_t geometry_section_offset = 0; + uint64_t v_comp_off = 0, v_comp_size = 0, v_raw_size = 0; + uint64_t i_comp_off = 0, i_comp_size = 0, i_raw_size = 0; }; struct Result { diff --git a/src/ifcviewer/ViewportCore.cpp b/src/ifcviewer/ViewportCore.cpp index 000bb9cab8..85e46f339b 100644 --- a/src/ifcviewer/ViewportCore.cpp +++ b/src/ifcviewer/ViewportCore.cpp @@ -1384,6 +1384,15 @@ bool ViewportCore::createPool() { std::max(MIN_POOL_CAPACITY, INITIAL_SUB_BUFFER)); pool_.configure(instance_, device_, pool_usage, per_sub, "ifcviewer-wgpu.pool"); +#if defined(__EMSCRIPTEN__) + // Cap total pool capacity below the wasm heap ceiling. On web a growth that + // would push the heap past MAXIMUM_MEMORY is a bad_alloc → uncatchable + // abort, so the pool must stop growing (and evict) before then. Leave + // headroom for metadata (instances/maps), transient decompression buffers, + // and wgpu overhead. Big federations then keep a bounded, highest-priority + // resident set instead of aborting. + pool_.setMaxTotalCapacity(3072ull * 1024 * 1024); // 3 GB (heap ceiling 4 GB) +#endif Log::info() << "wgpu: pool per-sub-buffer capacity = " << (per_sub / (1024 * 1024)) << " MB (grows lazily on " << "demand; device maxBufferSize = " @@ -1723,6 +1732,7 @@ void ViewportCore::shutdown() { // =========================================================================== #include "StreamingLoader.h" +#include "SidecarCompress.h" namespace { @@ -1941,37 +1951,17 @@ StreamingThread::Request ViewportCore::makeChunkRequest( std::uint32_t model_id) { const auto& c = m.chunks[chunk_idx]; StreamingThread::Request req; - req.model_id = model_id; - req.chunk_idx = chunk_idx; - req.file_path = m.streaming_file_path; - req.vertex_section_offset = m.streaming_vertex_section_offset; - req.index_section_offset = m.streaming_index_section_offset; - req.v_ranges.reserve(c.mesh_ids.size()); - req.i_ranges.reserve(c.mesh_ids.size()); - for (std::uint32_t mi : c.mesh_ids) { - const MeshInfo& mesh = m.meshes[mi]; - const std::uint64_t v_bytes = - std::uint64_t(mesh.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES; - if (v_bytes > 0) { - req.v_ranges.emplace_back(std::uint64_t(mesh.vbo_byte_offset), v_bytes); - } - if (mesh.index_count > 0) { - req.i_ranges.emplace_back( - std::uint64_t(mesh.ebo_byte_offset / sizeof(std::uint32_t)), - std::uint64_t(mesh.index_count)); - } - } - // LOD1 indices second pass — matches the chunk-local packing order - // (all LOD0 first, then LOD1) so the worker's concatenated index - // result lands at the offsets recorded in - // m.mesh_chunk_local_lod1_first_u32. - for (std::uint32_t mi : c.mesh_ids) { - const MeshInfo& mesh = m.meshes[mi]; - if (mesh.lod1_index_count == 0) continue; - req.i_ranges.emplace_back( - std::uint64_t(mesh.lod1_ebo_byte_offset / sizeof(std::uint32_t)), - std::uint64_t(mesh.lod1_index_count)); - } + req.model_id = model_id; + req.chunk_idx = chunk_idx; + req.file_path = m.streaming_file_path; + // v16: one compressed vertex frame + one compressed index frame per chunk. + req.geometry_section_offset = m.geometry_section_offset; + req.v_comp_off = c.v_comp_off; + req.v_comp_size = c.v_comp_size; + req.v_raw_size = c.vertex_byte_size; + req.i_comp_off = c.i_comp_off; + req.i_comp_size = c.i_comp_size; + req.i_raw_size = c.index_count * sizeof(std::uint32_t); return req; } @@ -1982,34 +1972,18 @@ bool ViewportCore::loadChunkBytesAndUploadGpu(ModelGpuData& m, if (c.is_resident) return true; if (m.streaming_file_path.empty()) return false; - // Synchronous fallback: build the request, do the disk read inline, - // apply. Used only when the async path can't be — i.e. by the - // screenshot test on first frame. - StreamingThread::Request req = makeChunkRequest(m, chunk_idx, /*model_id*/ 0); - + // Synchronous fallback: read + decompress the chunk inline, apply. Used + // only when the async path can't be — i.e. by the screenshot test on the + // first frame. std::vector vbytes; std::vector idx; - if (!req.v_ranges.empty()) { - if (!readSidecarVertexRanges(req.file_path, - req.vertex_section_offset, - req.v_ranges, vbytes)) { - Log::warn() << "[wgpu stream] failed to read vertex chunk " - << chunk_idx - << " (" << req.v_ranges.size() << " ranges, total " - << c.vertex_byte_size << " B)"; - return false; - } - } - if (!req.i_ranges.empty()) { - if (!readSidecarIndexRanges(req.file_path, - req.index_section_offset, - req.i_ranges, idx)) { - Log::warn() << "[wgpu stream] failed to read index chunk " - << chunk_idx - << " (" << req.i_ranges.size() << " ranges, total " - << c.index_count << " indices)"; - return false; - } + if (!readChunkGeometryCompressed( + m.streaming_file_path, m.geometry_section_offset, + c.v_comp_off, c.v_comp_size, c.vertex_byte_size, + c.i_comp_off, c.i_comp_size, c.index_count * sizeof(std::uint32_t), + vbytes, idx)) { + Log::warn() << "[wgpu stream] failed to read/decompress chunk " << chunk_idx; + return false; } return applyStreamedChunk(m, chunk_idx, vbytes, idx); } @@ -2316,24 +2290,34 @@ void ViewportCore::driveStreamingLoads() { if (cand.m->streaming_from_web && pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) { // Not enough VALIDATED pool space for this chunk plus what's already - // in flight. Don't fetch — the bytes would arrive, fail the alloc on - // a not-yet-grown pool, and re-fetch (network thrash). Instead grow - // the pool first (async on web: a provisional sub-buffer validates a - // frame or two later, then this chunk fits and is fetched exactly - // once). When the pool is saturated (model exceeds GPU memory) hold - // the chunk off for the full cooldown so we keep a stable resident - // subset instead of re-fetching what will never fit. Blocking before - // the evictor also avoids phase-2 visible↔visible swap thrash. + // in flight. If the pool can still grow, grow FIRST (async on web: a + // provisional sub-buffer validates a frame or two later) — cheaper + // than evict/refetch thrash while the model still fits by growing. if (pool_.can_grow()) { pool_.requestGrowth(); c.blocked_cooldown_until_frame_idx = streaming_frame_idx_ + kGrowBackoffFrames; - } else { + more_pending = true; + continue; + } + // At the hard capacity budget (setMaxTotalCapacity): growth can't + // help — growing further would abort the wasm heap. Evict lower- + // priority / LRU resident chunks so this priority-sorted candidate + // fits. This is the only path that keeps a big federation navigable + // once it exceeds the memory budget (highest-contribution chunks win). + while (pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) { + if (evict_one_lru()) continue; + if (evict_lowest_priority_than(cand.mid, std::uint32_t(cand.ci), + cand.priority)) continue; + break; + } + if (pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) { c.blocked_cooldown_until_frame_idx = streaming_frame_idx_ + kBlockedCooldownFrames; + more_pending = true; + continue; // couldn't free enough — hold off this frame } - more_pending = true; - continue; + // Freed enough — fall through to the web load below. } #endif while (!pool_can_fit(c.vertex_byte_size) @@ -2853,13 +2837,12 @@ void ViewportCore::applyCachedModel(std::uint32_t model_id, } ModelGpuData m; - m.vertex_bytes = metadata.vertex_total_bytes; - m.index_count = std::uint32_t(metadata.index_total_count); + m.vertex_bytes = 0; // accumulated from chunks below (v16 has no section) + m.index_count = 0; m.mesh_count = std::uint32_t(metadata.meta.meshes.size()); m.instance_count = std::uint32_t(metadata.meta.instances.size()); - m.streaming_file_path = metadata.file_path; - m.streaming_vertex_section_offset = metadata.vertex_section_offset; - m.streaming_index_section_offset = metadata.index_section_offset; + m.streaming_file_path = metadata.file_path; + m.geometry_section_offset = metadata.geometry_section_offset; // ---- Spatial chunk plan ---------------------------------------------- // A sidecar carries a baked chunk TOC (v14): each chunk is a contiguous @@ -2978,6 +2961,14 @@ void ViewportCore::applyCachedModel(std::uint32_t model_id, c.vertex_byte_size = std::uint64_t(chunk_local_v) * INSTANCED_VERTEX_STRIDE_BYTES; c.index_count = chunk_local_i + chunk_local_lod1; c.lod1_index_count = chunk_local_lod1; + // v16: compressed-blob locators from the baked TOC (streaming path). + if (ci < metadata.meta.chunks.size()) { + const SidecarChunk& sc = metadata.meta.chunks[ci]; + c.v_comp_off = sc.v_comp_off; c.v_comp_size = sc.v_comp_size; + c.i_comp_off = sc.i_comp_off; c.i_comp_size = sc.i_comp_size; + } + m.vertex_bytes += c.vertex_byte_size; + m.index_count += std::uint32_t(c.index_count); // Small per-chunk buffers, allocated upfront so cull can write into // them. visible_draws_buffer cap = chunk's instance count. @@ -3385,44 +3376,33 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i ModelGpuData& m = it->second; if (chunk_idx >= m.chunks.size()) return; - const StreamingThread::Request req = makeChunkRequest(m, chunk_idx, model_id); + const ModelGpuData::Chunk& c = m.chunks[chunk_idx]; const int sid = m.web_source_id; // which registered byte-source to read from - const std::uint64_t vsec = req.vertex_section_offset; - const std::uint64_t isec = req.index_section_offset; - const std::vector> v_ranges = req.v_ranges; - // i_ranges are (first_u32, count_u32); convert to byte ranges. - std::vector> i_byte_ranges; - i_byte_ranges.reserve(req.i_ranges.size()); - for (const auto& [first_u32, count] : req.i_ranges) - i_byte_ranges.emplace_back(first_u32 * 4u, count * 4u); + const std::uint64_t geom = m.geometry_section_offset; + const std::uint64_t v_comp_off = c.v_comp_off, v_comp_size = c.v_comp_size; + const std::uint64_t i_comp_off = c.i_comp_off, i_comp_size = c.i_comp_size; + const std::uint64_t v_raw = c.vertex_byte_size; + const std::uint64_t i_raw = std::uint64_t(c.index_count) * sizeof(std::uint32_t); - // Reserve this load's pool footprint while it's in flight (released when it - // resolves) so driveStreamingLoads doesn't over-commit the pool — see - // streaming_web_inflight_bytes_. - const std::uint64_t need = m.chunks[chunk_idx].vertex_byte_size - + std::uint64_t(m.chunks[chunk_idx].index_count) * sizeof(std::uint32_t); + // Reserve the RAW (decompressed) footprint while in flight so + // driveStreamingLoads doesn't over-commit the pool. + const std::uint64_t need = v_raw + i_raw; streaming_web_inflight_bytes_ += need; - // Fire the vertex and index range reads CONCURRENTLY and join when both - // land — over a network this halves per-chunk latency vs reading vertices - // then indices serially (two round trips → one). The join holds both - // payloads + completion flags; whichever read finishes second runs the - // apply. Re-look-up the model at apply time: a resetScene() could have - // landed mid-flight, in which case the model id is gone and we drop it. + // Fetch the chunk's two zstd frames CONCURRENTLY (vertex + index) and join + // when both land, then decompress and apply. Re-look-up the model at apply + // time: a resetScene() could have landed mid-flight. struct ChunkJoin { - std::vector vbytes; - std::vector idx; + std::vector vz, iz; // compressed frames bool v_done = false, i_done = false, v_ok = false, i_ok = false; }; auto join = std::make_shared(); - std::function finish = [this, model_id, chunk_idx, need, join]() { - if (!join->v_done || !join->i_done) return; // wait for the other read - // Release the in-flight reservation (clamped — a mid-flight resetScene - // could have zeroed it) + the concurrency slot, regardless of outcome. - streaming_web_inflight_bytes_ -= - std::min(streaming_web_inflight_bytes_, need); + std::function finish = + [this, model_id, chunk_idx, need, v_raw, i_raw, join]() { + if (!join->v_done || !join->i_done) return; // wait for the other frame + streaming_web_inflight_bytes_ -= std::min(streaming_web_inflight_bytes_, need); if (streaming_web_inflight_count_ > 0) --streaming_web_inflight_count_; - host_->requestFrame(); // a slot freed — let driveStreamingLoads issue more + host_->requestFrame(); auto mit = models_gpu_.find(model_id); if (mit == models_gpu_.end()) return; @@ -3430,12 +3410,16 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i if (chunk_idx >= mm.chunks.size()) return; auto& cc = mm.chunks[chunk_idx]; cc.is_loading = false; - // On read failure or a full pool, back off instead of re-candidating - // next frame → re-fetch thrash. If the pool can still grow (its async - // sub-buffer is mid-validation), retry soon; if it's saturated, hold - // off for the full cooldown. - if (!join->v_ok || !join->i_ok - || !applyStreamedChunk(mm, chunk_idx, join->vbytes, join->idx)) { + + std::vector vbytes(static_cast(v_raw)); + std::vector idx(static_cast(i_raw / sizeof(std::uint32_t))); + const bool ok = join->v_ok && join->i_ok + && SidecarCompress::decompress(join->vz.data(), join->vz.size(), + vbytes.data(), vbytes.size()) + && SidecarCompress::decompress(join->iz.data(), join->iz.size(), + reinterpret_cast(idx.data()), + std::size_t(i_raw)); + if (!ok || !applyStreamedChunk(mm, chunk_idx, vbytes, idx)) { cc.blocked_cooldown_until_frame_idx = streaming_frame_idx_ + (pool_.can_grow() ? kGrowBackoffFrames : kBlockedCooldownFrames); return; @@ -3443,22 +3427,13 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i host_->requestFrame(); }; - webReadRangesAsync(sid, vsec, v_ranges, - [join, finish](bool ok, std::vector&& vbytes) { - join->v_ok = ok; - join->vbytes = std::move(vbytes); - join->v_done = true; - finish(); + webReadRangesAsync(sid, geom, {{v_comp_off, v_comp_size}}, + [join, finish](bool ok, std::vector&& vz) { + join->v_ok = ok; join->vz = std::move(vz); join->v_done = true; finish(); }); - webReadRangesAsync(sid, isec, i_byte_ranges, - [join, finish](bool ok, std::vector&& ibytes) { - join->i_ok = ok; - join->idx.resize(ibytes.size() / sizeof(std::uint32_t)); - if (!join->idx.empty()) - std::memcpy(join->idx.data(), ibytes.data(), - join->idx.size() * sizeof(std::uint32_t)); - join->i_done = true; - finish(); + webReadRangesAsync(sid, geom, {{i_comp_off, i_comp_size}}, + [join, finish](bool ok, std::vector&& iz) { + join->i_ok = ok; join->iz = std::move(iz); join->i_done = true; finish(); }); } @@ -3481,80 +3456,76 @@ void ViewportCore::loadSidecarMetadataWeb(int source_id, std::string source_labe return; } + // Head (v16): [header 12][geom_bytes 8]. The two compressed metadata blocks + // follow the compressed geometry at SIDECAR_HEAD_BYTES + geom_bytes. webReadRangesAsync(source_id, 0, {{0, SIDECAR_HEAD_BYTES}}, [this, fsize, source_id, source_label](bool ok, std::vector&& head) { - std::uint32_t nvb = 0; - if (!ok || !parseSidecarHead(head.data(), head.size(), nvb)) { - Log::warn() << "loadSidecarMetadataWeb: bad sidecar head"; + std::uint64_t geom_bytes = 0; + if (!ok || !parseSidecarHead(head.data(), head.size(), geom_bytes)) { + Log::warn() << "loadSidecarMetadataWeb: bad sidecar head (wrong version?)"; return; } - const std::uint64_t vsec = SIDECAR_HEAD_BYTES; - const std::uint64_t idx_count_off = std::uint64_t(SIDECAR_HEAD_BYTES) + nvb; - - webReadRangesAsync(source_id, 0, {{idx_count_off, 4}}, - [this, fsize, nvb, vsec, idx_count_off, source_id, source_label] - (bool ok2, std::vector&& cnt) { - if (!ok2 || cnt.size() < 4) { - Log::warn() << "loadSidecarMetadataWeb: short index count"; - return; - } - std::uint32_t num_indices = 0; - std::memcpy(&num_indices, cnt.data(), 4); - const std::uint64_t isec = idx_count_off + 4; - const std::uint64_t crit_size_off = isec + std::uint64_t(num_indices) * 4u; - if (double(crit_size_off + 8) > fsize) { - Log::warn() << "loadSidecarMetadataWeb: critical size past EOF"; - return; - } - // v15: read the 8-byte critical-block length, then the - // render-critical metadata only. The deferred block - // (elements/strings) is fetched on demand later — first - // paint no longer waits on the property tree. - webReadRangesAsync(source_id, 0, {{crit_size_off, 8}}, - [this, vsec, nvb, isec, num_indices, source_id, source_label, - crit_size_off, fsize] - (bool okc, std::vector&& cb) { - if (!okc || cb.size() < 8) { - Log::warn() << "loadSidecarMetadataWeb: short critical size"; + const std::uint64_t meta_off = std::uint64_t(SIDECAR_HEAD_BYTES) + geom_bytes; + if (double(meta_off + 16) > fsize) { + Log::warn() << "loadSidecarMetadataWeb: metadata past EOF"; + return; + } + // Critical block on disk: [comp u64][raw u64][zstd frame]. + webReadRangesAsync(source_id, 0, {{meta_off, 16}}, + [this, fsize, meta_off, source_id, source_label] + (bool ok2, std::vector&& h) { + if (!ok2 || h.size() < 16) { Log::warn() << "loadSidecarMetadataWeb: short crit header"; return; } + std::uint64_t crit_comp = 0, crit_raw = 0; + std::memcpy(&crit_comp, h.data(), 8); + std::memcpy(&crit_raw, h.data() + 8, 8); + const std::uint64_t crit_off = meta_off + 16; + if (double(crit_off + crit_comp + 16) > fsize) { Log::warn() << "loadSidecarMetadataWeb: crit past EOF"; return; } + webReadRangesAsync(source_id, 0, {{crit_off, crit_comp}}, + [this, crit_off, crit_comp, crit_raw, source_id, source_label] + (bool ok3, std::vector&& cz) { + if (!ok3) { Log::warn() << "loadSidecarMetadataWeb: critical read failed"; return; } + std::vector crit(static_cast(crit_raw)); + if (!SidecarCompress::decompress(cz.data(), cz.size(), crit.data(), crit.size())) { + Log::warn() << "loadSidecarMetadataWeb: critical decompress failed"; return; } - std::uint64_t crit_bytes = 0; - std::memcpy(&crit_bytes, cb.data(), 8); - const std::uint64_t crit_off = crit_size_off + 8; - if (double(crit_off + crit_bytes) > fsize) { - Log::warn() << "loadSidecarMetadataWeb: critical block past EOF"; + StreamingSidecar sc; + sc.file_path = source_label; + sc.geometry_section_offset = SIDECAR_HEAD_BYTES; + if (!parseSidecarCritical(crit.data(), crit.size(), sc.meta)) { + Log::warn() << "loadSidecarMetadataWeb: bad critical metadata"; return; } - webReadRangesAsync(source_id, 0, {{crit_off, crit_bytes}}, - [this, vsec, nvb, isec, num_indices, source_id, source_label, - crit_off, crit_bytes, fsize] - (bool ok3, std::vector&& crit) { - if (!ok3) { - Log::warn() << "loadSidecarMetadataWeb: critical read failed"; - return; - } - StreamingSidecar sc; - sc.file_path = source_label; - sc.vertex_section_offset = vsec; - sc.vertex_total_bytes = nvb; - sc.index_section_offset = isec; - sc.index_total_count = num_indices; - if (!parseSidecarCritical(crit.data(), crit.size(), sc.meta)) { - Log::warn() << "loadSidecarMetadataWeb: bad critical metadata"; - return; - } - const std::size_t n_meshes = sc.meta.meshes.size(); - const std::size_t n_instances = sc.meta.instances.size(); - const std::uint32_t mid = next_model_id_++; - applyCachedModel(mid, std::move(sc)); + const std::size_t n_meshes = sc.meta.meshes.size(); + const std::size_t n_instances = sc.meta.instances.size(); + const std::uint32_t mid = next_model_id_++; + applyCachedModel(mid, std::move(sc)); + // Mark web-streamed + set the source IMMEDIATELY — the + // model now has non-resident chunks and the RAF loop's + // driveStreamingLoads will run before the deferred-header + // read below returns. If streaming_from_web weren't set + // yet it would take the sync fopen path and fail + // ("failed to read/decompress chunk 0"). + if (auto m0 = models_gpu_.find(mid); m0 != models_gpu_.end()) { + m0->second.streaming_from_web = true; + m0->second.web_source_id = source_id; + } + // Read the deferred block header to record its locator + // (the property block is fetched on demand later). + const std::uint64_t def_hdr_off = crit_off + crit_comp; + webReadRangesAsync(source_id, 0, {{def_hdr_off, 16}}, + [this, mid, def_hdr_off, source_id, source_label, n_meshes, n_instances] + (bool ok4, std::vector&& dh) { auto mit = models_gpu_.find(mid); if (mit != models_gpu_.end()) { - mit->second.streaming_from_web = true; - mit->second.web_source_id = source_id; - // Deferred block: [end of critical, EOF). - mit->second.deferred_meta_offset = crit_off + crit_bytes; - mit->second.deferred_meta_bytes = - std::uint64_t(fsize) - (crit_off + crit_bytes); + if (ok4 && dh.size() >= 16) { + std::uint64_t dc = 0, dr = 0; + std::memcpy(&dc, dh.data(), 8); + std::memcpy(&dr, dh.data() + 8, 8); + mit->second.deferred_comp_offset = def_hdr_off + 16; + mit->second.deferred_comp_size = dc; + mit->second.deferred_raw_size = dr; + } } viewAll(); host_->requestFrame(); @@ -3577,18 +3548,22 @@ void ViewportCore::loadDeferredMetadataWeb(std::uint32_t model_id, auto it = models_gpu_.find(model_id); if (it == models_gpu_.end()) { if (done) done(false); return; } ModelGpuData& m = it->second; - if (m.deferred_meta_loaded || m.deferred_meta_bytes == 0) { + if (m.deferred_meta_loaded || m.deferred_comp_size == 0) { m.deferred_meta_loaded = true; if (done) done(true); return; } - webReadRangesAsync(m.web_source_id, 0, {{m.deferred_meta_offset, m.deferred_meta_bytes}}, - [this, model_id, done](bool ok, std::vector&& buf) { + const std::uint64_t raw_size = m.deferred_raw_size; + webReadRangesAsync(m.web_source_id, 0, {{m.deferred_comp_offset, m.deferred_comp_size}}, + [this, model_id, raw_size, done](bool ok, std::vector&& cz) { auto mit = models_gpu_.find(model_id); if (mit == models_gpu_.end()) { if (done) done(false); return; } + std::vector buf(static_cast(raw_size)); SidecarData tmp; - if (!ok || !parseSidecarDeferred(buf.data(), buf.size(), tmp)) { - Log::warn() << "loadDeferredMetadataWeb: read/parse failed"; + if (!ok || + !SidecarCompress::decompress(cz.data(), cz.size(), buf.data(), buf.size()) || + !parseSidecarDeferred(buf.data(), buf.size(), tmp)) { + Log::warn() << "loadDeferredMetadataWeb: read/decompress/parse failed"; if (done) done(false); return; } @@ -3678,8 +3653,11 @@ void ViewportCore::streamingByteProgress(std::uint64_t& total_bytes, for (const auto& [mid, m] : models_gpu_) { if (m.hidden) continue; for (const auto& c : m.chunks) { - const std::uint64_t bytes = - c.vertex_byte_size + c.index_count * sizeof(std::uint32_t); + // Report COMPRESSED bytes — what actually crosses the network. Fall + // back to raw for direct (in-memory) loads that have no blobs. + const std::uint64_t bytes = (c.v_comp_size + c.i_comp_size > 0) + ? c.v_comp_size + c.i_comp_size + : c.vertex_byte_size + c.index_count * sizeof(std::uint32_t); total_bytes += bytes; if (c.contribution_visible_count > 0) { needed_bytes += bytes; @@ -3719,10 +3697,9 @@ void ViewportCore::finalizeModel(std::uint32_t model_id) { // applyStreamedChunk loop below). StreamingSidecar metadata; metadata.meta = std::move(s); - metadata.vertex_section_offset = 0; - metadata.vertex_total_bytes = metadata.meta.vertices.size(); - metadata.index_section_offset = 0; - metadata.index_total_count = metadata.meta.indices.size(); + // Direct load: geometry is already in memory (uploaded below), streamed + // from nothing — leave file_path empty so the streaming worker skips it. + metadata.geometry_section_offset = 0; metadata.file_path.clear(); std::vector raw_vertices = std::move(metadata.meta.vertices); diff --git a/src/ifcviewer/tests/CMakeLists.txt b/src/ifcviewer/tests/CMakeLists.txt index bc85395525..cf839e2c10 100644 --- a/src/ifcviewer/tests/CMakeLists.txt +++ b/src/ifcviewer/tests/CMakeLists.txt @@ -46,8 +46,18 @@ if(WITH_MESH_OPTIMIZER) target_compile_definitions(test_lod_builder PRIVATE -DWITH_MESH_OPTIMIZER) endif() +# SidecarCompress: zstd compress/decompress wrappers. Links the system libzstd. +# SidecarCache now (de)compresses geometry + metadata, so every test that +# compiles SidecarCache.cpp needs SidecarCompress.cpp + libzstd too. +find_library(ZSTD_LIBRARY NAMES zstd libzstd) +add_ifcviewer_unit_test(test_sidecar_compress + SOURCES ${IFCVIEWER_SRC}/SidecarCompress.cpp + LIBS ${ZSTD_LIBRARY} +) + add_ifcviewer_unit_test(test_sidecar_cache - SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp + SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp ${IFCVIEWER_SRC}/SidecarCompress.cpp + LIBS ${ZSTD_LIBRARY} ) # StreamingLoader: metadata-only read + range readers + the pure buffer-based @@ -57,6 +67,8 @@ add_ifcviewer_unit_test(test_streaming_loader SOURCES ${IFCVIEWER_SRC}/StreamingLoader.cpp ${IFCVIEWER_SRC}/SidecarCache.cpp + ${IFCVIEWER_SRC}/SidecarCompress.cpp + LIBS ${ZSTD_LIBRARY} ) add_ifcviewer_unit_test(test_instanced_geometry) @@ -74,6 +86,8 @@ add_ifcviewer_unit_test(test_sidecar_layout ${IFCVIEWER_SRC}/SidecarLayout.cpp ${IFCVIEWER_SRC}/ChunkPlanner.cpp ${IFCVIEWER_SRC}/SidecarCache.cpp + ${IFCVIEWER_SRC}/SidecarCompress.cpp + LIBS ${ZSTD_LIBRARY} ) # InstanceCompose: matrix composition + cross-model object_id lookup. diff --git a/src/ifcviewer/tests/test_sidecar_cache.cpp b/src/ifcviewer/tests/test_sidecar_cache.cpp index 71c9fb5362..09d2c1b2e7 100644 --- a/src/ifcviewer/tests/test_sidecar_cache.cpp +++ b/src/ifcviewer/tests/test_sidecar_cache.cpp @@ -118,6 +118,9 @@ SidecarData buildFixture() { e.name_offset = 1; e.name_length = 4; // "Wall" e.type_offset = 6; e.type_length = 4; // "Slab" } + // v16 stores geometry per-chunk (compressed), so a fixture with geometry + // needs a chunk TOC covering its meshes for write/read to round-trip. + sd.chunks = { {0, 2} }; return sd; } @@ -155,8 +158,8 @@ bool sidecarDataEqual(const SidecarData& a, const SidecarData& b) { TEST_CASE("MeshInfo and InstanceCpu have stable layouts (sidecar wire format)", "[sidecar]") { REQUIRE(sizeof(MeshInfo) == 56); REQUIRE(sizeof(InstanceGpu) == 80); - REQUIRE(SIDECAR_VERSION == 15); - REQUIRE(sizeof(SidecarChunk) == 8); + REQUIRE(SIDECAR_VERSION == 16); + REQUIRE(sizeof(SidecarChunk) == 56); REQUIRE(SIDECAR_MAGIC == 0x49465657u); } diff --git a/src/ifcviewer/tests/test_sidecar_compress.cpp b/src/ifcviewer/tests/test_sidecar_compress.cpp new file mode 100644 index 0000000000..5dbc06f4b0 --- /dev/null +++ b/src/ifcviewer/tests/test_sidecar_compress.cpp @@ -0,0 +1,68 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#include "SidecarCompress.h" + +#include + +#include +#include + +TEST_CASE("zstd round-trips arbitrary bytes", "[compress]") { + // Structured data like the sidecar carries (repeated matrices, patterned + // indices) — should both round-trip AND actually shrink. + std::vector raw; + for (int i = 0; i < 20000; ++i) { + raw.push_back(std::uint8_t(i & 0xFF)); + raw.push_back(std::uint8_t((i >> 8) & 0x07)); // low-entropy high byte + raw.push_back(0); + raw.push_back(0xAA); + } + + auto packed = SidecarCompress::compress(raw.data(), raw.size(), 19); + REQUIRE_FALSE(packed.empty()); + REQUIRE(packed.size() < raw.size()); // it compressed + + std::vector out(raw.size()); + REQUIRE(SidecarCompress::decompress(packed.data(), packed.size(), + out.data(), out.size())); + REQUIRE(out == raw); +} + +TEST_CASE("decompress rejects a wrong raw size / garbage", "[compress]") { + std::vector raw(1024, 0x42); + auto packed = SidecarCompress::compress(raw.data(), raw.size(), 3); + REQUIRE_FALSE(packed.empty()); + + // Wrong declared raw size must fail, not silently truncate. + std::vector too_small(512); + REQUIRE_FALSE(SidecarCompress::decompress(packed.data(), packed.size(), + too_small.data(), too_small.size())); + + // Garbage input fails cleanly. + std::vector junk = { 1, 2, 3, 4, 5, 6, 7, 8 }; + std::vector dst(1024); + REQUIRE_FALSE(SidecarCompress::decompress(junk.data(), junk.size(), + dst.data(), dst.size())); +} + +TEST_CASE("empty round-trips to empty", "[compress]") { + std::vector dst; + REQUIRE(SidecarCompress::decompress(nullptr, 0, dst.data(), 0)); +} diff --git a/src/ifcviewer/tests/test_streaming_loader.cpp b/src/ifcviewer/tests/test_streaming_loader.cpp index 6a2087fc2b..4f031aa4ad 100644 --- a/src/ifcviewer/tests/test_streaming_loader.cpp +++ b/src/ifcviewer/tests/test_streaming_loader.cpp @@ -18,6 +18,7 @@ ********************************************************************************/ #include "SidecarCache.h" +#include "SidecarCompress.h" #include "StreamingLoader.h" #include @@ -85,12 +86,15 @@ SidecarData buildFixture() { sd.elements[i].ifc_id = int32_t(1000 + i); sd.elements[i].parent_id = (i == 0) ? -1 : int32_t(100); } + // v16 stores geometry per-chunk (compressed); a fixture with geometry needs + // a chunk TOC covering its meshes (one chunk per mesh here). + sd.chunks = { {0, 1}, {1, 1} }; return sd; } } // namespace -TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bulk", +TEST_CASE("readSidecarMetadataOnly returns metadata, skips bulk geometry", "[streaming]") { fs::path dir = makeScratchDir("metaonly"); fs::path ifc = dir / "model.ifc"; @@ -100,19 +104,19 @@ TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bul auto meta = readSidecarMetadataOnly(ifc.string()); REQUIRE(meta.has_value()); - // Bulk sections are skipped, not loaded. + // Bulk geometry is skipped, not loaded. REQUIRE(meta->meta.vertices.empty()); REQUIRE(meta->meta.indices.empty()); - // Offsets locate the two skipped sections. The vertex section starts - // right after the 16-byte head. - REQUIRE(meta->vertex_section_offset == SIDECAR_HEAD_BYTES); - REQUIRE(meta->vertex_total_bytes == sd.vertices.size()); - REQUIRE(meta->index_total_count == sd.indices.size()); - REQUIRE(meta->index_section_offset == - SIDECAR_HEAD_BYTES + sd.vertices.size() + 4); + // v16: the compressed geometry section starts right after the 20-byte head. + REQUIRE(meta->geometry_section_offset == SIDECAR_HEAD_BYTES); + // Chunk TOC carries compressed blob locators for each chunk. + REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); + REQUIRE(meta->meta.chunks[0].v_comp_size > 0); + // The deferred (property) block locator is recorded for on-demand fetch. + REQUIRE(meta->deferred_comp_size > 0); - // Tail metadata round-trips. + // Metadata round-trips. REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); REQUIRE(meta->meta.instances.size() == sd.instances.size()); REQUIRE(meta->meta.elements.size() == sd.elements.size()); @@ -140,100 +144,90 @@ TEST_CASE("readSidecarMetadataOnly rejects missing / corrupt files", "[streaming REQUIRE_FALSE(readSidecarMetadataOnly(bad.string()).has_value()); } -TEST_CASE("readSidecarVertexRanges scatters byte ranges in input order", "[streaming]") { - fs::path dir = makeScratchDir("vranges"); +TEST_CASE("readChunkGeometryCompressed decompresses a chunk's blobs", "[streaming]") { + fs::path dir = makeScratchDir("chunkgeom"); fs::path ifc = dir / "model.ifc"; SidecarData sd = buildFixture(); REQUIRE(writeSidecar(ifc.string(), sd)); auto meta = readSidecarMetadataOnly(ifc.string()); REQUIRE(meta.has_value()); + REQUIRE(meta->meta.chunks.size() == 2); - // Two section-relative ranges given out of file order; the destination - // must preserve input order (second mesh's bytes first, then first). + // Chunk 0 = mesh 0: vertices [0, 2*stride), indices {0,1,2}. + const auto& c0 = meta->meta.chunks[0]; const uint64_t stride = INSTANCED_VERTEX_STRIDE_BYTES; - std::vector> ranges = { - {2 * stride, 2 * stride}, // last 2 vertices - {0, 2 * stride}, // first 2 vertices - }; - std::vector out; - REQUIRE(readSidecarVertexRanges(ifc.string(), meta->vertex_section_offset, - ranges, out)); - REQUIRE(out.size() == 4 * stride); - REQUIRE(std::memcmp(out.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0); - REQUIRE(std::memcmp(out.data() + 2 * stride, sd.vertices.data(), 2 * stride) == 0); + std::vector vbytes; + std::vector idx; + REQUIRE(readChunkGeometryCompressed( + ifc.string(), meta->geometry_section_offset, + c0.v_comp_off, c0.v_comp_size, c0.v_raw_size, + c0.i_comp_off, c0.i_comp_size, c0.i_raw_size, vbytes, idx)); + REQUIRE(vbytes.size() == 2 * stride); + REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data(), 2 * stride) == 0); + REQUIRE(idx == std::vector({0, 1, 2})); + + // Chunk 1 = mesh 1: indices {1,2,3}. + const auto& c1 = meta->meta.chunks[1]; + REQUIRE(readChunkGeometryCompressed( + ifc.string(), meta->geometry_section_offset, + c1.v_comp_off, c1.v_comp_size, c1.v_raw_size, + c1.i_comp_off, c1.i_comp_size, c1.i_raw_size, vbytes, idx)); + REQUIRE(idx == std::vector({1, 2, 3})); + REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0); } -TEST_CASE("readSidecarIndexRanges reads u32 index ranges", "[streaming]") { - fs::path dir = makeScratchDir("iranges"); - fs::path ifc = dir / "model.ifc"; - SidecarData sd = buildFixture(); - REQUIRE(writeSidecar(ifc.string(), sd)); - auto meta = readSidecarMetadataOnly(ifc.string()); - REQUIRE(meta.has_value()); - - std::vector> ranges = {{3, 3}}; // indices[3..6) - std::vector out; - REQUIRE(readSidecarIndexRanges(ifc.string(), meta->index_section_offset, - ranges, out)); - REQUIRE(out == std::vector({1, 2, 3})); -} - -TEST_CASE("parseSidecarHead validates magic / version / length", "[streaming]") { +TEST_CASE("parseSidecarHead validates magic / version, reads geom length", "[streaming]") { uint8_t head[SIDECAR_HEAD_BYTES] = {}; uint32_t magic = SIDECAR_MAGIC, version = SIDECAR_VERSION, endian = SIDECAR_ENDIAN; - uint32_t nvb = 4096; + uint64_t geom = 123456; std::memcpy(head + 0, &magic, 4); std::memcpy(head + 4, &version, 4); std::memcpy(head + 8, &endian, 4); - std::memcpy(head + 12, &nvb, 4); + std::memcpy(head + 12, &geom, 8); - uint32_t got = 0; + uint64_t got = 0; REQUIRE(parseSidecarHead(head, sizeof(head), got)); - REQUIRE(got == 4096); + REQUIRE(got == 123456); - // Short buffer. REQUIRE_FALSE(parseSidecarHead(head, SIDECAR_HEAD_BYTES - 1, got)); - // Wrong magic. uint8_t bad[SIDECAR_HEAD_BYTES]; std::memcpy(bad, head, sizeof(bad)); bad[0] ^= 0xFF; REQUIRE_FALSE(parseSidecarHead(bad, sizeof(bad), got)); } -TEST_CASE("v15 critical/deferred metadata split round-trips + rejects truncation", - "[streaming]") { - fs::path dir = makeScratchDir("v15split"); +TEST_CASE("v16 deferred block: fetch via locator, decompress, parse", "[streaming]") { + fs::path dir = makeScratchDir("v16def"); fs::path ifc = dir / "model.ifc"; SidecarData sd = buildFixture(); - sd.chunks = { {0, 1}, {1, 1} }; // a TOC, so the critical block carries chunks REQUIRE(writeSidecar(ifc.string(), sd)); - // readSidecarMetadataOnly (desktop) reads BOTH blocks + records the locator. auto meta = readSidecarMetadataOnly(ifc.string()); REQUIRE(meta.has_value()); - REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical - REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); // critical - REQUIRE(meta->meta.elements.size() == sd.elements.size()); // deferred - REQUIRE(meta->meta.string_table == sd.string_table); // deferred - REQUIRE(meta->critical_meta_bytes > 0); + REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical + REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); + REQUIRE(meta->meta.elements.size() == sd.elements.size()); // desktop reads deferred too + REQUIRE(meta->deferred_comp_size > 0); - // Pull the raw critical block via the recorded locator and parse it alone — - // exactly what the web loader does before painting. + // The on-demand path (web) fetches the compressed deferred frame via the + // recorded locator and decompresses it — verify that round-trips. FILE* f = std::fopen((dir / "model.ifcview").string().c_str(), "rb"); REQUIRE(f); - std::vector crit(size_t(meta->critical_meta_bytes)); - std::fseek(f, long(meta->critical_meta_offset), SEEK_SET); - REQUIRE(std::fread(crit.data(), 1, crit.size(), f) == crit.size()); + std::vector cz(size_t(meta->deferred_comp_size)); + std::fseek(f, long(meta->deferred_comp_offset), SEEK_SET); + REQUIRE(std::fread(cz.data(), 1, cz.size(), f) == cz.size()); std::fclose(f); - SidecarData c; - REQUIRE(parseSidecarCritical(crit.data(), crit.size(), c)); - REQUIRE(c.meshes.size() == sd.meshes.size()); - REQUIRE(c.chunks.size() == sd.chunks.size()); - REQUIRE(c.elements.empty()); // the critical block has no property data + std::vector raw(size_t(meta->deferred_raw_size)); + REQUIRE(SidecarCompress::decompress(cz.data(), cz.size(), raw.data(), raw.size())); + SidecarData d; + REQUIRE(parseSidecarDeferred(raw.data(), raw.size(), d)); + REQUIRE(d.elements.size() == sd.elements.size()); + REQUIRE(d.string_table == sd.string_table); + SidecarData chopped; - REQUIRE_FALSE(parseSidecarCritical(crit.data(), crit.size() - 1, chopped)); + REQUIRE_FALSE(parseSidecarDeferred(raw.data(), raw.size() - 1, chopped)); } TEST_CASE("planSidecarReadRanges coalesces adjacent ranges, keeps far ones split",