Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-09-16 21:42:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcviewer: v16 zstd-compressed sidecars (~10x smaller over the wire)

Browse Source 
The .ifcview data is hugely redundant (repeated double instance matrices,
patterned indices) — measured 12x zstd whole-file. Server Content-Encoding
can't be used (it breaks HTTP Range), so compress PER-CHUNK into the format.

Format (v16): geometry becomes per-chunk zstd(vertices)+zstd(indices) frames —
each independently Range-fetchable, so streaming is intact — and the critical +
deferred metadata blocks are single zstd frames. SidecarChunk carries the
compressed blob offsets/sizes; applyStreamedChunk (render/upload) is UNCHANGED —
decompression slots into the fetch. Full readSidecar (test/tooling) reconstructs
by decompress+scatter. zstd: desktop links libzstd (also compresses at bake);
the web build (Emscripten has no zstd port) FetchContent's the pinned zstd
source and compiles its decompress-only subset for wasm — no vendored blob,
same version as desktop. New SidecarCompress wraps it (compress guarded off
under Emscripten). Both stream paths — desktop StreamingThread worker + sync
fallback (readChunkGeometryCompressed) and web beginWebChunkLoad — decompress;
readSidecarMetadataOnly / the web bootstrap / loadDeferredMetadataWeb decompress
the metadata blocks. streamingByteProgress reports COMPRESSED bytes. MEASURED: a
752 MB v15 federation → 75 MB v16 (10x; per-file 6.7-15.3x); PP-PLP 118→15 MB,
loads 13/13 chunks on web, 0 errors.

Three fixes found while testing big federations on a real server:
- Web-streamed race: streaming_from_web was set in the deferred-header callback
  (a round-trip after the model+chunks exist), so driveStreamingLoads could take
  the sync fopen path meanwhile → "failed to read/decompress chunk 0". Now set
  immediately after applyCachedModel.
- OOM abort on 18 models: the pool grew unbounded until an alloc failed, but on
  web that's an uncatchable bad_alloc abort. Cap total pool capacity
  (setMaxTotalCapacity, 3 GB) so it stops before the heap ceiling, and raise
  MAXIMUM_MEMORY 2→4 GB (wasm32 max) for headroom.
- Web never evicted (grow-or-block only). At the hard budget, fall through to the
  LRU/priority evictor so a big federation stays navigable (highest-contribution
  chunks win) instead of freezing with holes.

113/113 desktop + 6/6 web smoke pass. No back-compat: regenerate sidecars
(desktop bakes v16; scratch conv tool migrates v15→v16).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dion Moult
2026-07-02 10:24:59 +10:00
parent 5299f6c13c
commit 0b8c787ac0
19 changed files with 862 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcviewer/tests/CMakeLists.txt
+15 -1
View File
@@ -46,8 +46,18 @@ if(WITH_MESH_OPTIMIZER)
target_compile_definitions(test_lod_builder PRIVATE -DWITH_MESH_OPTIMIZER)
endif()
# SidecarCompress: zstd compress/decompress wrappers. Links the system libzstd.
# SidecarCache now (de)compresses geometry + metadata, so every test that
# compiles SidecarCache.cpp needs SidecarCompress.cpp + libzstd too.
find_library(ZSTD_LIBRARY NAMES zstd libzstd)
add_ifcviewer_unit_test(test_sidecar_compress
SOURCES ${IFCVIEWER_SRC}/SidecarCompress.cpp
LIBS ${ZSTD_LIBRARY}
)
add_ifcviewer_unit_test(test_sidecar_cache
SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp
SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp ${IFCVIEWER_SRC}/SidecarCompress.cpp
LIBS ${ZSTD_LIBRARY}
)
# StreamingLoader: metadata-only read + range readers + the pure buffer-based
@@ -57,6 +67,8 @@ add_ifcviewer_unit_test(test_streaming_loader
SOURCES
${IFCVIEWER_SRC}/StreamingLoader.cpp
${IFCVIEWER_SRC}/SidecarCache.cpp
${IFCVIEWER_SRC}/SidecarCompress.cpp
LIBS ${ZSTD_LIBRARY}
)
add_ifcviewer_unit_test(test_instanced_geometry)
@@ -74,6 +86,8 @@ add_ifcviewer_unit_test(test_sidecar_layout
${IFCVIEWER_SRC}/SidecarLayout.cpp
${IFCVIEWER_SRC}/ChunkPlanner.cpp
${IFCVIEWER_SRC}/SidecarCache.cpp
${IFCVIEWER_SRC}/SidecarCompress.cpp
LIBS ${ZSTD_LIBRARY}
)
# InstanceCompose: matrix composition + cross-model object_id lookup.
src/ifcviewer/tests/test_sidecar_cache.cpp
+5 -2
View File
@@ -118,6 +118,9 @@ SidecarData buildFixture() {
e.name_offset = 1; e.name_length = 4; // "Wall"
e.type_offset = 6; e.type_length = 4; // "Slab"
}
// v16 stores geometry per-chunk (compressed), so a fixture with geometry
// needs a chunk TOC covering its meshes for write/read to round-trip.
sd.chunks = { {0, 2} };
return sd;
}
@@ -155,8 +158,8 @@ bool sidecarDataEqual(const SidecarData& a, const SidecarData& b) {
TEST_CASE("MeshInfo and InstanceCpu have stable layouts (sidecar wire format)", "[sidecar]") {
REQUIRE(sizeof(MeshInfo) == 56);
REQUIRE(sizeof(InstanceGpu) == 80);
REQUIRE(SIDECAR_VERSION == 15);
REQUIRE(sizeof(SidecarChunk) == 8);
REQUIRE(SIDECAR_VERSION == 16);
REQUIRE(sizeof(SidecarChunk) == 56);
REQUIRE(SIDECAR_MAGIC == 0x49465657u);
}
src/ifcviewer/tests/test_sidecar_compress.cpp
+68
View File
@@ -0,0 +1,68 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#include "SidecarCompress.h"
#include <catch2/catch_test_macros.hpp>
#include <cstdint>
#include <vector>
TEST_CASE("zstd round-trips arbitrary bytes", "[compress]") {
// Structured data like the sidecar carries (repeated matrices, patterned
// indices) — should both round-trip AND actually shrink.
std::vector<std::uint8_t> raw;
for (int i = 0; i < 20000; ++i) {
raw.push_back(std::uint8_t(i & 0xFF));
raw.push_back(std::uint8_t((i >> 8) & 0x07)); // low-entropy high byte
raw.push_back(0);
raw.push_back(0xAA);
}
auto packed = SidecarCompress::compress(raw.data(), raw.size(), 19);
REQUIRE_FALSE(packed.empty());
REQUIRE(packed.size() < raw.size()); // it compressed
std::vector<std::uint8_t> out(raw.size());
REQUIRE(SidecarCompress::decompress(packed.data(), packed.size(),
out.data(), out.size()));
REQUIRE(out == raw);
}
TEST_CASE("decompress rejects a wrong raw size / garbage", "[compress]") {
std::vector<std::uint8_t> raw(1024, 0x42);
auto packed = SidecarCompress::compress(raw.data(), raw.size(), 3);
REQUIRE_FALSE(packed.empty());
// Wrong declared raw size must fail, not silently truncate.
std::vector<std::uint8_t> too_small(512);
REQUIRE_FALSE(SidecarCompress::decompress(packed.data(), packed.size(),
too_small.data(), too_small.size()));
// Garbage input fails cleanly.
std::vector<std::uint8_t> junk = { 1, 2, 3, 4, 5, 6, 7, 8 };
std::vector<std::uint8_t> dst(1024);
REQUIRE_FALSE(SidecarCompress::decompress(junk.data(), junk.size(),
dst.data(), dst.size()));
}
TEST_CASE("empty round-trips to empty", "[compress]") {
std::vector<std::uint8_t> dst;
REQUIRE(SidecarCompress::decompress(nullptr, 0, dst.data(), 0));
}
src/ifcviewer/tests/test_streaming_loader.cpp
+61 -67
View File
@@ -18,6 +18,7 @@
********************************************************************************/
#include "SidecarCache.h"
#include "SidecarCompress.h"
#include "StreamingLoader.h"
#include <catch2/catch_test_macros.hpp>
@@ -85,12 +86,15 @@ SidecarData buildFixture() {
sd.elements[i].ifc_id = int32_t(1000 + i);
sd.elements[i].parent_id = (i == 0) ? -1 : int32_t(100);
}
// v16 stores geometry per-chunk (compressed); a fixture with geometry needs
// a chunk TOC covering its meshes (one chunk per mesh here).
sd.chunks = { {0, 1}, {1, 1} };
return sd;
}
} // namespace
TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bulk",
TEST_CASE("readSidecarMetadataOnly returns metadata, skips bulk geometry",
"[streaming]") {
fs::path dir = makeScratchDir("metaonly");
fs::path ifc = dir / "model.ifc";
@@ -100,19 +104,19 @@ TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bul
auto meta = readSidecarMetadataOnly(ifc.string());
REQUIRE(meta.has_value());
// Bulk sections are skipped, not loaded.
// Bulk geometry is skipped, not loaded.
REQUIRE(meta->meta.vertices.empty());
REQUIRE(meta->meta.indices.empty());
// Offsets locate the two skipped sections. The vertex section starts
// right after the 16-byte head.
REQUIRE(meta->vertex_section_offset == SIDECAR_HEAD_BYTES);
REQUIRE(meta->vertex_total_bytes == sd.vertices.size());
REQUIRE(meta->index_total_count == sd.indices.size());
REQUIRE(meta->index_section_offset ==
SIDECAR_HEAD_BYTES + sd.vertices.size() + 4);
// v16: the compressed geometry section starts right after the 20-byte head.
REQUIRE(meta->geometry_section_offset == SIDECAR_HEAD_BYTES);
// Chunk TOC carries compressed blob locators for each chunk.
REQUIRE(meta->meta.chunks.size() == sd.chunks.size());
REQUIRE(meta->meta.chunks[0].v_comp_size > 0);
// The deferred (property) block locator is recorded for on-demand fetch.
REQUIRE(meta->deferred_comp_size > 0);
// Tail metadata round-trips.
// Metadata round-trips.
REQUIRE(meta->meta.meshes.size() == sd.meshes.size());
REQUIRE(meta->meta.instances.size() == sd.instances.size());
REQUIRE(meta->meta.elements.size() == sd.elements.size());
@@ -140,100 +144,90 @@ TEST_CASE("readSidecarMetadataOnly rejects missing / corrupt files", "[streaming
REQUIRE_FALSE(readSidecarMetadataOnly(bad.string()).has_value());
}
TEST_CASE("readSidecarVertexRanges scatters byte ranges in input order", "[streaming]") {
fs::path dir = makeScratchDir("vranges");
TEST_CASE("readChunkGeometryCompressed decompresses a chunk's blobs", "[streaming]") {
fs::path dir = makeScratchDir("chunkgeom");
fs::path ifc = dir / "model.ifc";
SidecarData sd = buildFixture();
REQUIRE(writeSidecar(ifc.string(), sd));
auto meta = readSidecarMetadataOnly(ifc.string());
REQUIRE(meta.has_value());
REQUIRE(meta->meta.chunks.size() == 2);
// Two section-relative ranges given out of file order; the destination
// must preserve input order (second mesh's bytes first, then first).
// Chunk 0 = mesh 0: vertices [0, 2*stride), indices {0,1,2}.
const auto& c0 = meta->meta.chunks[0];
const uint64_t stride = INSTANCED_VERTEX_STRIDE_BYTES;
std::vector<std::pair<uint64_t, uint64_t>> ranges = {
{2 * stride, 2 * stride}, // last 2 vertices
{0, 2 * stride}, // first 2 vertices
};
std::vector<uint8_t> out;
REQUIRE(readSidecarVertexRanges(ifc.string(), meta->vertex_section_offset,
ranges, out));
REQUIRE(out.size() == 4 * stride);
REQUIRE(std::memcmp(out.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0);
REQUIRE(std::memcmp(out.data() + 2 * stride, sd.vertices.data(), 2 * stride) == 0);
std::vector<uint8_t> vbytes;
std::vector<uint32_t> idx;
REQUIRE(readChunkGeometryCompressed(
ifc.string(), meta->geometry_section_offset,
c0.v_comp_off, c0.v_comp_size, c0.v_raw_size,
c0.i_comp_off, c0.i_comp_size, c0.i_raw_size, vbytes, idx));
REQUIRE(vbytes.size() == 2 * stride);
REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data(), 2 * stride) == 0);
REQUIRE(idx == std::vector<uint32_t>({0, 1, 2}));
// Chunk 1 = mesh 1: indices {1,2,3}.
const auto& c1 = meta->meta.chunks[1];
REQUIRE(readChunkGeometryCompressed(
ifc.string(), meta->geometry_section_offset,
c1.v_comp_off, c1.v_comp_size, c1.v_raw_size,
c1.i_comp_off, c1.i_comp_size, c1.i_raw_size, vbytes, idx));
REQUIRE(idx == std::vector<uint32_t>({1, 2, 3}));
REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0);
}
TEST_CASE("readSidecarIndexRanges reads u32 index ranges", "[streaming]") {
fs::path dir = makeScratchDir("iranges");
fs::path ifc = dir / "model.ifc";
SidecarData sd = buildFixture();
REQUIRE(writeSidecar(ifc.string(), sd));
auto meta = readSidecarMetadataOnly(ifc.string());
REQUIRE(meta.has_value());
std::vector<std::pair<uint64_t, uint64_t>> ranges = {{3, 3}}; // indices[3..6)
std::vector<uint32_t> out;
REQUIRE(readSidecarIndexRanges(ifc.string(), meta->index_section_offset,
ranges, out));
REQUIRE(out == std::vector<uint32_t>({1, 2, 3}));
}
TEST_CASE("parseSidecarHead validates magic / version / length", "[streaming]") {
TEST_CASE("parseSidecarHead validates magic / version, reads geom length", "[streaming]") {
uint8_t head[SIDECAR_HEAD_BYTES] = {};
uint32_t magic = SIDECAR_MAGIC, version = SIDECAR_VERSION, endian = SIDECAR_ENDIAN;
uint32_t nvb = 4096;
uint64_t geom = 123456;
std::memcpy(head + 0, &magic, 4);
std::memcpy(head + 4, &version, 4);
std::memcpy(head + 8, &endian, 4);
std::memcpy(head + 12, &nvb, 4);
std::memcpy(head + 12, &geom, 8);
uint32_t got = 0;
uint64_t got = 0;
REQUIRE(parseSidecarHead(head, sizeof(head), got));
REQUIRE(got == 4096);
REQUIRE(got == 123456);
// Short buffer.
REQUIRE_FALSE(parseSidecarHead(head, SIDECAR_HEAD_BYTES - 1, got));
// Wrong magic.
uint8_t bad[SIDECAR_HEAD_BYTES];
std::memcpy(bad, head, sizeof(bad));
bad[0] ^= 0xFF;
REQUIRE_FALSE(parseSidecarHead(bad, sizeof(bad), got));
}
TEST_CASE("v15 critical/deferred metadata split round-trips + rejects truncation",
"[streaming]") {
fs::path dir = makeScratchDir("v15split");
TEST_CASE("v16 deferred block: fetch via locator, decompress, parse", "[streaming]") {
fs::path dir = makeScratchDir("v16def");
fs::path ifc = dir / "model.ifc";
SidecarData sd = buildFixture();
sd.chunks = { {0, 1}, {1, 1} }; // a TOC, so the critical block carries chunks
REQUIRE(writeSidecar(ifc.string(), sd));
// readSidecarMetadataOnly (desktop) reads BOTH blocks + records the locator.
auto meta = readSidecarMetadataOnly(ifc.string());
REQUIRE(meta.has_value());
REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical
REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); // critical
REQUIRE(meta->meta.elements.size() == sd.elements.size()); // deferred
REQUIRE(meta->meta.string_table == sd.string_table); // deferred
REQUIRE(meta->critical_meta_bytes > 0);
REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical
REQUIRE(meta->meta.chunks.size() == sd.chunks.size());
REQUIRE(meta->meta.elements.size() == sd.elements.size()); // desktop reads deferred too
REQUIRE(meta->deferred_comp_size > 0);
// Pull the raw critical block via the recorded locator and parse it alone —
// exactly what the web loader does before painting.
// The on-demand path (web) fetches the compressed deferred frame via the
// recorded locator and decompresses it — verify that round-trips.
FILE* f = std::fopen((dir / "model.ifcview").string().c_str(), "rb");
REQUIRE(f);
std::vector<uint8_t> crit(size_t(meta->critical_meta_bytes));
std::fseek(f, long(meta->critical_meta_offset), SEEK_SET);
REQUIRE(std::fread(crit.data(), 1, crit.size(), f) == crit.size());
std::vector<uint8_t> cz(size_t(meta->deferred_comp_size));
std::fseek(f, long(meta->deferred_comp_offset), SEEK_SET);
REQUIRE(std::fread(cz.data(), 1, cz.size(), f) == cz.size());
std::fclose(f);
SidecarData c;
REQUIRE(parseSidecarCritical(crit.data(), crit.size(), c));
REQUIRE(c.meshes.size() == sd.meshes.size());
REQUIRE(c.chunks.size() == sd.chunks.size());
REQUIRE(c.elements.empty()); // the critical block has no property data
std::vector<uint8_t> raw(size_t(meta->deferred_raw_size));
REQUIRE(SidecarCompress::decompress(cz.data(), cz.size(), raw.data(), raw.size()));
SidecarData d;
REQUIRE(parseSidecarDeferred(raw.data(), raw.size(), d));
REQUIRE(d.elements.size() == sd.elements.size());
REQUIRE(d.string_table == sd.string_table);
SidecarData chopped;
REQUIRE_FALSE(parseSidecarCritical(crit.data(), crit.size() - 1, chopped));
REQUIRE_FALSE(parseSidecarDeferred(raw.data(), raw.size() - 1, chopped));
}
TEST_CASE("planSidecarReadRanges coalesces adjacent ranges, keeps far ones split",