Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-14 03:14:23 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcviewer: v16 zstd-compressed sidecars (~10x smaller over the wire)

Browse Source 
The .ifcview data is hugely redundant (repeated double instance matrices,
patterned indices) — measured 12x zstd whole-file. Server Content-Encoding
can't be used (it breaks HTTP Range), so compress PER-CHUNK into the format.

Format (v16): geometry becomes per-chunk zstd(vertices)+zstd(indices) frames —
each independently Range-fetchable, so streaming is intact — and the critical +
deferred metadata blocks are single zstd frames. SidecarChunk carries the
compressed blob offsets/sizes; applyStreamedChunk (render/upload) is UNCHANGED —
decompression slots into the fetch. Full readSidecar (test/tooling) reconstructs
by decompress+scatter. zstd: desktop links libzstd (also compresses at bake);
the web build (Emscripten has no zstd port) FetchContent's the pinned zstd
source and compiles its decompress-only subset for wasm — no vendored blob,
same version as desktop. New SidecarCompress wraps it (compress guarded off
under Emscripten). Both stream paths — desktop StreamingThread worker + sync
fallback (readChunkGeometryCompressed) and web beginWebChunkLoad — decompress;
readSidecarMetadataOnly / the web bootstrap / loadDeferredMetadataWeb decompress
the metadata blocks. streamingByteProgress reports COMPRESSED bytes. MEASURED: a
752 MB v15 federation → 75 MB v16 (10x; per-file 6.7-15.3x); PP-PLP 118→15 MB,
loads 13/13 chunks on web, 0 errors.

Three fixes found while testing big federations on a real server:
- Web-streamed race: streaming_from_web was set in the deferred-header callback
  (a round-trip after the model+chunks exist), so driveStreamingLoads could take
  the sync fopen path meanwhile → "failed to read/decompress chunk 0". Now set
  immediately after applyCachedModel.
- OOM abort on 18 models: the pool grew unbounded until an alloc failed, but on
  web that's an uncatchable bad_alloc abort. Cap total pool capacity
  (setMaxTotalCapacity, 3 GB) so it stops before the heap ceiling, and raise
  MAXIMUM_MEMORY 2→4 GB (wasm32 max) for headroom.
- Web never evicted (grow-or-block only). At the hard budget, fall through to the
  LRU/priority evictor so a big federation stays navigable (highest-contribution
  chunks win) instead of freezing with holes.

113/113 desktop + 6/6 web smoke pass. No back-compat: regenerate sidecars
(desktop bakes v16; scratch conv tool migrates v15→v16).

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dion Moult
2026-07-02 10:24:59 +10:00
parent 5299f6c13c
commit 0b8c787ac0
19 changed files with 862 additions and 442 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcviewer/SidecarCache.cpp
+225 -68
View File
@@ -43,10 +43,70 @@
// char[string_table_bytes]
#include "SidecarCache.h"
#include "SidecarCompress.h"
#include <cstdio>
#include <cstring>
// The baker (writeSidecar) compresses — desktop only; the web build never bakes
// and links a decompress-only zstd. Everything from here to writeSidecar's end
// is guarded off under Emscripten.
#if !defined(__EMSCRIPTEN__)
// zstd level for baking. 19 is near-max ratio; decode speed is level-
// independent and the bake is offline, so favour ratio.
static constexpr int kSidecarZstdLevel = 19;
// --- In-memory serialisation (a block is built in RAM, then compressed) ------
template<typename T>
static void appendVec(std::vector<std::uint8_t>& b, const std::vector<T>& v) {
std::uint32_t n = static_cast<std::uint32_t>(v.size());
const auto* np = reinterpret_cast<const std::uint8_t*>(&n);
b.insert(b.end(), np, np + 4);
if (n > 0) {
const auto* p = reinterpret_cast<const std::uint8_t*>(v.data());
b.insert(b.end(), p, p + std::size_t(sizeof(T)) * n);
}
}
static void appendBytes(std::vector<std::uint8_t>& b, const void* p, std::size_t n) {
const auto* c = static_cast<const std::uint8_t*>(p);
b.insert(b.end(), c, c + n);
}
// Pull one chunk's geometry out of the whole-model vertex/index arrays into the
// chunk-LOCAL layout applyStreamedChunk expects: vertices of its meshes in chunk
// order, then indices as LOD0 (per mesh) followed by LOD1 (per mesh).
static void extractChunkGeometry(const SidecarData& d, const SidecarChunk& c,
std::vector<std::uint8_t>& vbytes,
std::vector<std::uint8_t>& ibytes) {
vbytes.clear();
ibytes.clear();
const std::uint32_t end = c.first_mesh + c.mesh_count;
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
const MeshInfo& m = d.meshes[mi];
const std::size_t voff = m.vbo_byte_offset;
const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES;
if (voff + vn <= d.vertices.size())
vbytes.insert(vbytes.end(), d.vertices.begin() + voff,
d.vertices.begin() + voff + vn);
}
auto appendIdx = [&](std::size_t first_u32, std::size_t count) {
if (first_u32 + count > d.indices.size()) return;
const auto* p = reinterpret_cast<const std::uint8_t*>(d.indices.data() + first_u32);
ibytes.insert(ibytes.end(), p, p + count * sizeof(std::uint32_t));
};
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
const MeshInfo& m = d.meshes[mi];
if (m.index_count) appendIdx(m.ebo_byte_offset / sizeof(std::uint32_t), m.index_count);
}
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
const MeshInfo& m = d.meshes[mi];
if (m.lod1_index_count)
appendIdx(m.lod1_ebo_byte_offset / sizeof(std::uint32_t), m.lod1_index_count);
}
}
#endif // !__EMSCRIPTEN__ (bake-only serialisation helpers)
struct SidecarHeader {
uint32_t magic;
uint32_t version;
@@ -86,100 +146,197 @@ static bool readVec(FILE* f, std::vector<T>& v) {
return true;
}
#if !defined(__EMSCRIPTEN__) // bake path — compresses, desktop only
bool writeSidecar(const std::string& ifc_path, const SidecarData& data) {
std::string path = sidecarPath(ifc_path);
FILE* f = fopen(path.c_str(), "wb");
if (!f) return false;
auto wr = [&](const void* p, std::size_t n) {
return fwrite(p, 1, n, f) == n;
};
auto wrU64 = [&](std::uint64_t v) { return wr(&v, sizeof(v)); };
auto wrBlock = [&](const std::vector<std::uint8_t>& raw) -> bool {
auto z = SidecarCompress::compress(raw.data(), raw.size(), kSidecarZstdLevel);
if (raw.size() > 0 && z.empty()) return false; // compress failed
return wrU64(z.size()) && wrU64(raw.size()) && (z.empty() || wr(z.data(), z.size()));
};
SidecarHeader hdr = { SIDECAR_MAGIC, SIDECAR_VERSION, SIDECAR_ENDIAN };
if (fwrite(&hdr, sizeof(hdr), 1, f) != 1) { fclose(f); return false; }
if (!wr(&hdr, sizeof(hdr))) { fclose(f); return false; }
if (!writeVec(f, data.vertices)) { fclose(f); return false; }
if (!writeVec(f, data.indices)) { fclose(f); return false; }
// --- Geometry section: per-chunk zstd(vertex) + zstd(index) frames -------
// Offsets in the chunk TOC are relative to the geometry section start, so
// the loader range-fetches exactly one chunk without reading anything else.
const long geom_len_pos = ftell(f);
if (!wrU64(0)) { fclose(f); return false; } // geom_bytes placeholder
const long geom_start = ftell(f);
// v15: a render-critical metadata block (meshes, instances, georef, chunk
// TOC) preceded by its byte length, then a deferred block (elements +
// string_table). The length lets the web loader read just the critical
// block before painting and fetch the property data lazily / not at all.
const long crit_len_pos = ftell(f);
uint64_t crit_bytes = 0;
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
const long crit_start = ftell(f);
if (!writeVec(f, data.meshes)) { fclose(f); return false; }
if (!writeVec(f, data.instances)) { fclose(f); return false; }
// v11 georef block (148 B).
if (fwrite(&data.has_coordinate_operation, 4, 1, f) != 1) { fclose(f); return false; }
if (fwrite(data.coordinate_operation_meters,
sizeof(double), 16, f) != 16) { fclose(f); return false; }
if (fwrite(&data.project_length_to_meters,
sizeof(double), 1, f) != 1) { fclose(f); return false; }
if (fwrite(&data.map_unit_to_meters,
sizeof(double), 1, f) != 1) { fclose(f); return false; }
if (!writeVec(f, data.chunks)) { fclose(f); return false; }
// Backpatch the critical-block length.
const long crit_end = ftell(f);
if (crit_start < 0 || crit_end < 0) { fclose(f); return false; }
crit_bytes = uint64_t(crit_end - crit_start);
if (fseek(f, crit_len_pos, SEEK_SET) != 0) { fclose(f); return false; }
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
if (fseek(f, crit_end, SEEK_SET) != 0) { fclose(f); return false; }
// Deferred block: element tree + string table (UI/picking, never rendered).
if (!writeVec(f, data.elements)) { fclose(f); return false; }
uint32_t stbl_len = static_cast<uint32_t>(data.string_table.size());
if (fwrite(&stbl_len, 4, 1, f) != 1) { fclose(f); return false; }
if (stbl_len > 0 && fwrite(data.string_table.data(), 1, stbl_len, f) != stbl_len) {
fclose(f); return false;
std::vector<SidecarChunk> chunks = data.chunks; // fill blob offsets below
std::vector<std::uint8_t> vraw, iraw;
for (auto& c : chunks) {
extractChunkGeometry(data, c, vraw, iraw);
auto vz = SidecarCompress::compress(vraw.data(), vraw.size(), kSidecarZstdLevel);
auto iz = SidecarCompress::compress(iraw.data(), iraw.size(), kSidecarZstdLevel);
if ((vraw.size() && vz.empty()) || (iraw.size() && iz.empty())) { fclose(f); return false; }
c.v_comp_off = std::uint64_t(ftell(f) - geom_start);
c.v_comp_size = vz.size();
c.v_raw_size = vraw.size();
if (!vz.empty() && !wr(vz.data(), vz.size())) { fclose(f); return false; }
c.i_comp_off = std::uint64_t(ftell(f) - geom_start);
c.i_comp_size = iz.size();
c.i_raw_size = iraw.size();
if (!iz.empty() && !wr(iz.data(), iz.size())) { fclose(f); return false; }
}
const long geom_end = ftell(f);
if (geom_start < 0 || geom_end < 0) { fclose(f); return false; }
if (fseek(f, geom_len_pos, SEEK_SET) != 0) { fclose(f); return false; }
if (!wrU64(std::uint64_t(geom_end - geom_start))) { fclose(f); return false; }
if (fseek(f, geom_end, SEEK_SET) != 0) { fclose(f); return false; }
// --- Critical metadata block (zstd): meshes, instances, georef, chunk TOC
std::vector<std::uint8_t> crit;
appendVec(crit, data.meshes);
appendVec(crit, data.instances);
appendBytes(crit, &data.has_coordinate_operation, 4);
appendBytes(crit, data.coordinate_operation_meters, sizeof(double) * 16);
appendBytes(crit, &data.project_length_to_meters, sizeof(double));
appendBytes(crit, &data.map_unit_to_meters, sizeof(double));
appendVec(crit, chunks);
if (!wrBlock(crit)) { fclose(f); return false; }
// --- Deferred metadata block (zstd): element tree + string table ---------
std::vector<std::uint8_t> def;
appendVec(def, data.elements);
std::uint32_t stbl_len = static_cast<std::uint32_t>(data.string_table.size());
appendBytes(def, &stbl_len, 4);
appendBytes(def, data.string_table.data(), stbl_len);
if (!wrBlock(def)) { fclose(f); return false; }
fclose(f);
return true;
}
#endif // !__EMSCRIPTEN__
// Cursor over an in-memory (decompressed) metadata block.
namespace {
struct BufReader {
const std::uint8_t* p;
std::size_t n;
std::size_t pos = 0;
bool take(void* dst, std::size_t k) {
if (pos + k > n) return false;
std::memcpy(dst, p + pos, k);
pos += k;
return true;
}
template <typename T>
bool takeVec(std::vector<T>& v) {
std::uint32_t c = 0;
if (!take(&c, 4)) return false;
if (pos + std::size_t(c) * sizeof(T) > n) return false;
v.resize(c);
if (c) { std::memcpy(v.data(), p + pos, std::size_t(c) * sizeof(T)); pos += std::size_t(c) * sizeof(T); }
return true;
}
};
} // namespace
// Full read: reconstruct the whole SidecarData (test/tooling path — the runtime
// streams via readSidecarMetadataOnly + per-chunk loads and never calls this).
// Decompresses the metadata blocks, then scatters each chunk's decompressed
// geometry back into the whole-model vertex/index arrays using the mesh offsets.
std::optional<SidecarData> readSidecar(const std::string& ifc_path) {
std::string path = sidecarPath(ifc_path);
FILE* f = fopen(path.c_str(), "rb");
if (!f) return std::nullopt;
auto fail = [&]() -> std::optional<SidecarData> { fclose(f); return std::nullopt; };
SidecarHeader hdr;
if (fread(&hdr, sizeof(hdr), 1, f) != 1) return fail();
if (hdr.magic != SIDECAR_MAGIC ||
hdr.version != SIDECAR_VERSION ||
if (hdr.magic != SIDECAR_MAGIC || hdr.version != SIDECAR_VERSION ||
hdr.endian != SIDECAR_ENDIAN) return fail();
SidecarData data;
if (!readVec(f, data.vertices)) return fail();
if (!readVec(f, data.indices)) return fail();
auto rd = [&](void* p, std::size_t k) { return fread(p, 1, k, f) == k; };
auto rdU64 = [&](std::uint64_t& v) { return rd(&v, sizeof(v)); };
// v15 critical-block length (consumed; only the streaming/web readers need
// it for a one-shot range read — here we read sequentially).
uint64_t crit_bytes = 0;
if (fread(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) return fail();
// Critical block: meshes, instances, georef, chunk TOC.
if (!readVec(f, data.meshes)) return fail();
if (!readVec(f, data.instances)) return fail();
if (fread(&data.has_coordinate_operation, 4, 1, f) != 1) return fail();
if (fread(data.coordinate_operation_meters,
sizeof(double), 16, f) != 16) return fail();
if (fread(&data.project_length_to_meters,
sizeof(double), 1, f) != 1) return fail();
if (fread(&data.map_unit_to_meters,
sizeof(double), 1, f) != 1) return fail();
if (!readVec(f, data.chunks)) return fail();
// Deferred block: element tree + string table.
if (!readVec(f, data.elements)) return fail();
uint32_t stbl_len;
if (fread(&stbl_len, 4, 1, f) != 1) return fail();
data.string_table.resize(stbl_len);
if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len)
return fail();
std::uint64_t geom_bytes = 0;
if (!rdU64(geom_bytes)) return fail();
std::vector<std::uint8_t> geom(static_cast<std::size_t>(geom_bytes));
if (geom_bytes && !rd(geom.data(), geom.size())) return fail();
auto readBlock = [&](std::vector<std::uint8_t>& out) -> bool {
std::uint64_t comp = 0, raw = 0;
if (!rdU64(comp) || !rdU64(raw)) return false;
std::vector<std::uint8_t> z(static_cast<std::size_t>(comp));
if (comp && !rd(z.data(), z.size())) return false;
out.assign(std::size_t(raw), 0);
return SidecarCompress::decompress(z.data(), z.size(), out.data(), out.size());
};
std::vector<std::uint8_t> crit, def;
if (!readBlock(crit) || !readBlock(def)) return fail();
fclose(f);
SidecarData data;
BufReader cr{ crit.data(), crit.size() };
if (!cr.takeVec(data.meshes)) return std::nullopt;
if (!cr.takeVec(data.instances)) return std::nullopt;
if (!cr.take(&data.has_coordinate_operation, 4)) return std::nullopt;
if (!cr.take(data.coordinate_operation_meters, sizeof(double) * 16)) return std::nullopt;
if (!cr.take(&data.project_length_to_meters, sizeof(double))) return std::nullopt;
if (!cr.take(&data.map_unit_to_meters, sizeof(double))) return std::nullopt;
if (!cr.takeVec(data.chunks)) return std::nullopt;
BufReader dr{ def.data(), def.size() };
if (!dr.takeVec(data.elements)) return std::nullopt;
std::uint32_t stbl_len = 0;
if (!dr.take(&stbl_len, 4)) return std::nullopt;
data.string_table.resize(stbl_len);
if (stbl_len && !dr.take(data.string_table.data(), stbl_len)) return std::nullopt;
// Reconstruct the whole-model vertex/index arrays from the per-chunk blobs.
std::size_t vsize = 0, isize = 0;
for (const auto& m : data.meshes) {
vsize = std::max<std::size_t>(vsize,
std::size_t(m.vbo_byte_offset) + std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES);
isize = std::max<std::size_t>(isize, m.ebo_byte_offset / sizeof(std::uint32_t) + m.index_count);
if (m.lod1_index_count)
isize = std::max<std::size_t>(isize, m.lod1_ebo_byte_offset / sizeof(std::uint32_t) + m.lod1_index_count);
}
data.vertices.assign(vsize, 0);
data.indices.assign(isize, 0);
for (const auto& c : data.chunks) {
if (c.v_comp_off + c.v_comp_size > geom.size() ||
c.i_comp_off + c.i_comp_size > geom.size()) return std::nullopt;
std::vector<std::uint8_t> vraw(static_cast<std::size_t>(c.v_raw_size));
std::vector<std::uint8_t> iraw(static_cast<std::size_t>(c.i_raw_size));
if (!SidecarCompress::decompress(geom.data() + c.v_comp_off, c.v_comp_size, vraw.data(), vraw.size()) ||
!SidecarCompress::decompress(geom.data() + c.i_comp_off, c.i_comp_size, iraw.data(), iraw.size()))
return std::nullopt;
const auto* iu = reinterpret_cast<const std::uint32_t*>(iraw.data());
std::size_t vcur = 0, icur = 0;
const std::uint32_t end = c.first_mesh + c.mesh_count;
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
const MeshInfo& m = data.meshes[mi];
const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES;
if (vcur + vn <= vraw.size() && m.vbo_byte_offset + vn <= data.vertices.size())
std::memcpy(&data.vertices[m.vbo_byte_offset], vraw.data() + vcur, vn);
vcur += vn;
}
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
const MeshInfo& m = data.meshes[mi];
if (!m.index_count) continue;
if (icur + m.index_count <= iraw.size() / 4)
std::memcpy(&data.indices[m.ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.index_count * 4);
icur += m.index_count;
}
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
const MeshInfo& m = data.meshes[mi];
if (!m.lod1_index_count) continue;
if (icur + m.lod1_index_count <= iraw.size() / 4)
std::memcpy(&data.indices[m.lod1_ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.lod1_index_count * 4);
icur += m.lod1_index_count;
}
}
return data;
}