Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-09 17:31:45 +00:00
Code Issues Packages Projects Releases Wiki Activity

ifcviewer: v15 — defer property metadata off the first-paint path

Browse Source 
First-paint over a network is metadata-bound: the whole post-index metadata
(~10 MB on a 118 MB model) had to download before any geometry. But ~25% of
it — elements + string_table, the IFC element tree (names/GUIDs/hierarchy) —
is used only for UI/picking, never for rendering (ViewportCore never touches
it).

v15 splits the post-index metadata into a render-CRITICAL block (meshes,
instances, georef, chunk TOC) preceded by its byte length, then a DEFERRED
block (elements + string_table). The web loader reads only the critical block
before painting; the deferred block sits at a known, self-describing offset
([critical end, EOF)) and is fetched on demand. Desktop reads both (local).

Web on-demand path is wired and complete (not yet called — no UI consumer):
loadDeferredMetadataWeb(model_id) range-fetches + parses the deferred block
into ModelGpuData.elements/string_table, at most once; the first consumer
will be "show the selected object's name" on pick. No background prefetch —
view-only sessions never download the property data (saves 2.64 MB on this
model).

parseSidecarTail split into parseSidecarCritical + parseSidecarDeferred (pure,
unit-tested); StreamingSidecar gains the critical-block locator. Measured
(118 MB model): critical metadata 10.35 -> 7.71 MB, deferred 2.64 MB off the
path; first paint 10.5 -> 9.6 s @ 24 Mbps. (Instances still dominate the
critical block — the next metadata lever.) Format -> v15, no back-compat;
regenerate sidecars. 111/111 unit + 6/6 web smoke pass.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dion Moult
2026-07-01 08:01:15 +10:00
parent e1be2f208c
commit 41a85a70ba
10 changed files with 214 additions and 75 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcviewer-web/sample.ifcview
BIN
View File
Binary file not shown.
src/ifcviewer/ModelGpuData.h
+13
View File
@@ -34,6 +34,7 @@
#include "InstancedGeometry.h"
#include "BufferPool.h"
#include "ChunkPlanner.h" // WGPU_CHUNK_VERTEX_BYTES_LIMIT (shared with bake)
#include "SidecarCache.h" // PackedElementInfo (deferred property metadata)
// Per-model wgpu state. Mirrors the GL backend's ModelGpuData but with
// wgpu handles. Stage 2 only allocates and uploads the four core buffers;
@@ -287,6 +288,18 @@ struct ModelGpuData {
// instead of the MEMFS sync read.
bool streaming_from_web = false;
// v15 deferred property metadata (web, on-demand). The IFC element tree
// (elements + string_table — names/GUIDs/hierarchy, for UI/picking, never
// rendering) lives in a separate file block fetched only when a consumer
// asks, so first paint doesn't wait on it. Empty until
// loadDeferredMetadataWeb fetches [deferred_meta_offset, +bytes) and parses
// it; deferred_meta_loaded latches so it fetches at most once.
std::vector<PackedElementInfo> elements;
std::string string_table;
uint64_t deferred_meta_offset = 0;
uint64_t deferred_meta_bytes = 0;
bool deferred_meta_loaded = false;
// For each mesh in meshes[], the chunk it lives in plus the chunk-local
// offsets into that chunk's vertex_storage and index_buffer. Populated
// at applyCachedModel time; consumed by cullModelCpuCompute when it
src/ifcviewer/SidecarCache.cpp
+29 -11
View File
@@ -96,9 +96,18 @@ bool writeSidecar(const std::string& ifc_path, const SidecarData& data) {
if (!writeVec(f, data.vertices)) { fclose(f); return false; }
if (!writeVec(f, data.indices)) { fclose(f); return false; }
// v15: a render-critical metadata block (meshes, instances, georef, chunk
// TOC) preceded by its byte length, then a deferred block (elements +
// string_table). The length lets the web loader read just the critical
// block before painting and fetch the property data lazily / not at all.
const long crit_len_pos = ftell(f);
uint64_t crit_bytes = 0;
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
const long crit_start = ftell(f);
if (!writeVec(f, data.meshes)) { fclose(f); return false; }
if (!writeVec(f, data.instances)) { fclose(f); return false; }
// v11 georef block (148 B).
if (fwrite(&data.has_coordinate_operation, 4, 1, f) != 1) { fclose(f); return false; }
if (fwrite(data.coordinate_operation_meters,
@@ -107,18 +116,24 @@ bool writeSidecar(const std::string& ifc_path, const SidecarData& data) {
sizeof(double), 1, f) != 1) { fclose(f); return false; }
if (fwrite(&data.map_unit_to_meters,
sizeof(double), 1, f) != 1) { fclose(f); return false; }
if (!writeVec(f, data.chunks)) { fclose(f); return false; }
// Backpatch the critical-block length.
const long crit_end = ftell(f);
if (crit_start < 0 || crit_end < 0) { fclose(f); return false; }
crit_bytes = uint64_t(crit_end - crit_start);
if (fseek(f, crit_len_pos, SEEK_SET) != 0) { fclose(f); return false; }
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
if (fseek(f, crit_end, SEEK_SET) != 0) { fclose(f); return false; }
// Deferred block: element tree + string table (UI/picking, never rendered).
if (!writeVec(f, data.elements)) { fclose(f); return false; }
uint32_t stbl_len = static_cast<uint32_t>(data.string_table.size());
if (fwrite(&stbl_len, 4, 1, f) != 1) { fclose(f); return false; }
if (stbl_len > 0 && fwrite(data.string_table.data(), 1, stbl_len, f) != stbl_len) {
fclose(f); return false;
}
// v14 chunk TOC.
if (!writeVec(f, data.chunks)) { fclose(f); return false; }
fclose(f);
return true;
}
@@ -139,10 +154,15 @@ std::optional<SidecarData> readSidecar(const std::string& ifc_path) {
SidecarData data;
if (!readVec(f, data.vertices)) return fail();
if (!readVec(f, data.indices)) return fail();
// v15 critical-block length (consumed; only the streaming/web readers need
// it for a one-shot range read — here we read sequentially).
uint64_t crit_bytes = 0;
if (fread(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) return fail();
// Critical block: meshes, instances, georef, chunk TOC.
if (!readVec(f, data.meshes)) return fail();
if (!readVec(f, data.instances)) return fail();
// v11 georef block.
if (fread(&data.has_coordinate_operation, 4, 1, f) != 1) return fail();
if (fread(data.coordinate_operation_meters,
sizeof(double), 16, f) != 16) return fail();
@@ -150,18 +170,16 @@ std::optional<SidecarData> readSidecar(const std::string& ifc_path) {
sizeof(double), 1, f) != 1) return fail();
if (fread(&data.map_unit_to_meters,
sizeof(double), 1, f) != 1) return fail();
if (!readVec(f, data.chunks)) return fail();
// Deferred block: element tree + string table.
if (!readVec(f, data.elements)) return fail();
uint32_t stbl_len;
if (fread(&stbl_len, 4, 1, f) != 1) return fail();
data.string_table.resize(stbl_len);
if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len)
return fail();
// v14 chunk TOC.
if (!readVec(f, data.chunks)) return fail();
fclose(f);
return data;
}
src/ifcviewer/SidecarCache.h
+9 -1
View File
@@ -73,7 +73,15 @@ static constexpr uint32_t SIDECAR_MAGIC = 0x49465657; // "IFVW"
// Morton quantisation isn't bit-identical across toolchains (x86 baker vs
// wasm loader), so it must be baked in. No back-compat: v13 sidecars are
// rejected (regenerate them).
static constexpr uint32_t SIDECAR_VERSION = 14;
// v15 = The post-index metadata is split into a render-CRITICAL block (meshes,
// instances, georef, chunk TOC) followed by a DEFERRED block (elements +
// string_table — the IFC element tree, used for UI/picking, never for
// rendering), with the critical block's byte length written just after
// the index section. The web loader reads only the critical block before
// painting, so first geometry no longer waits on the property data; the
// deferred block is fetched lazily (or skipped where unused). Desktop
// reads both. No back-compat: regenerate sidecars.
static constexpr uint32_t SIDECAR_VERSION = 15;
static constexpr uint32_t SIDECAR_ENDIAN = 0x01020304;
// Chunk table-of-contents entry (v14+). A chunk is a CONTIGUOUS range of
src/ifcviewer/StreamingLoader.cpp
+24 -7
View File
@@ -99,26 +99,28 @@ bool parseSidecarHead(const uint8_t* data, size_t n, uint32_t& out_num_vertex_by
return true;
}
bool parseSidecarTail(const uint8_t* data, size_t n, SidecarData& out) {
bool parseSidecarCritical(const uint8_t* data, size_t n, SidecarData& out) {
// v15 render-critical block: meshes, instances, georef, chunk TOC.
BufCursor c{data, n};
if (!c.takeVec(out.meshes)) return false;
if (!c.takeVec(out.instances)) return false;
// v11 georef block (148 bytes total).
if (!c.take(&out.has_coordinate_operation, 4)) return false;
if (!c.take(out.coordinate_operation_meters, sizeof(double) * 16)) return false;
if (!c.take(&out.project_length_to_meters, sizeof(double))) return false;
if (!c.take(&out.map_unit_to_meters, sizeof(double))) return false;
if (!c.takeVec(out.chunks)) return false;
return true;
}
bool parseSidecarDeferred(const uint8_t* data, size_t n, SidecarData& out) {
// v15 deferred block: element tree + string table (UI/picking, not rendered).
BufCursor c{data, n};
if (!c.takeVec(out.elements)) return false;
uint32_t stbl_len = 0;
if (!c.take(&stbl_len, 4)) return false;
if (stbl_len > c.remaining) return false;
out.string_table.resize(stbl_len);
if (stbl_len > 0 && !c.take(out.string_table.data(), stbl_len)) return false;
// v14 chunk TOC.
if (!c.takeVec(out.chunks)) return false;
return true;
}
@@ -165,7 +167,22 @@ std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_p
return fail();
std::fclose(f);
if (!parseSidecarTail(tail.data(), tail.size(), out.meta)) return std::nullopt;
// Tail (v15) = [critical_meta_bytes (8)][critical block][deferred block].
// Desktop is local, so read both; the web path reads only the critical
// block before painting and the deferred block on demand.
if (tail.size() < sizeof(uint64_t)) return std::nullopt;
uint64_t crit_bytes = 0;
std::memcpy(&crit_bytes, tail.data(), sizeof(crit_bytes));
const size_t crit_off = sizeof(crit_bytes);
if (crit_off + crit_bytes > tail.size()) return std::nullopt;
out.critical_meta_offset = out.index_section_offset
+ uint64_t(num_indices) * 4u + crit_off;
out.critical_meta_bytes = crit_bytes;
if (!parseSidecarCritical(tail.data() + crit_off, size_t(crit_bytes), out.meta))
return std::nullopt;
if (!parseSidecarDeferred(tail.data() + crit_off + crit_bytes,
tail.size() - crit_off - size_t(crit_bytes), out.meta))
return std::nullopt;
return out;
}
src/ifcviewer/StreamingLoader.h
+20 -6
View File
@@ -58,6 +58,14 @@ struct StreamingSidecar {
uint64_t index_section_offset = 0;
uint64_t index_total_count = 0; // u32 indices, NOT bytes
// v15 deferred-metadata locator. The render-critical metadata block starts
// at critical_meta_offset and is critical_meta_bytes long; the deferred
// block (elements + string_table) runs from there to EOF. The web loader
// reads only the critical block before painting and fetches the deferred
// block on demand from [critical_meta_offset + critical_meta_bytes, EOF).
uint64_t critical_meta_offset = 0;
uint64_t critical_meta_bytes = 0;
// Resolved on-disk path so subsequent chunk reads can re-open / seek.
std::string file_path;
};
@@ -87,12 +95,18 @@ inline constexpr std::size_t SIDECAR_HEAD_BYTES = 16;
bool parseSidecarHead(const std::uint8_t* data, std::size_t n,
std::uint32_t& out_num_vertex_bytes);
// Parse the metadata tail (everything after the index section): mesh dict,
// instance dict, georef block, element table, string table. `data` points at
// the first tail byte; `n` is the tail length (read to EOF). Returns false on
// any bounds overrun (truncated buffer), leaving out_meta partially filled.
bool parseSidecarTail(const std::uint8_t* data, std::size_t n,
SidecarData& out_meta);
// Parse the v15 render-CRITICAL metadata block (mesh dict, instance dict,
// georef, chunk TOC) — everything needed to set up + draw the scene. `data`
// points at the first critical byte; `n` is critical_meta_bytes. Returns false
// on any bounds overrun, leaving out_meta partially filled.
bool parseSidecarCritical(const std::uint8_t* data, std::size_t n,
SidecarData& out_meta);
// Parse the v15 DEFERRED metadata block (element table + string table — the
// IFC property tree, used for UI/picking, never for rendering). Fetched on
// demand. `data` points at the first deferred byte; `n` is its length.
bool parseSidecarDeferred(const std::uint8_t* data, std::size_t n,
SidecarData& out_meta);
// A coalesced read plan: a single contiguous source read whose bytes are
// scattered into the destination at the recorded offsets. Merging adjacent
src/ifcviewer/ViewportCore.cpp
+88 -31
View File
@@ -3491,43 +3491,66 @@ void ViewportCore::loadSidecarMetadataWeb(std::string source_label) {
}
std::uint32_t num_indices = 0;
std::memcpy(&num_indices, cnt.data(), 4);
const std::uint64_t isec = idx_count_off + 4;
const std::uint64_t tail_off = isec + std::uint64_t(num_indices) * 4u;
if (double(tail_off) > fsize) {
Log::warn() << "loadSidecarMetadataWeb: tail offset past EOF";
const std::uint64_t isec = idx_count_off + 4;
const std::uint64_t crit_size_off = isec + std::uint64_t(num_indices) * 4u;
if (double(crit_size_off + 8) > fsize) {
Log::warn() << "loadSidecarMetadataWeb: critical size past EOF";
return;
}
const std::uint64_t tail_len = std::uint64_t(fsize) - tail_off;
webReadRangesAsync(0, {{tail_off, tail_len}},
[this, vsec, nvb, isec, num_indices, source_label]
(bool ok3, std::vector<std::uint8_t>&& tail) {
if (!ok3) {
Log::warn() << "loadSidecarMetadataWeb: tail read failed";
// v15: read the 8-byte critical-block length, then the
// render-critical metadata only. The deferred block
// (elements/strings) is fetched on demand later — first
// paint no longer waits on the property tree.
webReadRangesAsync(0, {{crit_size_off, 8}},
[this, vsec, nvb, isec, num_indices, source_label, crit_size_off, fsize]
(bool okc, std::vector<std::uint8_t>&& cb) {
if (!okc || cb.size() < 8) {
Log::warn() << "loadSidecarMetadataWeb: short critical size";
return;
}
StreamingSidecar sc;
sc.file_path = source_label;
sc.vertex_section_offset = vsec;
sc.vertex_total_bytes = nvb;
sc.index_section_offset = isec;
sc.index_total_count = num_indices;
if (!parseSidecarTail(tail.data(), tail.size(), sc.meta)) {
Log::warn() << "loadSidecarMetadataWeb: bad metadata tail";
std::uint64_t crit_bytes = 0;
std::memcpy(&crit_bytes, cb.data(), 8);
const std::uint64_t crit_off = crit_size_off + 8;
if (double(crit_off + crit_bytes) > fsize) {
Log::warn() << "loadSidecarMetadataWeb: critical block past EOF";
return;
}
const std::size_t n_meshes = sc.meta.meshes.size();
const std::size_t n_instances = sc.meta.instances.size();
const std::uint32_t mid = next_model_id_++;
applyCachedModel(mid, std::move(sc));
auto mit = models_gpu_.find(mid);
if (mit != models_gpu_.end())
mit->second.streaming_from_web = true;
viewAll();
host_->requestFrame();
Log::info() << "ifcviewer-web: loaded sidecar (" << source_label
<< ", id " << mid << ", " << n_meshes << " meshes, "
<< n_instances << " instances)";
webReadRangesAsync(0, {{crit_off, crit_bytes}},
[this, vsec, nvb, isec, num_indices, source_label,
crit_off, crit_bytes, fsize]
(bool ok3, std::vector<std::uint8_t>&& crit) {
if (!ok3) {
Log::warn() << "loadSidecarMetadataWeb: critical read failed";
return;
}
StreamingSidecar sc;
sc.file_path = source_label;
sc.vertex_section_offset = vsec;
sc.vertex_total_bytes = nvb;
sc.index_section_offset = isec;
sc.index_total_count = num_indices;
if (!parseSidecarCritical(crit.data(), crit.size(), sc.meta)) {
Log::warn() << "loadSidecarMetadataWeb: bad critical metadata";
return;
}
const std::size_t n_meshes = sc.meta.meshes.size();
const std::size_t n_instances = sc.meta.instances.size();
const std::uint32_t mid = next_model_id_++;
applyCachedModel(mid, std::move(sc));
auto mit = models_gpu_.find(mid);
if (mit != models_gpu_.end()) {
mit->second.streaming_from_web = true;
// Deferred block: [end of critical, EOF).
mit->second.deferred_meta_offset = crit_off + crit_bytes;
mit->second.deferred_meta_bytes =
std::uint64_t(fsize) - (crit_off + crit_bytes);
}
viewAll();
host_->requestFrame();
Log::info() << "ifcviewer-web: loaded sidecar (" << source_label
<< ", id " << mid << ", " << n_meshes << " meshes, "
<< n_instances << " instances)";
});
});
});
});
@@ -3549,6 +3572,40 @@ void ViewportCore::loadSidecarFromUrlWeb(std::string url) {
g_url_ready_core = this;
ifcvBeginUrlSource(url.c_str());
}
void ViewportCore::loadDeferredMetadataWeb(std::uint32_t model_id,
std::function<void(bool)> done) {
// On-demand fetch of the v15 deferred block (element tree + string table)
// for a web-streamed model — the property data a UI needs (tree, selected-
// object name, search) but rendering doesn't. Fetches at most once. Valid
// only for the currently-loaded model, since the JS byte-source
// (Module.__ifcvUrl/File) tracks the latest load.
auto it = models_gpu_.find(model_id);
if (it == models_gpu_.end()) { if (done) done(false); return; }
ModelGpuData& m = it->second;
if (m.deferred_meta_loaded || m.deferred_meta_bytes == 0) {
m.deferred_meta_loaded = true;
if (done) done(true);
return;
}
webReadRangesAsync(0, {{m.deferred_meta_offset, m.deferred_meta_bytes}},
[this, model_id, done](bool ok, std::vector<std::uint8_t>&& buf) {
auto mit = models_gpu_.find(model_id);
if (mit == models_gpu_.end()) { if (done) done(false); return; }
SidecarData tmp;
if (!ok || !parseSidecarDeferred(buf.data(), buf.size(), tmp)) {
Log::warn() << "loadDeferredMetadataWeb: read/parse failed";
if (done) done(false);
return;
}
mit->second.elements = std::move(tmp.elements);
mit->second.string_table = std::move(tmp.string_table);
mit->second.deferred_meta_loaded = true;
Log::info() << "ifcviewer-web: loaded deferred metadata ("
<< mit->second.elements.size() << " elements)";
if (done) done(true);
});
}
#endif // __EMSCRIPTEN__
void ViewportCore::finalizeModel(std::uint32_t model_id) {
src/ifcviewer/ViewportCore.h
+8
View File
@@ -371,6 +371,14 @@ public:
void loadSidecarFromUrlWeb(std::string url);
void loadSidecarMetadataWeb(std::string source_label);
// On-demand fetch of the v15 deferred property block (element tree + string
// table) for a web-streamed model — what a UI (object tree / selected-name
// / search) needs, fetched only when asked so first paint never waits on
// it. Populates ModelGpuData.elements/string_table; fires done(ok). At most
// one fetch per model. Currently unwired (no consumer yet) but complete.
void loadDeferredMetadataWeb(std::uint32_t model_id,
std::function<void(bool)> done = {});
// Kick off the async read of one chunk's vertex + index byte ranges (from
// the active web source). applyStreamedChunk runs in the JS completion
// callback; c.is_loading is held until then. No-op if the model/chunk
src/ifcviewer/tests/test_sidecar_cache.cpp
+1 -1
View File
@@ -155,7 +155,7 @@ bool sidecarDataEqual(const SidecarData& a, const SidecarData& b) {
TEST_CASE("MeshInfo and InstanceCpu have stable layouts (sidecar wire format)", "[sidecar]") {
REQUIRE(sizeof(MeshInfo) == 56);
REQUIRE(sizeof(InstanceGpu) == 80);
REQUIRE(SIDECAR_VERSION == 14);
REQUIRE(SIDECAR_VERSION == 15);
REQUIRE(sizeof(SidecarChunk) == 8);
REQUIRE(SIDECAR_MAGIC == 0x49465657u);
}
src/ifcviewer/tests/test_streaming_loader.cpp
+22 -18
View File
@@ -201,35 +201,39 @@ TEST_CASE("parseSidecarHead validates magic / version / length", "[streaming]")
REQUIRE_FALSE(parseSidecarHead(bad, sizeof(bad), got));
}
TEST_CASE("parseSidecarTail rejects a truncated tail", "[streaming]") {
// A valid full tail, then everything but its last byte must fail.
fs::path dir = makeScratchDir("tailtrunc");
TEST_CASE("v15 critical/deferred metadata split round-trips + rejects truncation",
"[streaming]") {
fs::path dir = makeScratchDir("v15split");
fs::path ifc = dir / "model.ifc";
SidecarData sd = buildFixture();
sd.chunks = { {0, 1}, {1, 1} }; // a TOC, so the critical block carries chunks
REQUIRE(writeSidecar(ifc.string(), sd));
// readSidecarMetadataOnly (desktop) reads BOTH blocks + records the locator.
auto meta = readSidecarMetadataOnly(ifc.string());
REQUIRE(meta.has_value());
REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical
REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); // critical
REQUIRE(meta->meta.elements.size() == sd.elements.size()); // deferred
REQUIRE(meta->meta.string_table == sd.string_table); // deferred
REQUIRE(meta->critical_meta_bytes > 0);
// Re-read the raw tail bytes from disk (offset = index section end).
const uint64_t tail_off =
meta->index_section_offset + meta->index_total_count * 4u;
// Pull the raw critical block via the recorded locator and parse it alone —
// exactly what the web loader does before painting.
FILE* f = std::fopen((dir / "model.ifcview").string().c_str(), "rb");
REQUIRE(f);
std::fseek(f, 0, SEEK_END);
const long end = std::ftell(f);
const size_t tail_len = size_t(end - long(tail_off));
std::vector<uint8_t> tail(tail_len);
std::fseek(f, long(tail_off), SEEK_SET);
REQUIRE(std::fread(tail.data(), 1, tail_len, f) == tail_len);
std::vector<uint8_t> crit(size_t(meta->critical_meta_bytes));
std::fseek(f, long(meta->critical_meta_offset), SEEK_SET);
REQUIRE(std::fread(crit.data(), 1, crit.size(), f) == crit.size());
std::fclose(f);
SidecarData full;
REQUIRE(parseSidecarTail(tail.data(), tail.size(), full));
REQUIRE(full.meshes.size() == sd.meshes.size());
REQUIRE(full.string_table == sd.string_table);
SidecarData c;
REQUIRE(parseSidecarCritical(crit.data(), crit.size(), c));
REQUIRE(c.meshes.size() == sd.meshes.size());
REQUIRE(c.chunks.size() == sd.chunks.size());
REQUIRE(c.elements.empty()); // the critical block has no property data
SidecarData chopped;
REQUIRE_FALSE(parseSidecarTail(tail.data(), tail.size() - 1, chopped));
REQUIRE_FALSE(parseSidecarCritical(crit.data(), crit.size() - 1, chopped));
}
TEST_CASE("planSidecarReadRanges coalesces adjacent ranges, keeps far ones split",