From ababb49ae7536f2e60d995f7d99a4948fcf2498a Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Sun, 12 Apr 2026 20:10:20 +1000 Subject: [PATCH] Sidecar v4: persist instanced geometry + metadata Commit B of the instancing migration. The sidecar on-disk format is reintroduced at version 4 with MeshInfo + InstanceCpu sections in place of v3's flat per-object draw-info array. After streaming finishes, MainWindow asks the viewport for a post- finalise snapshot (VBO + EBO are read back from the GPU, meshes and instances come from the CPU-side arrays) and writes it alongside PackedElementInfo + the string table. On a subsequent load, readSidecar rehydrates the whole struct and ViewportWindow:: applyCachedModel uploads VBO/EBO/SSBO in a single step, bypassing the iterator entirely. Staleness check is still by source file size. Co-Authored-By: Claude Opus 4.6 --- src/ifcviewer/MainWindow.cpp | 92 ++++++++++++++++++++++-- src/ifcviewer/SidecarCache.cpp | 118 ++++++++++++++++++++++++++++--- src/ifcviewer/ViewportWindow.cpp | 97 +++++++++++++++++++++++++ src/ifcviewer/ViewportWindow.h | 10 +++ 4 files changed, 300 insertions(+), 17 deletions(-) diff --git a/src/ifcviewer/MainWindow.cpp b/src/ifcviewer/MainWindow.cpp index 86a787a0e2..ceeedc8cbd 100644 --- a/src/ifcviewer/MainWindow.cpp +++ b/src/ifcviewer/MainWindow.cpp @@ -210,7 +210,7 @@ void MainWindow::startNextLoad() { qDebug(" Sidecar read: %lld ms (%s)", rt.elapsed(), ifc_path.c_str()); auto result = std::make_shared>(std::move(cached)); QMetaObject::invokeMethod(this, [this, mid, result]() { - if (*result && !(*result)->meshes.empty()) { + if (*result && !(*result)->instances.empty()) { applySidecarData(mid, std::move(**result)); } else { // No sidecar — fall back to streaming from IFC. @@ -229,10 +229,54 @@ void MainWindow::startNextLoad() { }); } -void MainWindow::applySidecarData(ModelId /*mid*/, SidecarData /*data*/) { - // Commit A: readSidecar() always returns nullopt, so this is unreachable. - // Restored in Commit B along with the v4 on-disk format. - qWarning("applySidecarData called but sidecar is disabled in Commit A"); +void MainWindow::applySidecarData(ModelId mid, SidecarData data) { + auto it = models_.find(mid); + if (it == models_.end()) return; + auto& model = it->second; + + qDebug("Sidecar hit: %s (%zu verts, %zu indices, %zu meshes, %zu instances, %zu elements)", + model.file_path.toStdString().c_str(), + data.vertices.size() / INSTANCED_VERTEX_STRIDE_FLOATS, + data.indices.size(), + data.meshes.size(), + data.instances.size(), + data.elements.size()); + + QElapsedTimer t; + t.start(); + + // Update next_object_id_ past all objects in this model before the + // extracted `elements` is moved out of `data`. + for (const auto& elem : data.elements) { + if (elem.object_id >= next_object_id_) + next_object_id_ = elem.object_id + 1; + } + + // Hand off geometry to GPU in a single call. + std::vector elements = std::move(data.elements); + std::string stbl = std::move(data.string_table); + viewport_->applyCachedModel(mid, std::move(data)); + qDebug(" GL upload: %lld ms", t.elapsed()); + + t.restart(); + element_tree_->setUpdatesEnabled(false); + populateTreeFromSidecar(model, elements, stbl); + element_tree_->setUpdatesEnabled(true); + qDebug(" Tree build: %lld ms (%zu elements)", t.elapsed(), elements.size()); + + progress_bar_->setVisible(false); + + qint64 ms = load_timer_.elapsed(); + QString elapsed = (ms >= 1000) + ? QString::number(ms / 1000.0, 'f', 2) + " s" + : QString::number(ms) + " ms"; + status_label_->setText(QString("%1 elements across %2 model(s) — loaded from cache in %3") + .arg(element_map_.size()) + .arg(models_.size()) + .arg(elapsed)); + + loading_model_id_ = 0; + QTimer::singleShot(0, this, &MainWindow::startNextLoad); } void MainWindow::populateTreeFromSidecar(ModelHandle& model, @@ -320,10 +364,44 @@ void MainWindow::onStreamingFinished() { .arg(num_models) .arg(elapsed)); - // Sort instances by mesh and upload the per-model instance SSBO. - // Sidecar write is stubbed in Commit A. + // Sort instances by mesh, upload the per-model instance SSBO, and + // persist a v4 sidecar for next load. if (loading_model_id_ != 0) { viewport_->finalizeModel(loading_model_id_); + + auto it = models_.find(loading_model_id_); + if (it != models_.end()) { + SidecarData sd; + if (viewport_->snapshotModel(loading_model_id_, sd)) { + // Pack this model's element metadata + string table. + for (const auto& [oid, info] : element_map_) { + if (info.model_id != loading_model_id_) continue; + PackedElementInfo pe; + pe.object_id = info.object_id; + pe.model_id = info.model_id; + pe.ifc_id = info.ifc_id; + pe.parent_id = info.parent_id; + pe.guid_offset = static_cast(sd.string_table.size()); + pe.guid_length = static_cast(info.guid.size()); + sd.string_table += info.guid; + pe.name_offset = static_cast(sd.string_table.size()); + pe.name_length = static_cast(info.name.size()); + sd.string_table += info.name; + pe.type_offset = static_cast(sd.string_table.size()); + pe.type_length = static_cast(info.type.size()); + sd.string_table += info.type; + sd.elements.push_back(pe); + } + + std::string ifc_path = it->second.file_path.toStdString(); + uint64_t file_size = static_cast( + QFileInfo(it->second.file_path).size()); + QElapsedTimer t; t.start(); + bool ok = writeSidecar(ifc_path, sd, file_size); + qDebug(" Sidecar write: %lld ms (%s)", + t.elapsed(), ok ? "ok" : "FAILED"); + } + } } // Start next model if queued. diff --git a/src/ifcviewer/SidecarCache.cpp b/src/ifcviewer/SidecarCache.cpp index be19c8698f..3c5ca9cd8d 100644 --- a/src/ifcviewer/SidecarCache.cpp +++ b/src/ifcviewer/SidecarCache.cpp @@ -17,20 +17,118 @@ * * ********************************************************************************/ -// Commit A: sidecar cache is temporarily disabled. The on-disk format is -// being rewritten from v3 (monolithic world-coord geometry) to v4 (instanced -// meshes + per-instance records). Until v4 is finalised, loads always go -// through the streaming path and writes are no-ops. +// v4 layout (all multi-byte fields native-endian; endianness marker in header): +// +// SidecarHeader (16 bytes) +// uint64_t source_file_size +// +// uint32_t num_vertices_floats +// float[] vertex data (28 B/vertex: pos3 + normal3 + color1_packed) +// uint32_t num_indices +// uint32_t[] index data (mesh-local indices; base_vertex applied at draw time) +// +// uint32_t num_meshes +// MeshInfo[num_meshes] +// +// uint32_t num_instances +// InstanceCpu[num_instances] (already sorted by mesh_id) +// +// uint32_t num_elements +// PackedElementInfo[num_elements] +// uint32_t string_table_bytes +// char[string_table_bytes] #include "SidecarCache.h" -bool writeSidecar(const std::string& /*ifc_path*/, - const SidecarData& /*data*/, - uint64_t /*ifc_file_size*/) { +#include +#include + +struct SidecarHeader { + uint32_t magic; + uint32_t version; + uint32_t endian; + uint32_t reserved; +}; + +static std::string sidecarPath(const std::string& ifc_path) { + return ifc_path + ".ifcview"; +} + +template +static bool writeVec(FILE* f, const std::vector& v) { + uint32_t n = static_cast(v.size()); + if (fwrite(&n, 4, 1, f) != 1) return false; + if (n > 0 && fwrite(v.data(), sizeof(T), n, f) != n) return false; return true; } -std::optional readSidecar(const std::string& /*ifc_path*/, - uint64_t /*ifc_file_size*/) { - return std::nullopt; +template +static bool readVec(FILE* f, std::vector& v) { + uint32_t n; + if (fread(&n, 4, 1, f) != 1) return false; + v.resize(n); + if (n > 0 && fread(v.data(), sizeof(T), n, f) != n) return false; + return true; +} + +bool writeSidecar(const std::string& ifc_path, + const SidecarData& data, + uint64_t ifc_file_size) { + std::string path = sidecarPath(ifc_path); + FILE* f = fopen(path.c_str(), "wb"); + if (!f) return false; + + SidecarHeader hdr = { SIDECAR_MAGIC, SIDECAR_VERSION, SIDECAR_ENDIAN, 0 }; + if (fwrite(&hdr, sizeof(hdr), 1, f) != 1) { fclose(f); return false; } + if (fwrite(&ifc_file_size, 8, 1, f) != 1) { fclose(f); return false; } + + if (!writeVec(f, data.vertices)) { fclose(f); return false; } + if (!writeVec(f, data.indices)) { fclose(f); return false; } + if (!writeVec(f, data.meshes)) { fclose(f); return false; } + if (!writeVec(f, data.instances)) { fclose(f); return false; } + if (!writeVec(f, data.elements)) { fclose(f); return false; } + + uint32_t stbl_len = static_cast(data.string_table.size()); + if (fwrite(&stbl_len, 4, 1, f) != 1) { fclose(f); return false; } + if (stbl_len > 0 && fwrite(data.string_table.data(), 1, stbl_len, f) != stbl_len) { + fclose(f); return false; + } + + fclose(f); + return true; +} + +std::optional readSidecar(const std::string& ifc_path, + uint64_t ifc_file_size) { + std::string path = sidecarPath(ifc_path); + FILE* f = fopen(path.c_str(), "rb"); + if (!f) return std::nullopt; + + auto fail = [&]() -> std::optional { fclose(f); return std::nullopt; }; + + SidecarHeader hdr; + if (fread(&hdr, sizeof(hdr), 1, f) != 1) return fail(); + if (hdr.magic != SIDECAR_MAGIC || + hdr.version != SIDECAR_VERSION || + hdr.endian != SIDECAR_ENDIAN) return fail(); + + uint64_t stored_size; + if (fread(&stored_size, 8, 1, f) != 1) return fail(); + if (stored_size != ifc_file_size) return fail(); + + SidecarData data; + if (!readVec(f, data.vertices)) return fail(); + if (!readVec(f, data.indices)) return fail(); + if (!readVec(f, data.meshes)) return fail(); + if (!readVec(f, data.instances)) return fail(); + if (!readVec(f, data.elements)) return fail(); + + uint32_t stbl_len; + if (fread(&stbl_len, 4, 1, f) != 1) return fail(); + data.string_table.resize(stbl_len); + if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len) + return fail(); + + fclose(f); + return data; } diff --git a/src/ifcviewer/ViewportWindow.cpp b/src/ifcviewer/ViewportWindow.cpp index e264f990e4..48558fc64f 100644 --- a/src/ifcviewer/ViewportWindow.cpp +++ b/src/ifcviewer/ViewportWindow.cpp @@ -523,6 +523,103 @@ void ViewportWindow::finalizeModel(uint32_t model_id) { ssbo_bytes / (1024.0*1024.0)); } +bool ViewportWindow::snapshotModel(uint32_t model_id, SidecarData& out) const { + auto it = models_gpu_.find(model_id); + if (!gl_ || it == models_gpu_.end()) return false; + const auto& m = it->second; + if (!m.finalized) return false; + + // GPU readback of the packed VBO/EBO ranges actually in use. + if (m.vbo_used > 0) { + out.vertices.resize(m.vbo_used / sizeof(float)); + gl_->glGetNamedBufferSubData(m.vbo, 0, m.vbo_used, out.vertices.data()); + } + if (m.ebo_used > 0) { + out.indices.resize(m.ebo_used / sizeof(uint32_t)); + gl_->glGetNamedBufferSubData(m.ebo, 0, m.ebo_used, out.indices.data()); + } + + out.meshes = m.meshes; + out.instances = m.instances; + return true; +} + +void ViewportWindow::applyCachedModel(uint32_t model_id, SidecarData data) { + if (!gl_initialized_) return; + context_->makeCurrent(this); + + // Drop any existing state for this model_id. + auto existing = models_gpu_.find(model_id); + if (existing != models_gpu_.end()) { + if (existing->second.vao) gl_->glDeleteVertexArrays(1, &existing->second.vao); + if (existing->second.vbo) gl_->glDeleteBuffers(1, &existing->second.vbo); + if (existing->second.ebo) gl_->glDeleteBuffers(1, &existing->second.ebo); + if (existing->second.ssbo) gl_->glDeleteBuffers(1, &existing->second.ssbo); + models_gpu_.erase(existing); + } + + ModelGpuData m; + gl_->glCreateVertexArrays(1, &m.vao); + gl_->glCreateBuffers(1, &m.vbo); + gl_->glCreateBuffers(1, &m.ebo); + + const size_t vb_bytes = data.vertices.size() * sizeof(float); + const size_t ib_bytes = data.indices.size() * sizeof(uint32_t); + m.vbo_capacity = std::max(vb_bytes, 1); + m.ebo_capacity = std::max(ib_bytes, 1); + gl_->glNamedBufferStorage(m.vbo, m.vbo_capacity, + vb_bytes ? data.vertices.data() : nullptr, + GL_DYNAMIC_STORAGE_BIT); + gl_->glNamedBufferStorage(m.ebo, m.ebo_capacity, + ib_bytes ? data.indices.data() : nullptr, + GL_DYNAMIC_STORAGE_BIT); + setupVaoLayout(m.vao, m.vbo, m.ebo); + + m.vbo_used = vb_bytes; + m.ebo_used = ib_bytes; + m.vertex_count = static_cast( + data.vertices.size() / INSTANCED_VERTEX_STRIDE_FLOATS); + m.meshes = std::move(data.meshes); + m.instances = std::move(data.instances); + + uint32_t total_tri = 0; + for (const auto& mesh : m.meshes) { + total_tri += (mesh.index_count / 3) * mesh.instance_count; + } + m.total_triangles = total_tri; + + // Build and upload the instance SSBO. + std::vector gpu(m.instances.size()); + for (size_t i = 0; i < m.instances.size(); ++i) { + const InstanceCpu& src = m.instances[i]; + InstanceGpu& dst = gpu[i]; + std::memcpy(dst.transform, src.transform, sizeof(dst.transform)); + dst.object_id = src.object_id; + dst.color_override_rgba8 = src.color_override_rgba8; + dst._pad0 = 0; + dst._pad1 = 0; + } + gl_->glCreateBuffers(1, &m.ssbo); + const size_t ssbo_bytes = gpu.size() * sizeof(InstanceGpu); + if (ssbo_bytes > 0) { + gl_->glNamedBufferStorage(m.ssbo, ssbo_bytes, gpu.data(), 0); + } + m.ssbo_instance_count = static_cast(gpu.size()); + + m.finalized = true; + models_gpu_.emplace(model_id, std::move(m)); + + qDebug("Sidecar apply: model %u %zu verts, %zu meshes, %zu instances " + "%.1f MB vram (vbo %.1f + ebo %.1f + ssbo %.1f)", + model_id, data.vertices.size() / INSTANCED_VERTEX_STRIDE_FLOATS, + models_gpu_[model_id].meshes.size(), + models_gpu_[model_id].instances.size(), + (vb_bytes + ib_bytes + ssbo_bytes) / (1024.0*1024.0), + vb_bytes / (1024.0*1024.0), + ib_bytes / (1024.0*1024.0), + ssbo_bytes / (1024.0*1024.0)); +} + void ViewportWindow::resetScene() { if (!gl_initialized_) return; context_->makeCurrent(this); diff --git a/src/ifcviewer/ViewportWindow.h b/src/ifcviewer/ViewportWindow.h index 9fbdcf054b..65b15412e9 100644 --- a/src/ifcviewer/ViewportWindow.h +++ b/src/ifcviewer/ViewportWindow.h @@ -84,6 +84,16 @@ public: void resetScene(); + // Snapshot the finalised model into a SidecarData struct for caching. + // Vertices + indices are read back from the GPU; meshes/instances come + // from the CPU-side vectors. Leaves `elements` and `string_table` empty + // for the caller to fill in. + bool snapshotModel(uint32_t model_id, SidecarData& out) const; + + // Restore a finalised model from a cached SidecarData struct. Replaces + // any existing state for model_id and marks it drawable. + void applyCachedModel(uint32_t model_id, SidecarData data); + void hideModel(uint32_t model_id); void showModel(uint32_t model_id); void removeModel(uint32_t model_id);