diff --git a/src/ifcviewer-web/sample.ifcview b/src/ifcviewer-web/sample.ifcview index 52558e4d14..fed5f228e1 100644 Binary files a/src/ifcviewer-web/sample.ifcview and b/src/ifcviewer-web/sample.ifcview differ diff --git a/src/ifcviewer/ChunkPlanner.h b/src/ifcviewer/ChunkPlanner.h index 1cc9595263..c16dadcd68 100644 --- a/src/ifcviewer/ChunkPlanner.h +++ b/src/ifcviewer/ChunkPlanner.h @@ -30,6 +30,17 @@ #include #include +// Vertex-bytes ceiling for one streaming chunk. The greedy packer starts a new +// chunk before a mesh would push the running vertex bytes over this. Lives here +// (pure, no wgpu) so the bake-time layout pass and the GPU loader share one +// value. +// +// 4 MB (down from 16 MB): with v14 each chunk is one contiguous range read, so +// small chunks are cheap, and they paint progressively far sooner over a +// network — first-paint payload ≈ metadata + (concurrency cap × this). In line +// with what streaming viewers (Cesium 3D Tiles, xeokit, SVF2) use. +static constexpr std::uint64_t WGPU_CHUNK_VERTEX_BYTES_LIMIT = 4ull * 1024 * 1024; + namespace ChunkPlanner { // Interleave the low 21 bits of v with two zero bits between each, diff --git a/src/ifcviewer/ModelGpuData.h b/src/ifcviewer/ModelGpuData.h index 3aa141e574..762ecc32f1 100644 --- a/src/ifcviewer/ModelGpuData.h +++ b/src/ifcviewer/ModelGpuData.h @@ -33,6 +33,7 @@ #include "InstancedGeometry.h" #include "BufferPool.h" +#include "ChunkPlanner.h" // WGPU_CHUNK_VERTEX_BYTES_LIMIT (shared with bake) // Per-model wgpu state. Mirrors the GL backend's ModelGpuData but with // wgpu handles. Stage 2 only allocates and uploads the four core buffers; @@ -62,7 +63,9 @@ // (~4 MB) with single-fread chunk loads, but the difference between // 16 MB and 4 MB is much smaller than the difference between 128 MB // and 16 MB. -static constexpr uint64_t WGPU_CHUNK_VERTEX_BYTES_LIMIT = 16ull * 1024 * 1024; +// +// The limit itself lives in ChunkPlanner.h (pure, no wgpu) so the bake-time +// layout pass can share it; re-exported here for the existing call sites. struct ModelGpuData { // std430 layout: 16 bytes per entry, naturally aligned. base_vertex is diff --git a/src/ifcviewer/SceneLoader.cpp b/src/ifcviewer/SceneLoader.cpp index 4b96aae495..9a4b80195c 100644 --- a/src/ifcviewer/SceneLoader.cpp +++ b/src/ifcviewer/SceneLoader.cpp @@ -19,6 +19,7 @@ #include "SceneLoader.h" #include "AppSettings.h" +#include "SidecarLayout.h" #include #include @@ -392,6 +393,9 @@ void SceneLoader::onStreamerFinished() { } QElapsedTimer wt; wt.start(); SidecarData data = m.sidecar_builder->finalize(georef, m.streamed_elements); + // Lay geometry out in streaming-chunk order + bake the chunk TOC + // (v14) so it streams as one contiguous range per chunk. + reorderSidecarByMorton(data); const bool ok = writeSidecar(m.file_path.toStdString(), data); std::fprintf(stderr, "[info] Sidecar finalize + write: %lld ms (%s)\n", diff --git a/src/ifcviewer/SidecarCache.cpp b/src/ifcviewer/SidecarCache.cpp index d0ca2ccac2..e3cdbe90c1 100644 --- a/src/ifcviewer/SidecarCache.cpp +++ b/src/ifcviewer/SidecarCache.cpp @@ -116,6 +116,9 @@ bool writeSidecar(const std::string& ifc_path, const SidecarData& data) { fclose(f); return false; } + // v14 chunk TOC. + if (!writeVec(f, data.chunks)) { fclose(f); return false; } + fclose(f); return true; } @@ -156,6 +159,9 @@ std::optional readSidecar(const std::string& ifc_path) { if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len) return fail(); + // v14 chunk TOC. + if (!readVec(f, data.chunks)) return fail(); + fclose(f); return data; } diff --git a/src/ifcviewer/SidecarCache.h b/src/ifcviewer/SidecarCache.h index 76dd52e4b2..04e6827adb 100644 --- a/src/ifcviewer/SidecarCache.h +++ b/src/ifcviewer/SidecarCache.h @@ -65,9 +65,26 @@ static constexpr uint32_t SIDECAR_MAGIC = 0x49465657; // "IFVW" // composition has reduced them to viewport-local float-sized values. // v13 = Map unit scale in cached ModelGeoref is derived from // IfcMapConversion.Scale, not IfcProjectedCRS.MapUnit. -static constexpr uint32_t SIDECAR_VERSION = 13; +// v14 = Geometry is laid out in streaming-chunk order (SidecarLayout) and a +// chunk table-of-contents (`chunks`) is appended. The loader builds its +// chunks from the TOC instead of re-deriving the Morton/greedy plan, so +// each chunk is one CONTIGUOUS byte range — fixing network read +// amplification. The plan can't be re-derived at load because the float +// Morton quantisation isn't bit-identical across toolchains (x86 baker vs +// wasm loader), so it must be baked in. No back-compat: v13 sidecars are +// rejected (regenerate them). +static constexpr uint32_t SIDECAR_VERSION = 14; static constexpr uint32_t SIDECAR_ENDIAN = 0x01020304; +// Chunk table-of-contents entry (v14+). A chunk is a CONTIGUOUS range of +// meshes in the (reordered) meshes array — and therefore a contiguous span of +// vertex + index bytes, since the geometry is laid out in chunk order. The +// loader builds chunk `i` from meshes [first_mesh, first_mesh + mesh_count). +struct SidecarChunk { + uint32_t first_mesh; + uint32_t mesh_count; +}; + // Fixed-size element record. Strings are stored as (offset, length) pairs // into a separate string table. struct PackedElementInfo { @@ -111,6 +128,13 @@ struct SidecarData { // Element tree metadata. std::vector elements; std::string string_table; + + // Streaming chunk TOC. Always written on disk (v14); geometry is laid out + // in this chunk order (see SidecarLayout) so each chunk is one contiguous + // range and the loader builds chunks directly from it. Stays empty only for + // in-memory direct loads (finalizeModel), which don't stream and fall back + // to deriving the plan. + std::vector chunks; }; // Sidecar is keyed on the path stem: foo.ifc and foo.ifcdb/ both resolve to diff --git a/src/ifcviewer/SidecarLayout.cpp b/src/ifcviewer/SidecarLayout.cpp new file mode 100644 index 0000000000..c7f699440b --- /dev/null +++ b/src/ifcviewer/SidecarLayout.cpp @@ -0,0 +1,135 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#include "SidecarLayout.h" + +#include "ChunkPlanner.h" +#include "InstancedGeometry.h" + +#include +#include + +void reorderSidecarByMorton(SidecarData& sd) { + const std::size_t n = sd.meshes.size(); + if (n < 2) return; + + // Per-mesh centroid + instance count, exactly as the loader computes them + // before chunk planning (average of instance world-AABB centres). + std::vector cx(n, 0.0f), cy(n, 0.0f), cz(n, 0.0f); + std::vector cnt(n, 0); + for (const auto& inst : sd.instances) { + if (inst.mesh_id >= n) continue; + cx[inst.mesh_id] += 0.5f * (inst.world_aabb_min[0] + inst.world_aabb_max[0]); + cy[inst.mesh_id] += 0.5f * (inst.world_aabb_min[1] + inst.world_aabb_max[1]); + cz[inst.mesh_id] += 0.5f * (inst.world_aabb_min[2] + inst.world_aabb_max[2]); + ++cnt[inst.mesh_id]; + } + for (std::size_t i = 0; i < n; ++i) { + if (cnt[i] > 0) { + const float inv = 1.0f / float(cnt[i]); + cx[i] *= inv; cy[i] *= inv; cz[i] *= inv; + } + } + + // order[new_id] = old mesh id, in the loader's Morton order. + const std::vector order = + ChunkPlanner::sortMeshIdsByMorton(n, cx, cy, cz, cnt); + + // Greedy-pack the sorted order into chunks (the same plan the loader used + // to derive). Each chunk is a CONSECUTIVE run of `order`, so once we lay + // meshes out in `order` the chunk is a contiguous mesh range — recorded in + // the TOC as {first_mesh, mesh_count}. + std::vector mesh_vertex_count(n, 0); + for (std::size_t i = 0; i < n; ++i) mesh_vertex_count[i] = sd.meshes[i].vertex_count; + const std::vector> packed = ChunkPlanner::greedyPackChunks( + order, mesh_vertex_count, INSTANCED_VERTEX_STRIDE_BYTES, + WGPU_CHUNK_VERTEX_BYTES_LIMIT); + sd.chunks.clear(); + sd.chunks.reserve(packed.size()); + { + std::uint32_t first = 0; + for (const auto& chunk : packed) { + sd.chunks.push_back({first, std::uint32_t(chunk.size())}); + first += std::uint32_t(chunk.size()); + } + } + + // Bucket instances by their (authoritative) mesh_id. We must NOT rely on + // MeshInfo.first_instance: the baker leaves it 0 for every mesh and stores + // instances ungrouped, so first_instance describes nothing. Grouping here + // by mesh_id both reorders instances correctly AND fixes first_instance. + std::vector> insts_by_mesh(n); + for (std::uint32_t ii = 0; ii < sd.instances.size(); ++ii) { + const std::uint32_t mid = sd.instances[ii].mesh_id; + if (mid < n) insts_by_mesh[mid].push_back(ii); + } + + std::vector new_vertices; new_vertices.reserve(sd.vertices.size()); + std::vector new_indices; new_indices.reserve(sd.indices.size()); + std::vector new_meshes(n); + std::vector new_instances; new_instances.reserve(sd.instances.size()); + + // Pass A: vertices + LOD0 indices + instances, mesh-by-mesh in the new + // order, recording the new offsets on each MeshInfo. + for (std::uint32_t ni = 0; ni < n; ++ni) { + const std::uint32_t old = order[ni]; + const MeshInfo& om = sd.meshes[old]; + MeshInfo nm = om; // carries AABB; offsets/instance fields overwritten below + + nm.vbo_byte_offset = std::uint32_t(new_vertices.size()); + const std::size_t vbytes = std::size_t(om.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES; + new_vertices.insert(new_vertices.end(), + sd.vertices.begin() + om.vbo_byte_offset, + sd.vertices.begin() + om.vbo_byte_offset + vbytes); + + nm.ebo_byte_offset = std::uint32_t(new_indices.size() * sizeof(std::uint32_t)); + const std::size_t i0 = om.ebo_byte_offset / sizeof(std::uint32_t); + new_indices.insert(new_indices.end(), + sd.indices.begin() + i0, + sd.indices.begin() + i0 + om.index_count); + + nm.first_instance = std::uint32_t(new_instances.size()); + nm.instance_count = std::uint32_t(insts_by_mesh[old].size()); + for (std::uint32_t ii : insts_by_mesh[old]) { + InstanceCpu ic = sd.instances[ii]; + ic.mesh_id = ni; + new_instances.push_back(ic); + } + + new_meshes[ni] = nm; + } + + // Pass B: LOD1 indices appended after all LOD0 (same global layout as the + // baker), in the new order, so a chunk's LOD1 slice is contiguous too. + for (std::uint32_t ni = 0; ni < n; ++ni) { + const MeshInfo& om = sd.meshes[order[ni]]; + MeshInfo& nm = new_meshes[ni]; + if (om.lod1_index_count == 0) { nm.lod1_ebo_byte_offset = 0; continue; } + nm.lod1_ebo_byte_offset = std::uint32_t(new_indices.size() * sizeof(std::uint32_t)); + const std::size_t l0 = om.lod1_ebo_byte_offset / sizeof(std::uint32_t); + new_indices.insert(new_indices.end(), + sd.indices.begin() + l0, + sd.indices.begin() + l0 + om.lod1_index_count); + } + + sd.vertices = std::move(new_vertices); + sd.indices = std::move(new_indices); + sd.meshes = std::move(new_meshes); + sd.instances = std::move(new_instances); +} diff --git a/src/ifcviewer/SidecarLayout.h b/src/ifcviewer/SidecarLayout.h new file mode 100644 index 0000000000..40d21e6297 --- /dev/null +++ b/src/ifcviewer/SidecarLayout.h @@ -0,0 +1,58 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#ifndef SIDECARLAYOUT_H +#define SIDECARLAYOUT_H + +#include "SidecarCache.h" + +// Reorder a sidecar's geometry for streaming locality. +// +// The streaming loader chunks meshes by 3D Morton (Z-order) over their +// centroids, then greedy-packs them into ~16 MB chunks; a chunk is always a +// CONSECUTIVE run of that sorted order. But a freshly-baked sidecar stores +// vertices / indices / meshes in mesh-id (iterator) order, which has no +// relation to the spatial chunking — so a chunk's meshes are scattered through +// the file, and streaming one chunk over a network either issues hundreds of +// tiny range requests or reads (and discards) everything in between (~3× +// bandwidth amplification was measured on a 113 MB model). +// +// This pass permutes meshes into the loader's Morton order and rebuilds the +// vertex / index / instance sections to match, so that each spatial chunk +// becomes a CONTIGUOUS byte range. The loader then re-runs the same Morton +// sort, gets the identity permutation, and reads each chunk as one contiguous +// range — no amplification, ~1 request per chunk, and chunks appear +// progressively as they arrive. +// +// Pure transform (no Qt / no wgpu): meshes, vertices, indices (LOD0 + LOD1), +// and instances are all rebuilt in the new order with vbo/ebo/lod1 offsets, +// MeshInfo.first_instance, and InstanceCpu.mesh_id remapped consistently. +// Index values are mesh-local, so they move unchanged. Element/georef/string +// data is mesh-independent and untouched. No-op for < 2 meshes. +// +// Also populates `sd.chunks` (the v14 TOC): the loader must build chunks from +// this rather than re-deriving the plan, because the float Morton quantisation +// isn't bit-identical across toolchains (x86 baker vs wasm loader) — a re-derived +// plan disagrees on boundary meshes and the contiguity is lost. +// +// Run at bake (writeSidecar path) or as a one-shot migration over existing +// .ifcview files (read → reorder → write). +void reorderSidecarByMorton(SidecarData& sd); + +#endif // SIDECARLAYOUT_H diff --git a/src/ifcviewer/StreamingLoader.cpp b/src/ifcviewer/StreamingLoader.cpp index b9f25b0d60..760bf48fee 100644 --- a/src/ifcviewer/StreamingLoader.cpp +++ b/src/ifcviewer/StreamingLoader.cpp @@ -116,6 +116,9 @@ bool parseSidecarTail(const uint8_t* data, size_t n, SidecarData& out) { if (stbl_len > c.remaining) return false; out.string_table.resize(stbl_len); if (stbl_len > 0 && !c.take(out.string_table.data(), stbl_len)) return false; + + // v14 chunk TOC. + if (!c.takeVec(out.chunks)) return false; return true; } diff --git a/src/ifcviewer/ViewportCore.cpp b/src/ifcviewer/ViewportCore.cpp index f643039903..a33e81c9c3 100644 --- a/src/ifcviewer/ViewportCore.cpp +++ b/src/ifcviewer/ViewportCore.cpp @@ -2349,8 +2349,21 @@ void ViewportCore::driveStreamingLoads() { // hold is_loading until then so it isn't re-issued every frame. The // embedded MEMFS sample falls through to the synchronous fopen path. if (cand.m->streaming_from_web) { + // Cap concurrent chunk downloads. The browser multiplexes every + // in-flight Range request over one HTTP/2 connection, so without a + // cap all visible chunks download at once, split the bandwidth N + // ways, and finish together — nothing paints until ~the whole model + // has arrived (measured: 9 in flight → first paint after 113 of + // 118 MB). A small cap lets the highest-priority chunks (candidates + // are priority-sorted) finish first and paint, then the next — + // progressive, no special first-chunk handling. + if (streaming_web_inflight_count_ >= kMaxWebInflightChunks) { + more_pending = true; + break; // resume next frame as in-flight loads complete + } c.is_loading = true; c.last_visible_frame_idx = streaming_frame_idx_; + ++streaming_web_inflight_count_; beginWebChunkLoad(cand.mid, cand.ci); ++enqueued; continue; @@ -2805,49 +2818,67 @@ void ViewportCore::applyCachedModel(std::uint32_t model_id, m.streaming_index_section_offset = metadata.index_section_offset; // ---- Spatial chunk plan ---------------------------------------------- - // Sort meshes by 3D Morton code over centroids, then greedy-pack into - // chunks <= WGPU_CHUNK_VERTEX_BYTES_LIMIT. Each chunk's AABB ends up - // tight rather than spanning the whole model, so the distance-based - // streaming evictor can meaningfully distinguish chunks. + // A sidecar carries a baked chunk TOC (v14): each chunk is a contiguous + // run of meshes, laid out contiguously in the file (see SidecarLayout), so + // we build chunks straight from it — one contiguous byte range per chunk. + // The plan is NOT re-derived here because the float Morton quantisation + // isn't bit-identical across toolchains (x86 baker vs wasm loader), which + // would scatter the chunks. In-memory direct loads (finalizeModel) carry + // no TOC, so they fall back to deriving the same Morton + greedy plan. const std::size_t n_meshes = metadata.meta.meshes.size(); m.mesh_chunk_idx.assign(n_meshes, 0); m.mesh_chunk_local_base_vertex.assign(n_meshes, 0); m.mesh_chunk_local_ebo_first_u32.assign(n_meshes, 0); m.mesh_chunk_local_lod1_first_u32.assign(n_meshes, 0); - std::vector mesh_cx(n_meshes, 0.0f), - mesh_cy(n_meshes, 0.0f), - mesh_cz(n_meshes, 0.0f); - std::vector mesh_inst_count(n_meshes, 0); - for (const auto& inst : metadata.meta.instances) { - if (inst.mesh_id >= n_meshes) continue; - mesh_cx[inst.mesh_id] += 0.5f * (inst.world_aabb_min[0] + inst.world_aabb_max[0]); - mesh_cy[inst.mesh_id] += 0.5f * (inst.world_aabb_min[1] + inst.world_aabb_max[1]); - mesh_cz[inst.mesh_id] += 0.5f * (inst.world_aabb_min[2] + inst.world_aabb_max[2]); - ++mesh_inst_count[inst.mesh_id]; - } - for (std::size_t i = 0; i < n_meshes; ++i) { - if (mesh_inst_count[i] > 0) { - const float inv = 1.0f / float(mesh_inst_count[i]); - mesh_cx[i] *= inv; mesh_cy[i] *= inv; mesh_cz[i] *= inv; - } - } - std::vector> chunk_mesh_ids; std::vector instance_to_chunk; instance_to_chunk.assign(metadata.meta.instances.size(), 0); - { + + if (!metadata.meta.chunks.empty()) { + // Baked TOC: chunk ci is meshes [first_mesh, first_mesh + mesh_count). + chunk_mesh_ids.reserve(metadata.meta.chunks.size()); + for (const auto& ch : metadata.meta.chunks) { + std::vector ids; + ids.reserve(ch.mesh_count); + for (std::uint32_t k = 0; k < ch.mesh_count; ++k) { + const std::uint32_t mi = ch.first_mesh + k; + if (mi < n_meshes) ids.push_back(mi); + } + chunk_mesh_ids.push_back(std::move(ids)); + } + } else { + // No TOC (direct load): derive the plan from mesh centroids. + std::vector mesh_cx(n_meshes, 0.0f), + mesh_cy(n_meshes, 0.0f), + mesh_cz(n_meshes, 0.0f); + std::vector mesh_inst_count(n_meshes, 0); + for (const auto& inst : metadata.meta.instances) { + if (inst.mesh_id >= n_meshes) continue; + mesh_cx[inst.mesh_id] += 0.5f * (inst.world_aabb_min[0] + inst.world_aabb_max[0]); + mesh_cy[inst.mesh_id] += 0.5f * (inst.world_aabb_min[1] + inst.world_aabb_max[1]); + mesh_cz[inst.mesh_id] += 0.5f * (inst.world_aabb_min[2] + inst.world_aabb_max[2]); + ++mesh_inst_count[inst.mesh_id]; + } + for (std::size_t i = 0; i < n_meshes; ++i) { + if (mesh_inst_count[i] > 0) { + const float inv = 1.0f / float(mesh_inst_count[i]); + mesh_cx[i] *= inv; mesh_cy[i] *= inv; mesh_cz[i] *= inv; + } + } std::vector sorted_mesh_ids = ChunkPlanner::sortMeshIdsByMorton( n_meshes, mesh_cx, mesh_cy, mesh_cz, mesh_inst_count); std::vector mesh_vertex_count; mesh_vertex_count.reserve(n_meshes); - for (std::size_t i = 0; i < n_meshes; ++i) { + for (std::size_t i = 0; i < n_meshes; ++i) mesh_vertex_count.push_back(metadata.meta.meshes[i].vertex_count); - } chunk_mesh_ids = ChunkPlanner::greedyPackChunks( sorted_mesh_ids, mesh_vertex_count, INSTANCED_VERTEX_STRIDE_BYTES, WGPU_CHUNK_VERTEX_BYTES_LIMIT); + } + + { std::vector mesh_to_chunk(n_meshes, 0); for (std::size_t ci = 0; ci < chunk_mesh_ids.size(); ++ci) { for (std::uint32_t mi : chunk_mesh_ids[ci]) mesh_to_chunk[mi] = std::uint32_t(ci); @@ -3378,9 +3409,11 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i std::function finish = [this, model_id, chunk_idx, need, join]() { if (!join->v_done || !join->i_done) return; // wait for the other read // Release the in-flight reservation (clamped — a mid-flight resetScene - // could have zeroed it) regardless of what happens below. + // could have zeroed it) + the concurrency slot, regardless of outcome. streaming_web_inflight_bytes_ -= std::min(streaming_web_inflight_bytes_, need); + if (streaming_web_inflight_count_ > 0) --streaming_web_inflight_count_; + host_->requestFrame(); // a slot freed — let driveStreamingLoads issue more auto mit = models_gpu_.find(model_id); if (mit == models_gpu_.end()) return; diff --git a/src/ifcviewer/ViewportCore.h b/src/ifcviewer/ViewportCore.h index 9f7afd6dd4..350d9e8cb1 100644 --- a/src/ifcviewer/ViewportCore.h +++ b/src/ifcviewer/ViewportCore.h @@ -907,6 +907,15 @@ private: // candidates that won't fit total_free - this. std::uint64_t streaming_web_inflight_bytes_ = 0; + // Web only: number of chunk loads in flight, and the cap. The browser + // multiplexes all in-flight Range requests over one HTTP/2 connection, so + // an unbounded count splits the bandwidth N ways and nothing finishes (so + // nothing paints) until ~the whole model has downloaded. A small cap lets + // the highest-priority chunks finish + paint first, then the next — + // progressive streaming. Tune for first-paint vs latency-hiding. + static constexpr int kMaxWebInflightChunks = 2; + int streaming_web_inflight_count_ = 0; + // Settle burst: keep the render loop alive for a few frames after any // streaming activity so the cull→load→display latency (the draw + cull // precede driveStreamingLoads, so a freshly-resident chunk paints a frame diff --git a/src/ifcviewer/tests/CMakeLists.txt b/src/ifcviewer/tests/CMakeLists.txt index b9709cceed..bc85395525 100644 --- a/src/ifcviewer/tests/CMakeLists.txt +++ b/src/ifcviewer/tests/CMakeLists.txt @@ -66,6 +66,16 @@ add_ifcviewer_unit_test(test_chunk_planner SOURCES ${IFCVIEWER_SRC}/ChunkPlanner.cpp ) +# SidecarLayout: bake-time reorder of geometry into the loader's chunk order +# for streaming locality. Needs ChunkPlanner (Morton sort) + SidecarCache +# (SidecarData). Pure. +add_ifcviewer_unit_test(test_sidecar_layout + SOURCES + ${IFCVIEWER_SRC}/SidecarLayout.cpp + ${IFCVIEWER_SRC}/ChunkPlanner.cpp + ${IFCVIEWER_SRC}/SidecarCache.cpp +) + # InstanceCompose: matrix composition + cross-model object_id lookup. # Pulls in wgpu_native for the WGPUBuffer typedef via ModelGpuData.h # (never touched at runtime). Eigen for Matrix4d. diff --git a/src/ifcviewer/tests/test_sidecar_cache.cpp b/src/ifcviewer/tests/test_sidecar_cache.cpp index a9747462ce..d4a946b9eb 100644 --- a/src/ifcviewer/tests/test_sidecar_cache.cpp +++ b/src/ifcviewer/tests/test_sidecar_cache.cpp @@ -155,10 +155,26 @@ bool sidecarDataEqual(const SidecarData& a, const SidecarData& b) { TEST_CASE("MeshInfo and InstanceCpu have stable layouts (sidecar wire format)", "[sidecar]") { REQUIRE(sizeof(MeshInfo) == 56); REQUIRE(sizeof(InstanceGpu) == 80); - REQUIRE(SIDECAR_VERSION == 13); + REQUIRE(SIDECAR_VERSION == 14); + REQUIRE(sizeof(SidecarChunk) == 8); REQUIRE(SIDECAR_MAGIC == 0x49465657u); } +TEST_CASE("writeSidecar/readSidecar round-trip the v14 chunk TOC", "[sidecar]") { + fs::path dir = makeScratchDir("chunks"); + fs::path ifc = dir / "model.ifc"; + SidecarData sd = buildFixture(); + sd.chunks = { {0, 1}, {1, 1} }; // two chunks over the two meshes + REQUIRE(writeSidecar(ifc.string(), sd)); + auto loaded = readSidecar(ifc.string()); + REQUIRE(loaded.has_value()); + REQUIRE(loaded->chunks.size() == 2); + REQUIRE(loaded->chunks[0].first_mesh == 0); + REQUIRE(loaded->chunks[0].mesh_count == 1); + REQUIRE(loaded->chunks[1].first_mesh == 1); + REQUIRE(loaded->chunks[1].mesh_count == 1); +} + TEST_CASE("writeSidecar then readSidecar round-trips the full fixture", "[sidecar]") { fs::path dir = makeScratchDir("roundtrip"); fs::path ifc = dir / "model.ifc"; diff --git a/src/ifcviewer/tests/test_sidecar_layout.cpp b/src/ifcviewer/tests/test_sidecar_layout.cpp new file mode 100644 index 0000000000..dde36947b0 --- /dev/null +++ b/src/ifcviewer/tests/test_sidecar_layout.cpp @@ -0,0 +1,218 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#include "ChunkPlanner.h" +#include "InstancedGeometry.h" +#include "SidecarCache.h" +#include "SidecarLayout.h" + +#include + +#include +#include +#include + +namespace { + +constexpr int STRIDE = INSTANCED_VERTEX_STRIDE_BYTES; + +// A fixture with N meshes, each with a unique vertex/index pattern + LOD1 on +// some, and instances spread across 3D space so the Morton sort actually +// permutes (not already sorted). Geometry is stored in mesh-id order (as a +// fresh bake produces it). +SidecarData buildFixture() { + SidecarData sd; + const int N = 6; + + // Per-mesh: vertex_count = i+2, index_count = i+2 (mesh-local 0..vc-1), + // lod1 on even meshes (lod1_count = 1). Vertices encode (mesh, vert). + std::vector meshes(N); + for (int i = 0; i < N; ++i) { + MeshInfo& m = meshes[i]; + const uint32_t vc = uint32_t(i + 2); + m.vbo_byte_offset = uint32_t(sd.vertices.size()); + m.vertex_count = vc; + for (uint32_t v = 0; v < vc; ++v) + for (int b = 0; b < STRIDE; ++b) + sd.vertices.push_back(uint8_t((i * 37 + v * 7 + b) & 0xFF)); + + m.ebo_byte_offset = uint32_t(sd.indices.size() * sizeof(uint32_t)); + m.index_count = vc; + for (uint32_t k = 0; k < vc; ++k) sd.indices.push_back(k); // mesh-local + + m.local_aabb_min[0] = float(-i); m.local_aabb_max[0] = float(i + 1); + m.local_aabb_min[1] = 0; m.local_aabb_max[1] = 2; + m.local_aabb_min[2] = 0; m.local_aabb_max[2] = 3; + } + // LOD1 slices appended after all LOD0 (matches the baker's global layout). + for (int i = 0; i < N; ++i) { + if (i % 2 != 0) { meshes[i].lod1_index_count = 0; continue; } + meshes[i].lod1_ebo_byte_offset = uint32_t(sd.indices.size() * sizeof(uint32_t)); + meshes[i].lod1_index_count = 1; + sd.indices.push_back(uint32_t(i)); // distinctive lod1 index + } + + // Instances: deliberately UNGROUPED (round-robin across meshes) with + // first_instance left at 0 — mimicking the real baker, which never sets + // first_instance and stores instances in stream order. A reorder that + // trusts first_instance instead of per-instance mesh_id scrambles them. + auto ninst = [](int i) { return uint32_t((i % 3) + 1); }; + for (int i = 0; i < N; ++i) { + meshes[i].first_instance = 0; // as the baker leaves it + meshes[i].instance_count = ninst(i); // baker sets the count + } + uint32_t obj = 100; + for (uint32_t k = 0; k < 3; ++k) { // outer loop = interleave + for (int i = 0; i < N; ++i) { + if (k >= ninst(i)) continue; + InstanceCpu ic; + ic.mesh_id = uint32_t(i); // authoritative + ic.object_id = obj++; + ic.model_id = 1; + const float x = float((i * 13 + k * 5) % 11); + const float y = float((i * 7 + k * 3) % 9); + const float z = float((i * 5 + k * 2) % 7); + ic.world_aabb_min[0] = x; ic.world_aabb_max[0] = x + 1; + ic.world_aabb_min[1] = y; ic.world_aabb_max[1] = y + 1; + ic.world_aabb_min[2] = z; ic.world_aabb_max[2] = z + 1; + for (int t = 0; t < 16; ++t) ic.transform[t] = float(ic.object_id) + 0.1f * t; + sd.instances.push_back(ic); + } + } + sd.meshes = meshes; + return sd; +} + +// Everything an instance "draws", independent of storage order: its mesh's +// vertex bytes, LOD0 + LOD1 index VALUES, local AABB, and its own transform. +struct InstSig { + std::vector verts; + std::vector idx0, idx1; + float aabb[6]; + float xf[16]; + bool operator==(const InstSig& o) const { + if (verts != o.verts || idx0 != o.idx0 || idx1 != o.idx1) return false; + for (int i = 0; i < 6; ++i) if (aabb[i] != o.aabb[i]) return false; + for (int i = 0; i < 16; ++i) if (xf[i] != o.xf[i]) return false; + return true; + } +}; + +InstSig sigFor(const SidecarData& sd, const InstanceCpu& inst) { + const MeshInfo& m = sd.meshes.at(inst.mesh_id); + InstSig s{}; + s.verts.assign(sd.vertices.begin() + m.vbo_byte_offset, + sd.vertices.begin() + m.vbo_byte_offset + + std::size_t(m.vertex_count) * STRIDE); + const std::size_t i0 = m.ebo_byte_offset / sizeof(uint32_t); + s.idx0.assign(sd.indices.begin() + i0, sd.indices.begin() + i0 + m.index_count); + if (m.lod1_index_count > 0) { + const std::size_t l0 = m.lod1_ebo_byte_offset / sizeof(uint32_t); + s.idx1.assign(sd.indices.begin() + l0, sd.indices.begin() + l0 + m.lod1_index_count); + } + s.aabb[0]=m.local_aabb_min[0]; s.aabb[1]=m.local_aabb_min[1]; s.aabb[2]=m.local_aabb_min[2]; + s.aabb[3]=m.local_aabb_max[0]; s.aabb[4]=m.local_aabb_max[1]; s.aabb[5]=m.local_aabb_max[2]; + for (int t = 0; t < 16; ++t) s.xf[t] = inst.transform[t]; + return s; +} + +std::map sigMap(const SidecarData& sd) { + std::map m; + for (const auto& inst : sd.instances) m[inst.object_id] = sigFor(sd, inst); + return m; +} + +} // namespace + +TEST_CASE("reorderSidecarByMorton preserves every instance's drawn geometry", "[layout]") { + SidecarData before = buildFixture(); + const auto sig_before = sigMap(before); + + SidecarData after = before; + reorderSidecarByMorton(after); + + // Same counts. + REQUIRE(after.meshes.size() == before.meshes.size()); + REQUIRE(after.instances.size() == before.instances.size()); + REQUIRE(after.vertices.size() == before.vertices.size()); + REQUIRE(after.indices.size() == before.indices.size()); + + // The geometry each object draws is byte-for-byte identical — only the + // storage order changed. + REQUIRE(sigMap(after) == sig_before); + + // first_instance / instance_count now correctly describe contiguous, + // mesh-grouped instance ranges (the baker left first_instance = 0, so a + // reorder must rebuild them from per-instance mesh_id — getting this wrong + // scrambles every transform and collapses geometry to the origin). + for (uint32_t mi = 0; mi < after.meshes.size(); ++mi) { + const auto& m = after.meshes[mi]; + for (uint32_t k = 0; k < m.instance_count; ++k) + REQUIRE(after.instances.at(m.first_instance + k).mesh_id == mi); + } + + // It actually permuted (the fixture isn't already Morton-sorted). + bool moved = false; + for (std::size_t i = 0; i < after.meshes.size(); ++i) + if (after.meshes[i].vbo_byte_offset != before.meshes[i].vbo_byte_offset || + after.meshes[i].vertex_count != before.meshes[i].vertex_count) moved = true; + REQUIRE(moved); +} + +TEST_CASE("reorderSidecarByMorton lays meshes out contiguously per the loader", "[layout]") { + SidecarData sd = buildFixture(); + reorderSidecarByMorton(sd); + + // Meshes' vertex + LOD0-index slices are laid down back-to-back in array + // order (so consecutive meshes — i.e. a chunk — form one contiguous range). + std::uint32_t v_cursor = 0, i_cursor = 0; + for (const auto& m : sd.meshes) { + REQUIRE(m.vbo_byte_offset == v_cursor); + v_cursor += m.vertex_count * STRIDE; + REQUIRE(m.ebo_byte_offset == i_cursor * sizeof(std::uint32_t)); + i_cursor += m.index_count; + } + + // Re-running the loader's Morton sort on the laid-out data yields the + // identity permutation — which is exactly what makes the greedy-packed + // chunks consecutive (hence contiguous) byte ranges at load time. + const std::size_t n = sd.meshes.size(); + std::vector cx(n,0), cy(n,0), cz(n,0); std::vector cnt(n,0); + for (const auto& inst : sd.instances) { + cx[inst.mesh_id] += 0.5f*(inst.world_aabb_min[0]+inst.world_aabb_max[0]); + cy[inst.mesh_id] += 0.5f*(inst.world_aabb_min[1]+inst.world_aabb_max[1]); + cz[inst.mesh_id] += 0.5f*(inst.world_aabb_min[2]+inst.world_aabb_max[2]); + ++cnt[inst.mesh_id]; + } + for (std::size_t i=0;i0){float inv=1.0f/cnt[i]; cx[i]*=inv;cy[i]*=inv;cz[i]*=inv;} + const auto order = ChunkPlanner::sortMeshIdsByMorton(n, cx, cy, cz, cnt); + for (std::uint32_t i = 0; i < n; ++i) REQUIRE(order[i] == i); +} + +TEST_CASE("reorderSidecarByMorton is a no-op for trivial inputs", "[layout]") { + SidecarData empty; + reorderSidecarByMorton(empty); + REQUIRE(empty.meshes.empty()); + + SidecarData one = buildFixture(); + one.meshes.resize(1); + const auto v = one.vertices; + reorderSidecarByMorton(one); // n < 2 path doesn't touch anything + REQUIRE(one.vertices == v); +}