mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-08-12 18:43:26 +00:00
ifcviewer: v16 zstd-compressed sidecars (~10x smaller over the wire)
The .ifcview data is hugely redundant (repeated double instance matrices, patterned indices) — measured 12x zstd whole-file. Server Content-Encoding can't be used (it breaks HTTP Range), so compress PER-CHUNK into the format. Format (v16): geometry becomes per-chunk zstd(vertices)+zstd(indices) frames — each independently Range-fetchable, so streaming is intact — and the critical + deferred metadata blocks are single zstd frames. SidecarChunk carries the compressed blob offsets/sizes; applyStreamedChunk (render/upload) is UNCHANGED — decompression slots into the fetch. Full readSidecar (test/tooling) reconstructs by decompress+scatter. zstd: desktop links libzstd (also compresses at bake); the web build (Emscripten has no zstd port) FetchContent's the pinned zstd source and compiles its decompress-only subset for wasm — no vendored blob, same version as desktop. New SidecarCompress wraps it (compress guarded off under Emscripten). Both stream paths — desktop StreamingThread worker + sync fallback (readChunkGeometryCompressed) and web beginWebChunkLoad — decompress; readSidecarMetadataOnly / the web bootstrap / loadDeferredMetadataWeb decompress the metadata blocks. streamingByteProgress reports COMPRESSED bytes. MEASURED: a 752 MB v15 federation → 75 MB v16 (10x; per-file 6.7-15.3x); PP-PLP 118→15 MB, loads 13/13 chunks on web, 0 errors. Three fixes found while testing big federations on a real server: - Web-streamed race: streaming_from_web was set in the deferred-header callback (a round-trip after the model+chunks exist), so driveStreamingLoads could take the sync fopen path meanwhile → "failed to read/decompress chunk 0". Now set immediately after applyCachedModel. - OOM abort on 18 models: the pool grew unbounded until an alloc failed, but on web that's an uncatchable bad_alloc abort. Cap total pool capacity (setMaxTotalCapacity, 3 GB) so it stops before the heap ceiling, and raise MAXIMUM_MEMORY 2→4 GB (wasm32 max) for headroom. - Web never evicted (grow-or-block only). At the hard budget, fall through to the LRU/priority evictor so a big federation stays navigable (highest-contribution chunks win) instead of freezing with holes. 113/113 desktop + 6/6 web smoke pass. No back-compat: regenerate sidecars (desktop bakes v16; scratch conv tool migrates v15→v16). Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Dion Moult
@@ -38,7 +38,7 @@
|
||||
# Better to keep the web path's machinery local to this directory.
|
||||
|
||||
cmake_minimum_required(VERSION 3.21)
|
||||
project(IfcViewerWeb LANGUAGES CXX)
|
||||
project(IfcViewerWeb LANGUAGES C CXX) # C for the vendored zstd decoder
|
||||
|
||||
if(NOT EMSCRIPTEN)
|
||||
message(FATAL_ERROR
|
||||
@@ -114,7 +114,7 @@ target_link_options(IfcViewerWeb PRIVATE
|
||||
# cap; --shared64 / MEMORY64 would lift this later if we need it).
|
||||
"-sALLOW_MEMORY_GROWTH=1"
|
||||
"-sINITIAL_MEMORY=268435456" # 256 MB
|
||||
"-sMAXIMUM_MEMORY=2147483648" # 2 GB
|
||||
"-sMAXIMUM_MEMORY=4294967296" # 4 GB (wasm32 max) — big federations need the headroom
|
||||
# FETCH lets emscripten_fetch issue HTTP Range requests. The local
|
||||
# file path (#88) reads byte ranges via Blob.slice and does NOT need
|
||||
# this; it's retained for the remote-URL Range backend (follow-up).
|
||||
|
||||
Binary file not shown.
@@ -109,7 +109,18 @@ public:
|
||||
// false the first time addSubBuffer is refused even at the floor
|
||||
// size — eviction callers need this to know whether a future alloc
|
||||
// could rescue them, or whether eviction is the only path.
|
||||
bool can_grow() const { return !growth_disabled_ && per_sub_buffer_capacity_ > 0; }
|
||||
bool can_grow() const {
|
||||
return !growth_disabled_ && per_sub_buffer_capacity_ > 0
|
||||
&& (max_total_capacity_bytes_ == 0
|
||||
|| total_capacity_bytes() < max_total_capacity_bytes_);
|
||||
}
|
||||
|
||||
// Hard ceiling on total pool capacity (0 = unlimited). Once total capacity
|
||||
// reaches this, can_grow() returns false so the streaming driver EVICTS
|
||||
// instead of growing. Critical on web: a growth past the wasm heap ceiling
|
||||
// is a bad_alloc that -fno-exceptions turns into an uncatchable abort, so
|
||||
// the async grow-OOM detection can't save us — we must stop first.
|
||||
void setMaxTotalCapacity(uint64_t max_bytes) { max_total_capacity_bytes_ = max_bytes; }
|
||||
|
||||
// Proactively add a sub-buffer (no allocation). On web this kicks off the
|
||||
// async provisional-validation cycle so validated free space appears a
|
||||
@@ -175,6 +186,7 @@ private:
|
||||
WGPUDevice device_ = nullptr;
|
||||
WGPUBufferUsage usage_ = 0;
|
||||
uint64_t per_sub_buffer_capacity_ = 0;
|
||||
uint64_t max_total_capacity_bytes_ = 0; // 0 = unlimited (see can_grow)
|
||||
// The largest size addSubBuffer last *succeeded* at, in bytes.
|
||||
// Starts at per_sub_buffer_capacity_ (the probe's discovered max)
|
||||
// and decays as the driver refuses larger allocations. Future grow
|
||||
|
||||
@@ -143,10 +143,46 @@ set(IFCVIEWER_CORE_SOURCES
|
||||
InstanceCompose.cpp
|
||||
LodBuilder.cpp
|
||||
SidecarCache.cpp
|
||||
SidecarCompress.cpp
|
||||
StreamingLoader.cpp
|
||||
StreamingThread.cpp
|
||||
ViewportCore.cpp
|
||||
)
|
||||
# Web needs a zstd DECODER (Emscripten has no zstd port; the desktop links the
|
||||
# full libzstd below). Rather than vendor a generated blob, fetch the pinned
|
||||
# zstd source and compile its decompress-only subset — the exact set zstd's
|
||||
# own single-file decoder inlines — straight into the wasm.
|
||||
if(EMSCRIPTEN)
|
||||
include(FetchContent)
|
||||
FetchContent_Declare(zstd_dec
|
||||
GIT_REPOSITORY https://github.com/facebook/zstd.git
|
||||
GIT_TAG v1.5.7
|
||||
GIT_SHALLOW TRUE
|
||||
# zstd's CMake lives in build/cmake, not the root — point SOURCE_SUBDIR
|
||||
# at a non-existent dir so MakeAvailable only POPULATES the source and
|
||||
# never tries to configure zstd's own (non-emscripten) build.
|
||||
SOURCE_SUBDIR does-not-exist)
|
||||
FetchContent_MakeAvailable(zstd_dec)
|
||||
set(ZSTD_DEC_DIR "${zstd_dec_SOURCE_DIR}/lib")
|
||||
set(ZSTD_DEC_SRC
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_decompress.c
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_decompress_block.c
|
||||
${ZSTD_DEC_DIR}/decompress/zstd_ddict.c
|
||||
${ZSTD_DEC_DIR}/decompress/huf_decompress.c
|
||||
${ZSTD_DEC_DIR}/common/entropy_common.c
|
||||
${ZSTD_DEC_DIR}/common/error_private.c
|
||||
${ZSTD_DEC_DIR}/common/fse_decompress.c
|
||||
${ZSTD_DEC_DIR}/common/pool.c
|
||||
${ZSTD_DEC_DIR}/common/threading.c
|
||||
${ZSTD_DEC_DIR}/common/xxhash.c
|
||||
${ZSTD_DEC_DIR}/common/zstd_common.c
|
||||
${ZSTD_DEC_DIR}/common/debug.c)
|
||||
# wasm isn't x86-64 → no BMI2 asm path (belt-and-braces disable).
|
||||
set_source_files_properties(${ZSTD_DEC_SRC} PROPERTIES
|
||||
COMPILE_DEFINITIONS "ZSTD_DISABLE_ASM=1")
|
||||
# NOT appended to IFCVIEWER_CORE_SOURCES — those get a source-dir PREPEND
|
||||
# below which would mangle these absolute paths; added via target_sources.
|
||||
endif()
|
||||
set(IFCVIEWER_CORE_HEADERS
|
||||
BufferPool.h
|
||||
CameraMath.h
|
||||
@@ -162,6 +198,7 @@ set(IFCVIEWER_CORE_HEADERS
|
||||
SectionPlane.h
|
||||
SelectionState.h
|
||||
SidecarCache.h
|
||||
SidecarCompress.h
|
||||
StreamingLoader.h
|
||||
StreamingThread.h
|
||||
VertexQuantization.h
|
||||
@@ -183,6 +220,19 @@ if(UNIX AND NOT APPLE)
|
||||
find_package(Threads REQUIRED)
|
||||
target_link_libraries(IfcViewerCore PUBLIC Threads::Threads)
|
||||
endif()
|
||||
# Sidecar (de)compression. Web compiles the vendored single-file decoder (added
|
||||
# to the sources above) + uses the vendored zstd.h; desktop links the full
|
||||
# libzstd (compress + decompress) and takes zstd.h from the system.
|
||||
if(EMSCRIPTEN)
|
||||
target_sources(IfcViewerCore PRIVATE ${ZSTD_DEC_SRC})
|
||||
target_include_directories(IfcViewerCore PRIVATE ${ZSTD_DEC_DIR})
|
||||
else()
|
||||
find_library(ZSTD_LIBRARY NAMES zstd libzstd)
|
||||
if(NOT ZSTD_LIBRARY)
|
||||
message(FATAL_ERROR "libzstd not found (needed for .ifcview compression)")
|
||||
endif()
|
||||
target_link_libraries(IfcViewerCore PUBLIC ${ZSTD_LIBRARY})
|
||||
endif()
|
||||
install(TARGETS IfcViewerCore EXPORT ${IFCOPENSHELL_EXPORT_TARGETS})
|
||||
|
||||
# IfcViewer: the Qt + IfcGeom + OpenCASCADE shell — everything in this
|
||||
|
||||
@@ -177,6 +177,14 @@ struct ModelGpuData {
|
||||
// is recovered by walking mesh_ids and the model's MeshInfo[].
|
||||
uint64_t vertex_byte_size = 0;
|
||||
uint64_t index_count = 0;
|
||||
// v16: where this chunk's two zstd frames live in the file's geometry
|
||||
// section (offsets relative to model.geometry_section_offset) and their
|
||||
// compressed sizes. The raw sizes are vertex_byte_size / index_count*4.
|
||||
// A per-chunk load fetches [off, +comp) and decompresses.
|
||||
uint64_t v_comp_off = 0;
|
||||
uint64_t v_comp_size = 0;
|
||||
uint64_t i_comp_off = 0;
|
||||
uint64_t i_comp_size = 0;
|
||||
// Of `index_count`, how many are LOD1 indices. LOD0 indices occupy
|
||||
// chunk-local u32 offsets [0, index_count - lod1_index_count); LOD1
|
||||
// indices occupy [index_count - lod1_index_count, index_count). 0
|
||||
@@ -286,8 +294,9 @@ struct ModelGpuData {
|
||||
// streaming path: chunks may be non-resident and need byte-range reads
|
||||
// from this file. Empty path = legacy non-streaming load.
|
||||
std::string streaming_file_path;
|
||||
uint64_t streaming_vertex_section_offset = 0;
|
||||
uint64_t streaming_index_section_offset = 0;
|
||||
// v16: file offset of the compressed geometry section. A chunk's blobs are
|
||||
// at geometry_section_offset + chunk.{v_comp_off,i_comp_off}.
|
||||
uint64_t geometry_section_offset = 0;
|
||||
// Web only: chunk byte ranges come from the JS-side source — a picked File
|
||||
// (Blob.slice) or a remote URL (HTTP Range) — read asynchronously, not via
|
||||
// a synchronous fopen on streaming_file_path. Set by loadSidecarMetadataWeb
|
||||
@@ -308,8 +317,11 @@ struct ModelGpuData {
|
||||
// it; deferred_meta_loaded latches so it fetches at most once.
|
||||
std::vector<PackedElementInfo> elements;
|
||||
std::string string_table;
|
||||
uint64_t deferred_meta_offset = 0;
|
||||
uint64_t deferred_meta_bytes = 0;
|
||||
// v16: the deferred block is a single zstd frame at deferred_comp_offset of
|
||||
// deferred_comp_size bytes, expanding to deferred_raw_size.
|
||||
uint64_t deferred_comp_offset = 0;
|
||||
uint64_t deferred_comp_size = 0;
|
||||
uint64_t deferred_raw_size = 0;
|
||||
bool deferred_meta_loaded = false;
|
||||
// applyCachedModel rebases instance object_ids by this base to keep them
|
||||
// globally unique across models; deferred elements carry the sidecar's
|
||||
|
||||
@@ -189,6 +189,7 @@ void SceneLoader::startNextLoad() {
|
||||
(long long)rt.elapsed(), ifc_path.c_str());
|
||||
auto result = std::make_shared<std::optional<StreamingSidecar>>(std::move(cached));
|
||||
QMetaObject::invokeMethod(this, [this, mid, result, is_sidecar_source]() {
|
||||
auto it = models_.find(mid);
|
||||
if (*result && !(*result)->meta.instances.empty()) {
|
||||
applySidecarData(mid, std::move(**result));
|
||||
if (!is_sidecar_source) {
|
||||
@@ -197,7 +198,6 @@ void SceneLoader::startNextLoad() {
|
||||
return;
|
||||
}
|
||||
|
||||
auto it = models_.find(mid);
|
||||
if (it == models_.end()) return;
|
||||
|
||||
if (is_sidecar_source) {
|
||||
@@ -236,10 +236,9 @@ void SceneLoader::applySidecarData(uint32_t mid, StreamingSidecar metadata) {
|
||||
SidecarData& d = metadata.meta;
|
||||
|
||||
std::fprintf(stderr,
|
||||
"[info] Sidecar hit: %s (%zu metadata bytes, %zu indices, %zu meshes, %zu instances, %zu elements)\n",
|
||||
"[info] Sidecar hit: %s (%zu chunks, %zu meshes, %zu instances, %zu elements)\n",
|
||||
model.file_path.toStdString().c_str(),
|
||||
size_t(metadata.vertex_total_bytes),
|
||||
size_t(metadata.index_total_count),
|
||||
d.chunks.size(),
|
||||
d.meshes.size(),
|
||||
d.instances.size(),
|
||||
d.elements.size());
|
||||
|
||||
+225
-68
@@ -43,10 +43,70 @@
|
||||
// char[string_table_bytes]
|
||||
|
||||
#include "SidecarCache.h"
|
||||
#include "SidecarCompress.h"
|
||||
|
||||
#include <cstdio>
|
||||
#include <cstring>
|
||||
|
||||
// The baker (writeSidecar) compresses — desktop only; the web build never bakes
|
||||
// and links a decompress-only zstd. Everything from here to writeSidecar's end
|
||||
// is guarded off under Emscripten.
|
||||
#if !defined(__EMSCRIPTEN__)
|
||||
|
||||
// zstd level for baking. 19 is near-max ratio; decode speed is level-
|
||||
// independent and the bake is offline, so favour ratio.
|
||||
static constexpr int kSidecarZstdLevel = 19;
|
||||
|
||||
// --- In-memory serialisation (a block is built in RAM, then compressed) ------
|
||||
template<typename T>
|
||||
static void appendVec(std::vector<std::uint8_t>& b, const std::vector<T>& v) {
|
||||
std::uint32_t n = static_cast<std::uint32_t>(v.size());
|
||||
const auto* np = reinterpret_cast<const std::uint8_t*>(&n);
|
||||
b.insert(b.end(), np, np + 4);
|
||||
if (n > 0) {
|
||||
const auto* p = reinterpret_cast<const std::uint8_t*>(v.data());
|
||||
b.insert(b.end(), p, p + std::size_t(sizeof(T)) * n);
|
||||
}
|
||||
}
|
||||
static void appendBytes(std::vector<std::uint8_t>& b, const void* p, std::size_t n) {
|
||||
const auto* c = static_cast<const std::uint8_t*>(p);
|
||||
b.insert(b.end(), c, c + n);
|
||||
}
|
||||
|
||||
// Pull one chunk's geometry out of the whole-model vertex/index arrays into the
|
||||
// chunk-LOCAL layout applyStreamedChunk expects: vertices of its meshes in chunk
|
||||
// order, then indices as LOD0 (per mesh) followed by LOD1 (per mesh).
|
||||
static void extractChunkGeometry(const SidecarData& d, const SidecarChunk& c,
|
||||
std::vector<std::uint8_t>& vbytes,
|
||||
std::vector<std::uint8_t>& ibytes) {
|
||||
vbytes.clear();
|
||||
ibytes.clear();
|
||||
const std::uint32_t end = c.first_mesh + c.mesh_count;
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = d.meshes[mi];
|
||||
const std::size_t voff = m.vbo_byte_offset;
|
||||
const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES;
|
||||
if (voff + vn <= d.vertices.size())
|
||||
vbytes.insert(vbytes.end(), d.vertices.begin() + voff,
|
||||
d.vertices.begin() + voff + vn);
|
||||
}
|
||||
auto appendIdx = [&](std::size_t first_u32, std::size_t count) {
|
||||
if (first_u32 + count > d.indices.size()) return;
|
||||
const auto* p = reinterpret_cast<const std::uint8_t*>(d.indices.data() + first_u32);
|
||||
ibytes.insert(ibytes.end(), p, p + count * sizeof(std::uint32_t));
|
||||
};
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = d.meshes[mi];
|
||||
if (m.index_count) appendIdx(m.ebo_byte_offset / sizeof(std::uint32_t), m.index_count);
|
||||
}
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < d.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = d.meshes[mi];
|
||||
if (m.lod1_index_count)
|
||||
appendIdx(m.lod1_ebo_byte_offset / sizeof(std::uint32_t), m.lod1_index_count);
|
||||
}
|
||||
}
|
||||
#endif // !__EMSCRIPTEN__ (bake-only serialisation helpers)
|
||||
|
||||
struct SidecarHeader {
|
||||
uint32_t magic;
|
||||
uint32_t version;
|
||||
@@ -86,100 +146,197 @@ static bool readVec(FILE* f, std::vector<T>& v) {
|
||||
return true;
|
||||
}
|
||||
|
||||
#if !defined(__EMSCRIPTEN__) // bake path — compresses, desktop only
|
||||
bool writeSidecar(const std::string& ifc_path, const SidecarData& data) {
|
||||
std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = fopen(path.c_str(), "wb");
|
||||
if (!f) return false;
|
||||
|
||||
auto wr = [&](const void* p, std::size_t n) {
|
||||
return fwrite(p, 1, n, f) == n;
|
||||
};
|
||||
auto wrU64 = [&](std::uint64_t v) { return wr(&v, sizeof(v)); };
|
||||
auto wrBlock = [&](const std::vector<std::uint8_t>& raw) -> bool {
|
||||
auto z = SidecarCompress::compress(raw.data(), raw.size(), kSidecarZstdLevel);
|
||||
if (raw.size() > 0 && z.empty()) return false; // compress failed
|
||||
return wrU64(z.size()) && wrU64(raw.size()) && (z.empty() || wr(z.data(), z.size()));
|
||||
};
|
||||
|
||||
SidecarHeader hdr = { SIDECAR_MAGIC, SIDECAR_VERSION, SIDECAR_ENDIAN };
|
||||
if (fwrite(&hdr, sizeof(hdr), 1, f) != 1) { fclose(f); return false; }
|
||||
if (!wr(&hdr, sizeof(hdr))) { fclose(f); return false; }
|
||||
|
||||
if (!writeVec(f, data.vertices)) { fclose(f); return false; }
|
||||
if (!writeVec(f, data.indices)) { fclose(f); return false; }
|
||||
// --- Geometry section: per-chunk zstd(vertex) + zstd(index) frames -------
|
||||
// Offsets in the chunk TOC are relative to the geometry section start, so
|
||||
// the loader range-fetches exactly one chunk without reading anything else.
|
||||
const long geom_len_pos = ftell(f);
|
||||
if (!wrU64(0)) { fclose(f); return false; } // geom_bytes placeholder
|
||||
const long geom_start = ftell(f);
|
||||
|
||||
// v15: a render-critical metadata block (meshes, instances, georef, chunk
|
||||
// TOC) preceded by its byte length, then a deferred block (elements +
|
||||
// string_table). The length lets the web loader read just the critical
|
||||
// block before painting and fetch the property data lazily / not at all.
|
||||
const long crit_len_pos = ftell(f);
|
||||
uint64_t crit_bytes = 0;
|
||||
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
|
||||
const long crit_start = ftell(f);
|
||||
|
||||
if (!writeVec(f, data.meshes)) { fclose(f); return false; }
|
||||
if (!writeVec(f, data.instances)) { fclose(f); return false; }
|
||||
// v11 georef block (148 B).
|
||||
if (fwrite(&data.has_coordinate_operation, 4, 1, f) != 1) { fclose(f); return false; }
|
||||
if (fwrite(data.coordinate_operation_meters,
|
||||
sizeof(double), 16, f) != 16) { fclose(f); return false; }
|
||||
if (fwrite(&data.project_length_to_meters,
|
||||
sizeof(double), 1, f) != 1) { fclose(f); return false; }
|
||||
if (fwrite(&data.map_unit_to_meters,
|
||||
sizeof(double), 1, f) != 1) { fclose(f); return false; }
|
||||
if (!writeVec(f, data.chunks)) { fclose(f); return false; }
|
||||
|
||||
// Backpatch the critical-block length.
|
||||
const long crit_end = ftell(f);
|
||||
if (crit_start < 0 || crit_end < 0) { fclose(f); return false; }
|
||||
crit_bytes = uint64_t(crit_end - crit_start);
|
||||
if (fseek(f, crit_len_pos, SEEK_SET) != 0) { fclose(f); return false; }
|
||||
if (fwrite(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) { fclose(f); return false; }
|
||||
if (fseek(f, crit_end, SEEK_SET) != 0) { fclose(f); return false; }
|
||||
|
||||
// Deferred block: element tree + string table (UI/picking, never rendered).
|
||||
if (!writeVec(f, data.elements)) { fclose(f); return false; }
|
||||
uint32_t stbl_len = static_cast<uint32_t>(data.string_table.size());
|
||||
if (fwrite(&stbl_len, 4, 1, f) != 1) { fclose(f); return false; }
|
||||
if (stbl_len > 0 && fwrite(data.string_table.data(), 1, stbl_len, f) != stbl_len) {
|
||||
fclose(f); return false;
|
||||
std::vector<SidecarChunk> chunks = data.chunks; // fill blob offsets below
|
||||
std::vector<std::uint8_t> vraw, iraw;
|
||||
for (auto& c : chunks) {
|
||||
extractChunkGeometry(data, c, vraw, iraw);
|
||||
auto vz = SidecarCompress::compress(vraw.data(), vraw.size(), kSidecarZstdLevel);
|
||||
auto iz = SidecarCompress::compress(iraw.data(), iraw.size(), kSidecarZstdLevel);
|
||||
if ((vraw.size() && vz.empty()) || (iraw.size() && iz.empty())) { fclose(f); return false; }
|
||||
c.v_comp_off = std::uint64_t(ftell(f) - geom_start);
|
||||
c.v_comp_size = vz.size();
|
||||
c.v_raw_size = vraw.size();
|
||||
if (!vz.empty() && !wr(vz.data(), vz.size())) { fclose(f); return false; }
|
||||
c.i_comp_off = std::uint64_t(ftell(f) - geom_start);
|
||||
c.i_comp_size = iz.size();
|
||||
c.i_raw_size = iraw.size();
|
||||
if (!iz.empty() && !wr(iz.data(), iz.size())) { fclose(f); return false; }
|
||||
}
|
||||
const long geom_end = ftell(f);
|
||||
if (geom_start < 0 || geom_end < 0) { fclose(f); return false; }
|
||||
if (fseek(f, geom_len_pos, SEEK_SET) != 0) { fclose(f); return false; }
|
||||
if (!wrU64(std::uint64_t(geom_end - geom_start))) { fclose(f); return false; }
|
||||
if (fseek(f, geom_end, SEEK_SET) != 0) { fclose(f); return false; }
|
||||
|
||||
// --- Critical metadata block (zstd): meshes, instances, georef, chunk TOC
|
||||
std::vector<std::uint8_t> crit;
|
||||
appendVec(crit, data.meshes);
|
||||
appendVec(crit, data.instances);
|
||||
appendBytes(crit, &data.has_coordinate_operation, 4);
|
||||
appendBytes(crit, data.coordinate_operation_meters, sizeof(double) * 16);
|
||||
appendBytes(crit, &data.project_length_to_meters, sizeof(double));
|
||||
appendBytes(crit, &data.map_unit_to_meters, sizeof(double));
|
||||
appendVec(crit, chunks);
|
||||
if (!wrBlock(crit)) { fclose(f); return false; }
|
||||
|
||||
// --- Deferred metadata block (zstd): element tree + string table ---------
|
||||
std::vector<std::uint8_t> def;
|
||||
appendVec(def, data.elements);
|
||||
std::uint32_t stbl_len = static_cast<std::uint32_t>(data.string_table.size());
|
||||
appendBytes(def, &stbl_len, 4);
|
||||
appendBytes(def, data.string_table.data(), stbl_len);
|
||||
if (!wrBlock(def)) { fclose(f); return false; }
|
||||
|
||||
fclose(f);
|
||||
return true;
|
||||
}
|
||||
#endif // !__EMSCRIPTEN__
|
||||
|
||||
// Cursor over an in-memory (decompressed) metadata block.
|
||||
namespace {
|
||||
struct BufReader {
|
||||
const std::uint8_t* p;
|
||||
std::size_t n;
|
||||
std::size_t pos = 0;
|
||||
bool take(void* dst, std::size_t k) {
|
||||
if (pos + k > n) return false;
|
||||
std::memcpy(dst, p + pos, k);
|
||||
pos += k;
|
||||
return true;
|
||||
}
|
||||
template <typename T>
|
||||
bool takeVec(std::vector<T>& v) {
|
||||
std::uint32_t c = 0;
|
||||
if (!take(&c, 4)) return false;
|
||||
if (pos + std::size_t(c) * sizeof(T) > n) return false;
|
||||
v.resize(c);
|
||||
if (c) { std::memcpy(v.data(), p + pos, std::size_t(c) * sizeof(T)); pos += std::size_t(c) * sizeof(T); }
|
||||
return true;
|
||||
}
|
||||
};
|
||||
} // namespace
|
||||
|
||||
// Full read: reconstruct the whole SidecarData (test/tooling path — the runtime
|
||||
// streams via readSidecarMetadataOnly + per-chunk loads and never calls this).
|
||||
// Decompresses the metadata blocks, then scatters each chunk's decompressed
|
||||
// geometry back into the whole-model vertex/index arrays using the mesh offsets.
|
||||
std::optional<SidecarData> readSidecar(const std::string& ifc_path) {
|
||||
std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = fopen(path.c_str(), "rb");
|
||||
if (!f) return std::nullopt;
|
||||
|
||||
auto fail = [&]() -> std::optional<SidecarData> { fclose(f); return std::nullopt; };
|
||||
|
||||
SidecarHeader hdr;
|
||||
if (fread(&hdr, sizeof(hdr), 1, f) != 1) return fail();
|
||||
if (hdr.magic != SIDECAR_MAGIC ||
|
||||
hdr.version != SIDECAR_VERSION ||
|
||||
if (hdr.magic != SIDECAR_MAGIC || hdr.version != SIDECAR_VERSION ||
|
||||
hdr.endian != SIDECAR_ENDIAN) return fail();
|
||||
|
||||
SidecarData data;
|
||||
if (!readVec(f, data.vertices)) return fail();
|
||||
if (!readVec(f, data.indices)) return fail();
|
||||
auto rd = [&](void* p, std::size_t k) { return fread(p, 1, k, f) == k; };
|
||||
auto rdU64 = [&](std::uint64_t& v) { return rd(&v, sizeof(v)); };
|
||||
|
||||
// v15 critical-block length (consumed; only the streaming/web readers need
|
||||
// it for a one-shot range read — here we read sequentially).
|
||||
uint64_t crit_bytes = 0;
|
||||
if (fread(&crit_bytes, sizeof(crit_bytes), 1, f) != 1) return fail();
|
||||
|
||||
// Critical block: meshes, instances, georef, chunk TOC.
|
||||
if (!readVec(f, data.meshes)) return fail();
|
||||
if (!readVec(f, data.instances)) return fail();
|
||||
if (fread(&data.has_coordinate_operation, 4, 1, f) != 1) return fail();
|
||||
if (fread(data.coordinate_operation_meters,
|
||||
sizeof(double), 16, f) != 16) return fail();
|
||||
if (fread(&data.project_length_to_meters,
|
||||
sizeof(double), 1, f) != 1) return fail();
|
||||
if (fread(&data.map_unit_to_meters,
|
||||
sizeof(double), 1, f) != 1) return fail();
|
||||
if (!readVec(f, data.chunks)) return fail();
|
||||
|
||||
// Deferred block: element tree + string table.
|
||||
if (!readVec(f, data.elements)) return fail();
|
||||
uint32_t stbl_len;
|
||||
if (fread(&stbl_len, 4, 1, f) != 1) return fail();
|
||||
data.string_table.resize(stbl_len);
|
||||
if (stbl_len > 0 && fread(data.string_table.data(), 1, stbl_len, f) != stbl_len)
|
||||
return fail();
|
||||
std::uint64_t geom_bytes = 0;
|
||||
if (!rdU64(geom_bytes)) return fail();
|
||||
std::vector<std::uint8_t> geom(static_cast<std::size_t>(geom_bytes));
|
||||
if (geom_bytes && !rd(geom.data(), geom.size())) return fail();
|
||||
|
||||
auto readBlock = [&](std::vector<std::uint8_t>& out) -> bool {
|
||||
std::uint64_t comp = 0, raw = 0;
|
||||
if (!rdU64(comp) || !rdU64(raw)) return false;
|
||||
std::vector<std::uint8_t> z(static_cast<std::size_t>(comp));
|
||||
if (comp && !rd(z.data(), z.size())) return false;
|
||||
out.assign(std::size_t(raw), 0);
|
||||
return SidecarCompress::decompress(z.data(), z.size(), out.data(), out.size());
|
||||
};
|
||||
std::vector<std::uint8_t> crit, def;
|
||||
if (!readBlock(crit) || !readBlock(def)) return fail();
|
||||
fclose(f);
|
||||
|
||||
SidecarData data;
|
||||
BufReader cr{ crit.data(), crit.size() };
|
||||
if (!cr.takeVec(data.meshes)) return std::nullopt;
|
||||
if (!cr.takeVec(data.instances)) return std::nullopt;
|
||||
if (!cr.take(&data.has_coordinate_operation, 4)) return std::nullopt;
|
||||
if (!cr.take(data.coordinate_operation_meters, sizeof(double) * 16)) return std::nullopt;
|
||||
if (!cr.take(&data.project_length_to_meters, sizeof(double))) return std::nullopt;
|
||||
if (!cr.take(&data.map_unit_to_meters, sizeof(double))) return std::nullopt;
|
||||
if (!cr.takeVec(data.chunks)) return std::nullopt;
|
||||
|
||||
BufReader dr{ def.data(), def.size() };
|
||||
if (!dr.takeVec(data.elements)) return std::nullopt;
|
||||
std::uint32_t stbl_len = 0;
|
||||
if (!dr.take(&stbl_len, 4)) return std::nullopt;
|
||||
data.string_table.resize(stbl_len);
|
||||
if (stbl_len && !dr.take(data.string_table.data(), stbl_len)) return std::nullopt;
|
||||
|
||||
// Reconstruct the whole-model vertex/index arrays from the per-chunk blobs.
|
||||
std::size_t vsize = 0, isize = 0;
|
||||
for (const auto& m : data.meshes) {
|
||||
vsize = std::max<std::size_t>(vsize,
|
||||
std::size_t(m.vbo_byte_offset) + std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES);
|
||||
isize = std::max<std::size_t>(isize, m.ebo_byte_offset / sizeof(std::uint32_t) + m.index_count);
|
||||
if (m.lod1_index_count)
|
||||
isize = std::max<std::size_t>(isize, m.lod1_ebo_byte_offset / sizeof(std::uint32_t) + m.lod1_index_count);
|
||||
}
|
||||
data.vertices.assign(vsize, 0);
|
||||
data.indices.assign(isize, 0);
|
||||
for (const auto& c : data.chunks) {
|
||||
if (c.v_comp_off + c.v_comp_size > geom.size() ||
|
||||
c.i_comp_off + c.i_comp_size > geom.size()) return std::nullopt;
|
||||
std::vector<std::uint8_t> vraw(static_cast<std::size_t>(c.v_raw_size));
|
||||
std::vector<std::uint8_t> iraw(static_cast<std::size_t>(c.i_raw_size));
|
||||
if (!SidecarCompress::decompress(geom.data() + c.v_comp_off, c.v_comp_size, vraw.data(), vraw.size()) ||
|
||||
!SidecarCompress::decompress(geom.data() + c.i_comp_off, c.i_comp_size, iraw.data(), iraw.size()))
|
||||
return std::nullopt;
|
||||
const auto* iu = reinterpret_cast<const std::uint32_t*>(iraw.data());
|
||||
std::size_t vcur = 0, icur = 0;
|
||||
const std::uint32_t end = c.first_mesh + c.mesh_count;
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = data.meshes[mi];
|
||||
const std::size_t vn = std::size_t(m.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES;
|
||||
if (vcur + vn <= vraw.size() && m.vbo_byte_offset + vn <= data.vertices.size())
|
||||
std::memcpy(&data.vertices[m.vbo_byte_offset], vraw.data() + vcur, vn);
|
||||
vcur += vn;
|
||||
}
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = data.meshes[mi];
|
||||
if (!m.index_count) continue;
|
||||
if (icur + m.index_count <= iraw.size() / 4)
|
||||
std::memcpy(&data.indices[m.ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.index_count * 4);
|
||||
icur += m.index_count;
|
||||
}
|
||||
for (std::uint32_t mi = c.first_mesh; mi < end && mi < data.meshes.size(); ++mi) {
|
||||
const MeshInfo& m = data.meshes[mi];
|
||||
if (!m.lod1_index_count) continue;
|
||||
if (icur + m.lod1_index_count <= iraw.size() / 4)
|
||||
std::memcpy(&data.indices[m.lod1_ebo_byte_offset / sizeof(std::uint32_t)], iu + icur, m.lod1_index_count * 4);
|
||||
icur += m.lod1_index_count;
|
||||
}
|
||||
}
|
||||
return data;
|
||||
}
|
||||
|
||||
@@ -81,16 +81,32 @@ static constexpr uint32_t SIDECAR_MAGIC = 0x49465657; // "IFVW"
|
||||
// painting, so first geometry no longer waits on the property data; the
|
||||
// deferred block is fetched lazily (or skipped where unused). Desktop
|
||||
// reads both. No back-compat: regenerate sidecars.
|
||||
static constexpr uint32_t SIDECAR_VERSION = 15;
|
||||
// v16 = Geometry + metadata are zstd-COMPRESSED. Each chunk's vertex bytes and
|
||||
// index bytes are stored as two independent zstd frames (so per-chunk
|
||||
// Range streaming still works — you fetch + decompress just one chunk),
|
||||
// and the critical + deferred metadata blocks are single zstd frames.
|
||||
// The chunk TOC records each chunk's compressed blob offsets/sizes plus
|
||||
// the raw (decompressed) sizes. ~3-5x fewer bytes over the wire while
|
||||
// keeping HTTP Range intact (unlike server Content-Encoding). No
|
||||
// back-compat: regenerate sidecars.
|
||||
static constexpr uint32_t SIDECAR_VERSION = 16;
|
||||
static constexpr uint32_t SIDECAR_ENDIAN = 0x01020304;
|
||||
|
||||
// Chunk table-of-contents entry (v14+). A chunk is a CONTIGUOUS range of
|
||||
// meshes in the (reordered) meshes array — and therefore a contiguous span of
|
||||
// vertex + index bytes, since the geometry is laid out in chunk order. The
|
||||
// loader builds chunk `i` from meshes [first_mesh, first_mesh + mesh_count).
|
||||
// Chunk table-of-contents entry (v16). A chunk is a CONTIGUOUS range of meshes
|
||||
// [first_mesh, first_mesh + mesh_count). Its vertex + index bytes are stored as
|
||||
// two zstd frames in the geometry section; the loader fetches [v_comp_off,
|
||||
// +v_comp_size) / [i_comp_off, +i_comp_size) (offsets relative to the geometry
|
||||
// section start) and decompresses them to v_raw_size / i_raw_size bytes — the
|
||||
// chunk-local (vbytes, idx) applyStreamedChunk consumes.
|
||||
struct SidecarChunk {
|
||||
uint32_t first_mesh;
|
||||
uint32_t mesh_count;
|
||||
uint64_t v_comp_off;
|
||||
uint64_t v_comp_size;
|
||||
uint64_t v_raw_size;
|
||||
uint64_t i_comp_off;
|
||||
uint64_t i_comp_size;
|
||||
uint64_t i_raw_size;
|
||||
};
|
||||
|
||||
// Fixed-size element record. Strings are stored as (offset, length) pairs
|
||||
|
||||
@@ -0,0 +1,46 @@
|
||||
/********************************************************************************
|
||||
* *
|
||||
* This file is part of IfcOpenShell. *
|
||||
* *
|
||||
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the Lesser GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3.0 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* IfcOpenShell is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* Lesser GNU General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the Lesser GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
********************************************************************************/
|
||||
|
||||
#include "SidecarCompress.h"
|
||||
|
||||
#include <zstd.h>
|
||||
|
||||
namespace SidecarCompress {
|
||||
|
||||
bool decompress(const std::uint8_t* src, std::size_t src_size,
|
||||
std::uint8_t* dst, std::size_t raw_size) {
|
||||
if (raw_size == 0) return src_size == 0; // empty in ↔ empty out
|
||||
if (!src || !dst || src_size == 0) return false;
|
||||
const size_t got = ZSTD_decompress(dst, raw_size, src, src_size);
|
||||
return !ZSTD_isError(got) && got == raw_size;
|
||||
}
|
||||
|
||||
#if !defined(__EMSCRIPTEN__)
|
||||
std::vector<std::uint8_t> compress(const std::uint8_t* src, std::size_t n,
|
||||
int level) {
|
||||
if (n == 0) return {};
|
||||
std::vector<std::uint8_t> out(ZSTD_compressBound(n));
|
||||
const size_t got = ZSTD_compress(out.data(), out.size(), src, n, level);
|
||||
if (ZSTD_isError(got)) return {};
|
||||
out.resize(got);
|
||||
return out;
|
||||
}
|
||||
#endif
|
||||
|
||||
} // namespace SidecarCompress
|
||||
@@ -0,0 +1,51 @@
|
||||
/********************************************************************************
|
||||
* *
|
||||
* This file is part of IfcOpenShell. *
|
||||
* *
|
||||
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the Lesser GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3.0 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* IfcOpenShell is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* Lesser GNU General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the Lesser GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
********************************************************************************/
|
||||
|
||||
#ifndef SIDECARCOMPRESS_H
|
||||
#define SIDECARCOMPRESS_H
|
||||
|
||||
// zstd wrappers for the .ifcview format (v16+). Geometry chunks and metadata
|
||||
// blocks are stored zstd-compressed so the network pulls far fewer bytes while
|
||||
// keeping HTTP Range streaming intact (unlike server Content-Encoding, which
|
||||
// can't be byte-ranged). The baker compresses (desktop only); every loader —
|
||||
// desktop and web — decompresses. The web build links the vendored single-file
|
||||
// zstd DECODER (third_party/zstddeclib.c); desktop links the full libzstd.
|
||||
|
||||
#include <cstddef>
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
namespace SidecarCompress {
|
||||
|
||||
// Decompress a zstd frame in [src, src+src_size) into dst, which must have room
|
||||
// for exactly raw_size bytes. Returns false on any zstd error or if the frame
|
||||
// doesn't expand to exactly raw_size. Available on all platforms.
|
||||
bool decompress(const std::uint8_t* src, std::size_t src_size,
|
||||
std::uint8_t* dst, std::size_t raw_size);
|
||||
|
||||
#if !defined(__EMSCRIPTEN__)
|
||||
// Compress [src, src+n) with zstd at `level`. Returns the compressed frame, or
|
||||
// an empty vector on error. Bake/desktop only — the web build never compresses.
|
||||
std::vector<std::uint8_t> compress(const std::uint8_t* src, std::size_t n,
|
||||
int level);
|
||||
#endif
|
||||
|
||||
} // namespace SidecarCompress
|
||||
|
||||
#endif // SIDECARCOMPRESS_H
|
||||
@@ -34,6 +34,7 @@
|
||||
// can be range-read on demand. File handle is closed before return.
|
||||
|
||||
#include "StreamingLoader.h"
|
||||
#include "SidecarCompress.h"
|
||||
|
||||
#include <algorithm>
|
||||
#include <cstdio>
|
||||
@@ -88,14 +89,14 @@ std::string sidecarPath(const std::string& ifc_path) {
|
||||
|
||||
} // namespace
|
||||
|
||||
bool parseSidecarHead(const uint8_t* data, size_t n, uint32_t& out_num_vertex_bytes) {
|
||||
bool parseSidecarHead(const uint8_t* data, size_t n, uint64_t& out_geom_bytes) {
|
||||
if (n < SIDECAR_HEAD_BYTES) return false;
|
||||
SidecarHeaderRaw hdr;
|
||||
std::memcpy(&hdr, data, sizeof(hdr));
|
||||
if (hdr.magic != SIDECAR_MAGIC) return false;
|
||||
if (hdr.version != SIDECAR_VERSION) return false;
|
||||
if (hdr.endian != SIDECAR_ENDIAN) return false;
|
||||
std::memcpy(&out_num_vertex_bytes, data + sizeof(hdr), 4);
|
||||
std::memcpy(&out_geom_bytes, data + sizeof(hdr), 8);
|
||||
return true;
|
||||
}
|
||||
|
||||
@@ -134,58 +135,79 @@ std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_p
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
// Head: 12-byte header + the vertex-byte count. The vertex section starts
|
||||
// immediately after, at SIDECAR_HEAD_BYTES.
|
||||
// Head (v16): 12-byte header + the compressed-geometry-section length. The
|
||||
// metadata blocks follow the geometry at SIDECAR_HEAD_BYTES + geom_bytes.
|
||||
uint8_t head[SIDECAR_HEAD_BYTES];
|
||||
if (std::fread(head, 1, SIDECAR_HEAD_BYTES, f) != SIDECAR_HEAD_BYTES) return fail();
|
||||
uint32_t num_vertex_bytes = 0;
|
||||
if (!parseSidecarHead(head, SIDECAR_HEAD_BYTES, num_vertex_bytes)) return fail();
|
||||
uint64_t geom_bytes = 0;
|
||||
if (!parseSidecarHead(head, SIDECAR_HEAD_BYTES, geom_bytes)) return fail();
|
||||
|
||||
StreamingSidecar out;
|
||||
out.file_path = path;
|
||||
out.vertex_section_offset = SIDECAR_HEAD_BYTES;
|
||||
out.vertex_total_bytes = num_vertex_bytes;
|
||||
out.file_path = path;
|
||||
out.geometry_section_offset = SIDECAR_HEAD_BYTES;
|
||||
|
||||
// Skip the vertex section; read the index count that follows it.
|
||||
if (std::fseek(f, long(num_vertex_bytes), SEEK_CUR) != 0) return fail();
|
||||
uint32_t num_indices = 0;
|
||||
if (std::fread(&num_indices, 4, 1, f) != 1) return fail();
|
||||
out.index_section_offset = uint64_t(std::ftell(f));
|
||||
out.index_total_count = num_indices;
|
||||
|
||||
// Skip the index section; the metadata tail runs from there to EOF.
|
||||
if (std::fseek(f, long(num_indices) * 4, SEEK_CUR) != 0) return fail();
|
||||
const long tail_off = std::ftell(f);
|
||||
if (tail_off < 0) return fail();
|
||||
if (std::fseek(f, 0, SEEK_END) != 0) return fail();
|
||||
const long file_end = std::ftell(f);
|
||||
if (file_end < tail_off) return fail();
|
||||
if (std::fseek(f, tail_off, SEEK_SET) != 0) return fail();
|
||||
|
||||
std::vector<uint8_t> tail(size_t(file_end - tail_off));
|
||||
if (!tail.empty() && std::fread(tail.data(), 1, tail.size(), f) != tail.size())
|
||||
// Skip the geometry section; the two compressed metadata blocks follow.
|
||||
if (std::fseek(f, long(SIDECAR_HEAD_BYTES) + long(geom_bytes), SEEK_SET) != 0)
|
||||
return fail();
|
||||
|
||||
// Each metadata block on disk is [comp u64][raw u64][zstd frame].
|
||||
auto readBlock = [&](std::vector<uint8_t>& raw,
|
||||
uint64_t* comp_off = nullptr, uint64_t* comp_sz = nullptr,
|
||||
uint64_t* raw_sz = nullptr) -> bool {
|
||||
uint64_t comp = 0, rawn = 0;
|
||||
if (std::fread(&comp, 8, 1, f) != 1 || std::fread(&rawn, 8, 1, f) != 1) return false;
|
||||
const long here = std::ftell(f);
|
||||
std::vector<uint8_t> z(static_cast<size_t>(comp));
|
||||
if (comp && std::fread(z.data(), 1, z.size(), f) != z.size()) return false;
|
||||
raw.assign(size_t(rawn), 0);
|
||||
if (comp_off) *comp_off = uint64_t(here);
|
||||
if (comp_sz) *comp_sz = comp;
|
||||
if (raw_sz) *raw_sz = rawn;
|
||||
return SidecarCompress::decompress(z.data(), z.size(), raw.data(), raw.size());
|
||||
};
|
||||
|
||||
std::vector<uint8_t> crit, def;
|
||||
if (!readBlock(crit)) return fail();
|
||||
if (!readBlock(def, &out.deferred_comp_offset, &out.deferred_comp_size,
|
||||
&out.deferred_raw_size)) return fail();
|
||||
std::fclose(f);
|
||||
|
||||
// Tail (v15) = [critical_meta_bytes (8)][critical block][deferred block].
|
||||
// Desktop is local, so read both; the web path reads only the critical
|
||||
// block before painting and the deferred block on demand.
|
||||
if (tail.size() < sizeof(uint64_t)) return std::nullopt;
|
||||
uint64_t crit_bytes = 0;
|
||||
std::memcpy(&crit_bytes, tail.data(), sizeof(crit_bytes));
|
||||
const size_t crit_off = sizeof(crit_bytes);
|
||||
if (crit_off + crit_bytes > tail.size()) return std::nullopt;
|
||||
out.critical_meta_offset = out.index_section_offset
|
||||
+ uint64_t(num_indices) * 4u + crit_off;
|
||||
out.critical_meta_bytes = crit_bytes;
|
||||
if (!parseSidecarCritical(tail.data() + crit_off, size_t(crit_bytes), out.meta))
|
||||
return std::nullopt;
|
||||
if (!parseSidecarDeferred(tail.data() + crit_off + crit_bytes,
|
||||
tail.size() - crit_off - size_t(crit_bytes), out.meta))
|
||||
return std::nullopt;
|
||||
// Desktop reads both blocks up front; the web path reads only critical
|
||||
// before painting and fetches the deferred block on demand.
|
||||
if (!parseSidecarCritical(crit.data(), crit.size(), out.meta)) return std::nullopt;
|
||||
if (!parseSidecarDeferred(def.data(), def.size(), out.meta)) return std::nullopt;
|
||||
return out;
|
||||
}
|
||||
|
||||
bool readChunkGeometryCompressed(const std::string& ifc_path,
|
||||
std::uint64_t geometry_section_offset,
|
||||
std::uint64_t v_comp_off, std::uint64_t v_comp_size,
|
||||
std::uint64_t v_raw_size,
|
||||
std::uint64_t i_comp_off, std::uint64_t i_comp_size,
|
||||
std::uint64_t i_raw_size,
|
||||
std::vector<std::uint8_t>& out_vbytes,
|
||||
std::vector<std::uint32_t>& out_idx) {
|
||||
const std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = std::fopen(path.c_str(), "rb");
|
||||
if (!f) return false;
|
||||
auto readFrame = [&](std::uint64_t off, std::uint64_t comp, std::uint64_t raw,
|
||||
std::uint8_t* dst) -> bool {
|
||||
if (raw == 0) return comp == 0;
|
||||
std::vector<std::uint8_t> z(static_cast<size_t>(comp));
|
||||
if (std::fseek(f, long(geometry_section_offset + off), SEEK_SET) != 0) return false;
|
||||
if (comp && std::fread(z.data(), 1, z.size(), f) != z.size()) return false;
|
||||
return SidecarCompress::decompress(z.data(), z.size(), dst, size_t(raw));
|
||||
};
|
||||
out_vbytes.assign(size_t(v_raw_size), 0);
|
||||
out_idx.assign(size_t(i_raw_size / sizeof(std::uint32_t)), 0);
|
||||
const bool ok =
|
||||
readFrame(v_comp_off, v_comp_size, v_raw_size, out_vbytes.data()) &&
|
||||
readFrame(i_comp_off, i_comp_size, i_raw_size,
|
||||
reinterpret_cast<std::uint8_t*>(out_idx.data()));
|
||||
std::fclose(f);
|
||||
return ok;
|
||||
}
|
||||
|
||||
bool readSidecarVertexChunk(const std::string& ifc_path,
|
||||
uint64_t vertex_section_offset,
|
||||
uint64_t chunk_byte_offset,
|
||||
|
||||
@@ -46,25 +46,20 @@
|
||||
struct StreamingSidecar {
|
||||
// Everything except vertices + indices — same shape as SidecarData but
|
||||
// with empty vertices / indices vectors. The renderer uses meshes /
|
||||
// instances / georef / elements immediately.
|
||||
// instances / georef / chunks immediately (elements/strings deferred).
|
||||
SidecarData meta;
|
||||
|
||||
// Byte offsets in the on-disk file where the vertex and index sections
|
||||
// start (after their 4-byte count headers). Pair with vertex_total_bytes
|
||||
// / index_total_bytes for the section length; per-chunk reads slice
|
||||
// arbitrary ranges within these.
|
||||
uint64_t vertex_section_offset = 0;
|
||||
uint64_t vertex_total_bytes = 0;
|
||||
uint64_t index_section_offset = 0;
|
||||
uint64_t index_total_count = 0; // u32 indices, NOT bytes
|
||||
// v16: the compressed geometry section starts here. Each chunk's two zstd
|
||||
// blobs live at geometry_section_offset + SidecarChunk.{v_comp_off,i_comp_off};
|
||||
// a per-chunk load fetches [that, +*_comp_size) and decompresses to *_raw_size.
|
||||
uint64_t geometry_section_offset = 0;
|
||||
|
||||
// v15 deferred-metadata locator. The render-critical metadata block starts
|
||||
// at critical_meta_offset and is critical_meta_bytes long; the deferred
|
||||
// block (elements + string_table) runs from there to EOF. The web loader
|
||||
// reads only the critical block before painting and fetches the deferred
|
||||
// block on demand from [critical_meta_offset + critical_meta_bytes, EOF).
|
||||
uint64_t critical_meta_offset = 0;
|
||||
uint64_t critical_meta_bytes = 0;
|
||||
// v16 deferred (property) block locator: a single zstd frame at
|
||||
// deferred_comp_offset of deferred_comp_size bytes → deferred_raw_size. The
|
||||
// web loader fetches it on demand (elements/strings); desktop reads it up front.
|
||||
uint64_t deferred_comp_offset = 0;
|
||||
uint64_t deferred_comp_size = 0;
|
||||
uint64_t deferred_raw_size = 0;
|
||||
|
||||
// Resolved on-disk path so subsequent chunk reads can re-open / seek.
|
||||
std::string file_path;
|
||||
@@ -75,6 +70,20 @@ struct StreamingSidecar {
|
||||
// before return — callers re-open for per-chunk reads.
|
||||
std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_path);
|
||||
|
||||
// Read + decompress one chunk's geometry (v16) from disk: the vertex zstd frame
|
||||
// at [geometry_section_offset + v_comp_off, +v_comp_size) → out_vbytes (v_raw
|
||||
// bytes) and the index frame → out_idx (i_raw/4 u32s). Returns false on I/O or
|
||||
// decompress failure. Used by the desktop StreamingThread worker + the sync
|
||||
// first-frame fallback; the web path decompresses in beginWebChunkLoad instead.
|
||||
bool readChunkGeometryCompressed(const std::string& ifc_path,
|
||||
std::uint64_t geometry_section_offset,
|
||||
std::uint64_t v_comp_off, std::uint64_t v_comp_size,
|
||||
std::uint64_t v_raw_size,
|
||||
std::uint64_t i_comp_off, std::uint64_t i_comp_size,
|
||||
std::uint64_t i_raw_size,
|
||||
std::vector<std::uint8_t>& out_vbytes,
|
||||
std::vector<std::uint32_t>& out_idx);
|
||||
|
||||
// --- Pure, buffer-based building blocks ------------------------------------
|
||||
//
|
||||
// The metadata lives in two disjoint regions of the file: a small fixed
|
||||
@@ -85,15 +94,15 @@ std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_p
|
||||
// source and hand the bytes to these parsers, so the wire-format knowledge
|
||||
// lives in exactly one place and is unit-testable without touching a file.
|
||||
|
||||
// Bytes the head spans: SidecarHeader (12) + uint32 num_vertex_bytes (4).
|
||||
inline constexpr std::size_t SIDECAR_HEAD_BYTES = 16;
|
||||
// Bytes the head spans: SidecarHeader (12) + uint64 geometry-section length (8).
|
||||
inline constexpr std::size_t SIDECAR_HEAD_BYTES = 20;
|
||||
|
||||
// Parse the 16-byte head. Validates magic / version / endian and, on success,
|
||||
// writes the vertex-section byte count (which locates the index-count field at
|
||||
// SIDECAR_HEAD_BYTES + out_num_vertex_bytes). Returns false if `n` is short or
|
||||
// the header is wrong. `data` must point at the start of the file.
|
||||
// Parse the 20-byte head (v16). Validates magic / version / endian and, on
|
||||
// success, writes the compressed-geometry-section byte length (the metadata
|
||||
// blocks follow at SIDECAR_HEAD_BYTES + out_geom_bytes). Returns false if `n`
|
||||
// is short or the header is wrong. `data` must point at the start of the file.
|
||||
bool parseSidecarHead(const std::uint8_t* data, std::size_t n,
|
||||
std::uint32_t& out_num_vertex_bytes);
|
||||
std::uint64_t& out_geom_bytes);
|
||||
|
||||
// Parse the v15 render-CRITICAL metadata block (mesh dict, instance dict,
|
||||
// georef, chunk TOC) — everything needed to set up + draw the scene. `data`
|
||||
|
||||
@@ -97,21 +97,11 @@ void StreamingThread::workerLoop() {
|
||||
Result res;
|
||||
res.model_id = req.model_id;
|
||||
res.chunk_idx = req.chunk_idx;
|
||||
res.success = true;
|
||||
if (!req.v_ranges.empty()) {
|
||||
if (!readSidecarVertexRanges(req.file_path,
|
||||
req.vertex_section_offset,
|
||||
req.v_ranges, res.vbytes)) {
|
||||
res.success = false;
|
||||
}
|
||||
}
|
||||
if (res.success && !req.i_ranges.empty()) {
|
||||
if (!readSidecarIndexRanges(req.file_path,
|
||||
req.index_section_offset,
|
||||
req.i_ranges, res.idx)) {
|
||||
res.success = false;
|
||||
}
|
||||
}
|
||||
res.success = readChunkGeometryCompressed(
|
||||
req.file_path, req.geometry_section_offset,
|
||||
req.v_comp_off, req.v_comp_size, req.v_raw_size,
|
||||
req.i_comp_off, req.i_comp_size, req.i_raw_size,
|
||||
res.vbytes, res.idx);
|
||||
|
||||
{
|
||||
std::unique_lock lk(mu_);
|
||||
|
||||
@@ -44,15 +44,15 @@
|
||||
class StreamingThread {
|
||||
public:
|
||||
struct Request {
|
||||
uint32_t model_id;
|
||||
std::size_t chunk_idx;
|
||||
std::string file_path;
|
||||
uint64_t vertex_section_offset;
|
||||
uint64_t index_section_offset;
|
||||
// (section-relative byte_offset, byte_size)
|
||||
std::vector<std::pair<uint64_t, uint64_t>> v_ranges;
|
||||
// (first_u32, count_u32)
|
||||
std::vector<std::pair<uint64_t, uint64_t>> i_ranges;
|
||||
uint32_t model_id;
|
||||
std::size_t chunk_idx;
|
||||
std::string file_path;
|
||||
// v16: the chunk's two zstd frames in the geometry section. The reader
|
||||
// fetches [geometry_section_offset + *_comp_off, +*_comp_size) and
|
||||
// decompresses to *_raw_size.
|
||||
uint64_t geometry_section_offset = 0;
|
||||
uint64_t v_comp_off = 0, v_comp_size = 0, v_raw_size = 0;
|
||||
uint64_t i_comp_off = 0, i_comp_size = 0, i_raw_size = 0;
|
||||
};
|
||||
|
||||
struct Result {
|
||||
|
||||
+175
-198
@@ -1384,6 +1384,15 @@ bool ViewportCore::createPool() {
|
||||
std::max<uint64_t>(MIN_POOL_CAPACITY, INITIAL_SUB_BUFFER));
|
||||
pool_.configure(instance_, device_, pool_usage, per_sub,
|
||||
"ifcviewer-wgpu.pool");
|
||||
#if defined(__EMSCRIPTEN__)
|
||||
// Cap total pool capacity below the wasm heap ceiling. On web a growth that
|
||||
// would push the heap past MAXIMUM_MEMORY is a bad_alloc → uncatchable
|
||||
// abort, so the pool must stop growing (and evict) before then. Leave
|
||||
// headroom for metadata (instances/maps), transient decompression buffers,
|
||||
// and wgpu overhead. Big federations then keep a bounded, highest-priority
|
||||
// resident set instead of aborting.
|
||||
pool_.setMaxTotalCapacity(3072ull * 1024 * 1024); // 3 GB (heap ceiling 4 GB)
|
||||
#endif
|
||||
Log::info() << "wgpu: pool per-sub-buffer capacity = "
|
||||
<< (per_sub / (1024 * 1024)) << " MB (grows lazily on "
|
||||
<< "demand; device maxBufferSize = "
|
||||
@@ -1723,6 +1732,7 @@ void ViewportCore::shutdown() {
|
||||
// ===========================================================================
|
||||
|
||||
#include "StreamingLoader.h"
|
||||
#include "SidecarCompress.h"
|
||||
|
||||
namespace {
|
||||
|
||||
@@ -1941,37 +1951,17 @@ StreamingThread::Request ViewportCore::makeChunkRequest(
|
||||
std::uint32_t model_id) {
|
||||
const auto& c = m.chunks[chunk_idx];
|
||||
StreamingThread::Request req;
|
||||
req.model_id = model_id;
|
||||
req.chunk_idx = chunk_idx;
|
||||
req.file_path = m.streaming_file_path;
|
||||
req.vertex_section_offset = m.streaming_vertex_section_offset;
|
||||
req.index_section_offset = m.streaming_index_section_offset;
|
||||
req.v_ranges.reserve(c.mesh_ids.size());
|
||||
req.i_ranges.reserve(c.mesh_ids.size());
|
||||
for (std::uint32_t mi : c.mesh_ids) {
|
||||
const MeshInfo& mesh = m.meshes[mi];
|
||||
const std::uint64_t v_bytes =
|
||||
std::uint64_t(mesh.vertex_count) * INSTANCED_VERTEX_STRIDE_BYTES;
|
||||
if (v_bytes > 0) {
|
||||
req.v_ranges.emplace_back(std::uint64_t(mesh.vbo_byte_offset), v_bytes);
|
||||
}
|
||||
if (mesh.index_count > 0) {
|
||||
req.i_ranges.emplace_back(
|
||||
std::uint64_t(mesh.ebo_byte_offset / sizeof(std::uint32_t)),
|
||||
std::uint64_t(mesh.index_count));
|
||||
}
|
||||
}
|
||||
// LOD1 indices second pass — matches the chunk-local packing order
|
||||
// (all LOD0 first, then LOD1) so the worker's concatenated index
|
||||
// result lands at the offsets recorded in
|
||||
// m.mesh_chunk_local_lod1_first_u32.
|
||||
for (std::uint32_t mi : c.mesh_ids) {
|
||||
const MeshInfo& mesh = m.meshes[mi];
|
||||
if (mesh.lod1_index_count == 0) continue;
|
||||
req.i_ranges.emplace_back(
|
||||
std::uint64_t(mesh.lod1_ebo_byte_offset / sizeof(std::uint32_t)),
|
||||
std::uint64_t(mesh.lod1_index_count));
|
||||
}
|
||||
req.model_id = model_id;
|
||||
req.chunk_idx = chunk_idx;
|
||||
req.file_path = m.streaming_file_path;
|
||||
// v16: one compressed vertex frame + one compressed index frame per chunk.
|
||||
req.geometry_section_offset = m.geometry_section_offset;
|
||||
req.v_comp_off = c.v_comp_off;
|
||||
req.v_comp_size = c.v_comp_size;
|
||||
req.v_raw_size = c.vertex_byte_size;
|
||||
req.i_comp_off = c.i_comp_off;
|
||||
req.i_comp_size = c.i_comp_size;
|
||||
req.i_raw_size = c.index_count * sizeof(std::uint32_t);
|
||||
return req;
|
||||
}
|
||||
|
||||
@@ -1982,34 +1972,18 @@ bool ViewportCore::loadChunkBytesAndUploadGpu(ModelGpuData& m,
|
||||
if (c.is_resident) return true;
|
||||
if (m.streaming_file_path.empty()) return false;
|
||||
|
||||
// Synchronous fallback: build the request, do the disk read inline,
|
||||
// apply. Used only when the async path can't be — i.e. by the
|
||||
// screenshot test on first frame.
|
||||
StreamingThread::Request req = makeChunkRequest(m, chunk_idx, /*model_id*/ 0);
|
||||
|
||||
// Synchronous fallback: read + decompress the chunk inline, apply. Used
|
||||
// only when the async path can't be — i.e. by the screenshot test on the
|
||||
// first frame.
|
||||
std::vector<std::uint8_t> vbytes;
|
||||
std::vector<std::uint32_t> idx;
|
||||
if (!req.v_ranges.empty()) {
|
||||
if (!readSidecarVertexRanges(req.file_path,
|
||||
req.vertex_section_offset,
|
||||
req.v_ranges, vbytes)) {
|
||||
Log::warn() << "[wgpu stream] failed to read vertex chunk "
|
||||
<< chunk_idx
|
||||
<< " (" << req.v_ranges.size() << " ranges, total "
|
||||
<< c.vertex_byte_size << " B)";
|
||||
return false;
|
||||
}
|
||||
}
|
||||
if (!req.i_ranges.empty()) {
|
||||
if (!readSidecarIndexRanges(req.file_path,
|
||||
req.index_section_offset,
|
||||
req.i_ranges, idx)) {
|
||||
Log::warn() << "[wgpu stream] failed to read index chunk "
|
||||
<< chunk_idx
|
||||
<< " (" << req.i_ranges.size() << " ranges, total "
|
||||
<< c.index_count << " indices)";
|
||||
return false;
|
||||
}
|
||||
if (!readChunkGeometryCompressed(
|
||||
m.streaming_file_path, m.geometry_section_offset,
|
||||
c.v_comp_off, c.v_comp_size, c.vertex_byte_size,
|
||||
c.i_comp_off, c.i_comp_size, c.index_count * sizeof(std::uint32_t),
|
||||
vbytes, idx)) {
|
||||
Log::warn() << "[wgpu stream] failed to read/decompress chunk " << chunk_idx;
|
||||
return false;
|
||||
}
|
||||
return applyStreamedChunk(m, chunk_idx, vbytes, idx);
|
||||
}
|
||||
@@ -2316,24 +2290,34 @@ void ViewportCore::driveStreamingLoads() {
|
||||
if (cand.m->streaming_from_web
|
||||
&& pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) {
|
||||
// Not enough VALIDATED pool space for this chunk plus what's already
|
||||
// in flight. Don't fetch — the bytes would arrive, fail the alloc on
|
||||
// a not-yet-grown pool, and re-fetch (network thrash). Instead grow
|
||||
// the pool first (async on web: a provisional sub-buffer validates a
|
||||
// frame or two later, then this chunk fits and is fetched exactly
|
||||
// once). When the pool is saturated (model exceeds GPU memory) hold
|
||||
// the chunk off for the full cooldown so we keep a stable resident
|
||||
// subset instead of re-fetching what will never fit. Blocking before
|
||||
// the evictor also avoids phase-2 visible↔visible swap thrash.
|
||||
// in flight. If the pool can still grow, grow FIRST (async on web: a
|
||||
// provisional sub-buffer validates a frame or two later) — cheaper
|
||||
// than evict/refetch thrash while the model still fits by growing.
|
||||
if (pool_.can_grow()) {
|
||||
pool_.requestGrowth();
|
||||
c.blocked_cooldown_until_frame_idx =
|
||||
streaming_frame_idx_ + kGrowBackoffFrames;
|
||||
} else {
|
||||
more_pending = true;
|
||||
continue;
|
||||
}
|
||||
// At the hard capacity budget (setMaxTotalCapacity): growth can't
|
||||
// help — growing further would abort the wasm heap. Evict lower-
|
||||
// priority / LRU resident chunks so this priority-sorted candidate
|
||||
// fits. This is the only path that keeps a big federation navigable
|
||||
// once it exceeds the memory budget (highest-contribution chunks win).
|
||||
while (pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) {
|
||||
if (evict_one_lru()) continue;
|
||||
if (evict_lowest_priority_than(cand.mid, std::uint32_t(cand.ci),
|
||||
cand.priority)) continue;
|
||||
break;
|
||||
}
|
||||
if (pool_.total_free_bytes() < streaming_web_inflight_bytes_ + need) {
|
||||
c.blocked_cooldown_until_frame_idx =
|
||||
streaming_frame_idx_ + kBlockedCooldownFrames;
|
||||
more_pending = true;
|
||||
continue; // couldn't free enough — hold off this frame
|
||||
}
|
||||
more_pending = true;
|
||||
continue;
|
||||
// Freed enough — fall through to the web load below.
|
||||
}
|
||||
#endif
|
||||
while (!pool_can_fit(c.vertex_byte_size)
|
||||
@@ -2853,13 +2837,12 @@ void ViewportCore::applyCachedModel(std::uint32_t model_id,
|
||||
}
|
||||
|
||||
ModelGpuData m;
|
||||
m.vertex_bytes = metadata.vertex_total_bytes;
|
||||
m.index_count = std::uint32_t(metadata.index_total_count);
|
||||
m.vertex_bytes = 0; // accumulated from chunks below (v16 has no section)
|
||||
m.index_count = 0;
|
||||
m.mesh_count = std::uint32_t(metadata.meta.meshes.size());
|
||||
m.instance_count = std::uint32_t(metadata.meta.instances.size());
|
||||
m.streaming_file_path = metadata.file_path;
|
||||
m.streaming_vertex_section_offset = metadata.vertex_section_offset;
|
||||
m.streaming_index_section_offset = metadata.index_section_offset;
|
||||
m.streaming_file_path = metadata.file_path;
|
||||
m.geometry_section_offset = metadata.geometry_section_offset;
|
||||
|
||||
// ---- Spatial chunk plan ----------------------------------------------
|
||||
// A sidecar carries a baked chunk TOC (v14): each chunk is a contiguous
|
||||
@@ -2978,6 +2961,14 @@ void ViewportCore::applyCachedModel(std::uint32_t model_id,
|
||||
c.vertex_byte_size = std::uint64_t(chunk_local_v) * INSTANCED_VERTEX_STRIDE_BYTES;
|
||||
c.index_count = chunk_local_i + chunk_local_lod1;
|
||||
c.lod1_index_count = chunk_local_lod1;
|
||||
// v16: compressed-blob locators from the baked TOC (streaming path).
|
||||
if (ci < metadata.meta.chunks.size()) {
|
||||
const SidecarChunk& sc = metadata.meta.chunks[ci];
|
||||
c.v_comp_off = sc.v_comp_off; c.v_comp_size = sc.v_comp_size;
|
||||
c.i_comp_off = sc.i_comp_off; c.i_comp_size = sc.i_comp_size;
|
||||
}
|
||||
m.vertex_bytes += c.vertex_byte_size;
|
||||
m.index_count += std::uint32_t(c.index_count);
|
||||
|
||||
// Small per-chunk buffers, allocated upfront so cull can write into
|
||||
// them. visible_draws_buffer cap = chunk's instance count.
|
||||
@@ -3385,44 +3376,33 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i
|
||||
ModelGpuData& m = it->second;
|
||||
if (chunk_idx >= m.chunks.size()) return;
|
||||
|
||||
const StreamingThread::Request req = makeChunkRequest(m, chunk_idx, model_id);
|
||||
const ModelGpuData::Chunk& c = m.chunks[chunk_idx];
|
||||
const int sid = m.web_source_id; // which registered byte-source to read from
|
||||
const std::uint64_t vsec = req.vertex_section_offset;
|
||||
const std::uint64_t isec = req.index_section_offset;
|
||||
const std::vector<std::pair<std::uint64_t, std::uint64_t>> v_ranges = req.v_ranges;
|
||||
// i_ranges are (first_u32, count_u32); convert to byte ranges.
|
||||
std::vector<std::pair<std::uint64_t, std::uint64_t>> i_byte_ranges;
|
||||
i_byte_ranges.reserve(req.i_ranges.size());
|
||||
for (const auto& [first_u32, count] : req.i_ranges)
|
||||
i_byte_ranges.emplace_back(first_u32 * 4u, count * 4u);
|
||||
const std::uint64_t geom = m.geometry_section_offset;
|
||||
const std::uint64_t v_comp_off = c.v_comp_off, v_comp_size = c.v_comp_size;
|
||||
const std::uint64_t i_comp_off = c.i_comp_off, i_comp_size = c.i_comp_size;
|
||||
const std::uint64_t v_raw = c.vertex_byte_size;
|
||||
const std::uint64_t i_raw = std::uint64_t(c.index_count) * sizeof(std::uint32_t);
|
||||
|
||||
// Reserve this load's pool footprint while it's in flight (released when it
|
||||
// resolves) so driveStreamingLoads doesn't over-commit the pool — see
|
||||
// streaming_web_inflight_bytes_.
|
||||
const std::uint64_t need = m.chunks[chunk_idx].vertex_byte_size
|
||||
+ std::uint64_t(m.chunks[chunk_idx].index_count) * sizeof(std::uint32_t);
|
||||
// Reserve the RAW (decompressed) footprint while in flight so
|
||||
// driveStreamingLoads doesn't over-commit the pool.
|
||||
const std::uint64_t need = v_raw + i_raw;
|
||||
streaming_web_inflight_bytes_ += need;
|
||||
|
||||
// Fire the vertex and index range reads CONCURRENTLY and join when both
|
||||
// land — over a network this halves per-chunk latency vs reading vertices
|
||||
// then indices serially (two round trips → one). The join holds both
|
||||
// payloads + completion flags; whichever read finishes second runs the
|
||||
// apply. Re-look-up the model at apply time: a resetScene() could have
|
||||
// landed mid-flight, in which case the model id is gone and we drop it.
|
||||
// Fetch the chunk's two zstd frames CONCURRENTLY (vertex + index) and join
|
||||
// when both land, then decompress and apply. Re-look-up the model at apply
|
||||
// time: a resetScene() could have landed mid-flight.
|
||||
struct ChunkJoin {
|
||||
std::vector<std::uint8_t> vbytes;
|
||||
std::vector<std::uint32_t> idx;
|
||||
std::vector<std::uint8_t> vz, iz; // compressed frames
|
||||
bool v_done = false, i_done = false, v_ok = false, i_ok = false;
|
||||
};
|
||||
auto join = std::make_shared<ChunkJoin>();
|
||||
std::function<void()> finish = [this, model_id, chunk_idx, need, join]() {
|
||||
if (!join->v_done || !join->i_done) return; // wait for the other read
|
||||
// Release the in-flight reservation (clamped — a mid-flight resetScene
|
||||
// could have zeroed it) + the concurrency slot, regardless of outcome.
|
||||
streaming_web_inflight_bytes_ -=
|
||||
std::min(streaming_web_inflight_bytes_, need);
|
||||
std::function<void()> finish =
|
||||
[this, model_id, chunk_idx, need, v_raw, i_raw, join]() {
|
||||
if (!join->v_done || !join->i_done) return; // wait for the other frame
|
||||
streaming_web_inflight_bytes_ -= std::min(streaming_web_inflight_bytes_, need);
|
||||
if (streaming_web_inflight_count_ > 0) --streaming_web_inflight_count_;
|
||||
host_->requestFrame(); // a slot freed — let driveStreamingLoads issue more
|
||||
host_->requestFrame();
|
||||
|
||||
auto mit = models_gpu_.find(model_id);
|
||||
if (mit == models_gpu_.end()) return;
|
||||
@@ -3430,12 +3410,16 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i
|
||||
if (chunk_idx >= mm.chunks.size()) return;
|
||||
auto& cc = mm.chunks[chunk_idx];
|
||||
cc.is_loading = false;
|
||||
// On read failure or a full pool, back off instead of re-candidating
|
||||
// next frame → re-fetch thrash. If the pool can still grow (its async
|
||||
// sub-buffer is mid-validation), retry soon; if it's saturated, hold
|
||||
// off for the full cooldown.
|
||||
if (!join->v_ok || !join->i_ok
|
||||
|| !applyStreamedChunk(mm, chunk_idx, join->vbytes, join->idx)) {
|
||||
|
||||
std::vector<std::uint8_t> vbytes(static_cast<std::size_t>(v_raw));
|
||||
std::vector<std::uint32_t> idx(static_cast<std::size_t>(i_raw / sizeof(std::uint32_t)));
|
||||
const bool ok = join->v_ok && join->i_ok
|
||||
&& SidecarCompress::decompress(join->vz.data(), join->vz.size(),
|
||||
vbytes.data(), vbytes.size())
|
||||
&& SidecarCompress::decompress(join->iz.data(), join->iz.size(),
|
||||
reinterpret_cast<std::uint8_t*>(idx.data()),
|
||||
std::size_t(i_raw));
|
||||
if (!ok || !applyStreamedChunk(mm, chunk_idx, vbytes, idx)) {
|
||||
cc.blocked_cooldown_until_frame_idx = streaming_frame_idx_
|
||||
+ (pool_.can_grow() ? kGrowBackoffFrames : kBlockedCooldownFrames);
|
||||
return;
|
||||
@@ -3443,22 +3427,13 @@ void ViewportCore::beginWebChunkLoad(std::uint32_t model_id, std::size_t chunk_i
|
||||
host_->requestFrame();
|
||||
};
|
||||
|
||||
webReadRangesAsync(sid, vsec, v_ranges,
|
||||
[join, finish](bool ok, std::vector<std::uint8_t>&& vbytes) {
|
||||
join->v_ok = ok;
|
||||
join->vbytes = std::move(vbytes);
|
||||
join->v_done = true;
|
||||
finish();
|
||||
webReadRangesAsync(sid, geom, {{v_comp_off, v_comp_size}},
|
||||
[join, finish](bool ok, std::vector<std::uint8_t>&& vz) {
|
||||
join->v_ok = ok; join->vz = std::move(vz); join->v_done = true; finish();
|
||||
});
|
||||
webReadRangesAsync(sid, isec, i_byte_ranges,
|
||||
[join, finish](bool ok, std::vector<std::uint8_t>&& ibytes) {
|
||||
join->i_ok = ok;
|
||||
join->idx.resize(ibytes.size() / sizeof(std::uint32_t));
|
||||
if (!join->idx.empty())
|
||||
std::memcpy(join->idx.data(), ibytes.data(),
|
||||
join->idx.size() * sizeof(std::uint32_t));
|
||||
join->i_done = true;
|
||||
finish();
|
||||
webReadRangesAsync(sid, geom, {{i_comp_off, i_comp_size}},
|
||||
[join, finish](bool ok, std::vector<std::uint8_t>&& iz) {
|
||||
join->i_ok = ok; join->iz = std::move(iz); join->i_done = true; finish();
|
||||
});
|
||||
}
|
||||
|
||||
@@ -3481,80 +3456,76 @@ void ViewportCore::loadSidecarMetadataWeb(int source_id, std::string source_labe
|
||||
return;
|
||||
}
|
||||
|
||||
// Head (v16): [header 12][geom_bytes 8]. The two compressed metadata blocks
|
||||
// follow the compressed geometry at SIDECAR_HEAD_BYTES + geom_bytes.
|
||||
webReadRangesAsync(source_id, 0, {{0, SIDECAR_HEAD_BYTES}},
|
||||
[this, fsize, source_id, source_label](bool ok, std::vector<std::uint8_t>&& head) {
|
||||
std::uint32_t nvb = 0;
|
||||
if (!ok || !parseSidecarHead(head.data(), head.size(), nvb)) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: bad sidecar head";
|
||||
std::uint64_t geom_bytes = 0;
|
||||
if (!ok || !parseSidecarHead(head.data(), head.size(), geom_bytes)) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: bad sidecar head (wrong version?)";
|
||||
return;
|
||||
}
|
||||
const std::uint64_t vsec = SIDECAR_HEAD_BYTES;
|
||||
const std::uint64_t idx_count_off = std::uint64_t(SIDECAR_HEAD_BYTES) + nvb;
|
||||
|
||||
webReadRangesAsync(source_id, 0, {{idx_count_off, 4}},
|
||||
[this, fsize, nvb, vsec, idx_count_off, source_id, source_label]
|
||||
(bool ok2, std::vector<std::uint8_t>&& cnt) {
|
||||
if (!ok2 || cnt.size() < 4) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: short index count";
|
||||
return;
|
||||
}
|
||||
std::uint32_t num_indices = 0;
|
||||
std::memcpy(&num_indices, cnt.data(), 4);
|
||||
const std::uint64_t isec = idx_count_off + 4;
|
||||
const std::uint64_t crit_size_off = isec + std::uint64_t(num_indices) * 4u;
|
||||
if (double(crit_size_off + 8) > fsize) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: critical size past EOF";
|
||||
return;
|
||||
}
|
||||
// v15: read the 8-byte critical-block length, then the
|
||||
// render-critical metadata only. The deferred block
|
||||
// (elements/strings) is fetched on demand later — first
|
||||
// paint no longer waits on the property tree.
|
||||
webReadRangesAsync(source_id, 0, {{crit_size_off, 8}},
|
||||
[this, vsec, nvb, isec, num_indices, source_id, source_label,
|
||||
crit_size_off, fsize]
|
||||
(bool okc, std::vector<std::uint8_t>&& cb) {
|
||||
if (!okc || cb.size() < 8) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: short critical size";
|
||||
const std::uint64_t meta_off = std::uint64_t(SIDECAR_HEAD_BYTES) + geom_bytes;
|
||||
if (double(meta_off + 16) > fsize) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: metadata past EOF";
|
||||
return;
|
||||
}
|
||||
// Critical block on disk: [comp u64][raw u64][zstd frame].
|
||||
webReadRangesAsync(source_id, 0, {{meta_off, 16}},
|
||||
[this, fsize, meta_off, source_id, source_label]
|
||||
(bool ok2, std::vector<std::uint8_t>&& h) {
|
||||
if (!ok2 || h.size() < 16) { Log::warn() << "loadSidecarMetadataWeb: short crit header"; return; }
|
||||
std::uint64_t crit_comp = 0, crit_raw = 0;
|
||||
std::memcpy(&crit_comp, h.data(), 8);
|
||||
std::memcpy(&crit_raw, h.data() + 8, 8);
|
||||
const std::uint64_t crit_off = meta_off + 16;
|
||||
if (double(crit_off + crit_comp + 16) > fsize) { Log::warn() << "loadSidecarMetadataWeb: crit past EOF"; return; }
|
||||
webReadRangesAsync(source_id, 0, {{crit_off, crit_comp}},
|
||||
[this, crit_off, crit_comp, crit_raw, source_id, source_label]
|
||||
(bool ok3, std::vector<std::uint8_t>&& cz) {
|
||||
if (!ok3) { Log::warn() << "loadSidecarMetadataWeb: critical read failed"; return; }
|
||||
std::vector<std::uint8_t> crit(static_cast<std::size_t>(crit_raw));
|
||||
if (!SidecarCompress::decompress(cz.data(), cz.size(), crit.data(), crit.size())) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: critical decompress failed";
|
||||
return;
|
||||
}
|
||||
std::uint64_t crit_bytes = 0;
|
||||
std::memcpy(&crit_bytes, cb.data(), 8);
|
||||
const std::uint64_t crit_off = crit_size_off + 8;
|
||||
if (double(crit_off + crit_bytes) > fsize) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: critical block past EOF";
|
||||
StreamingSidecar sc;
|
||||
sc.file_path = source_label;
|
||||
sc.geometry_section_offset = SIDECAR_HEAD_BYTES;
|
||||
if (!parseSidecarCritical(crit.data(), crit.size(), sc.meta)) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: bad critical metadata";
|
||||
return;
|
||||
}
|
||||
webReadRangesAsync(source_id, 0, {{crit_off, crit_bytes}},
|
||||
[this, vsec, nvb, isec, num_indices, source_id, source_label,
|
||||
crit_off, crit_bytes, fsize]
|
||||
(bool ok3, std::vector<std::uint8_t>&& crit) {
|
||||
if (!ok3) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: critical read failed";
|
||||
return;
|
||||
}
|
||||
StreamingSidecar sc;
|
||||
sc.file_path = source_label;
|
||||
sc.vertex_section_offset = vsec;
|
||||
sc.vertex_total_bytes = nvb;
|
||||
sc.index_section_offset = isec;
|
||||
sc.index_total_count = num_indices;
|
||||
if (!parseSidecarCritical(crit.data(), crit.size(), sc.meta)) {
|
||||
Log::warn() << "loadSidecarMetadataWeb: bad critical metadata";
|
||||
return;
|
||||
}
|
||||
const std::size_t n_meshes = sc.meta.meshes.size();
|
||||
const std::size_t n_instances = sc.meta.instances.size();
|
||||
const std::uint32_t mid = next_model_id_++;
|
||||
applyCachedModel(mid, std::move(sc));
|
||||
const std::size_t n_meshes = sc.meta.meshes.size();
|
||||
const std::size_t n_instances = sc.meta.instances.size();
|
||||
const std::uint32_t mid = next_model_id_++;
|
||||
applyCachedModel(mid, std::move(sc));
|
||||
// Mark web-streamed + set the source IMMEDIATELY — the
|
||||
// model now has non-resident chunks and the RAF loop's
|
||||
// driveStreamingLoads will run before the deferred-header
|
||||
// read below returns. If streaming_from_web weren't set
|
||||
// yet it would take the sync fopen path and fail
|
||||
// ("failed to read/decompress chunk 0").
|
||||
if (auto m0 = models_gpu_.find(mid); m0 != models_gpu_.end()) {
|
||||
m0->second.streaming_from_web = true;
|
||||
m0->second.web_source_id = source_id;
|
||||
}
|
||||
// Read the deferred block header to record its locator
|
||||
// (the property block is fetched on demand later).
|
||||
const std::uint64_t def_hdr_off = crit_off + crit_comp;
|
||||
webReadRangesAsync(source_id, 0, {{def_hdr_off, 16}},
|
||||
[this, mid, def_hdr_off, source_id, source_label, n_meshes, n_instances]
|
||||
(bool ok4, std::vector<std::uint8_t>&& dh) {
|
||||
auto mit = models_gpu_.find(mid);
|
||||
if (mit != models_gpu_.end()) {
|
||||
mit->second.streaming_from_web = true;
|
||||
mit->second.web_source_id = source_id;
|
||||
// Deferred block: [end of critical, EOF).
|
||||
mit->second.deferred_meta_offset = crit_off + crit_bytes;
|
||||
mit->second.deferred_meta_bytes =
|
||||
std::uint64_t(fsize) - (crit_off + crit_bytes);
|
||||
if (ok4 && dh.size() >= 16) {
|
||||
std::uint64_t dc = 0, dr = 0;
|
||||
std::memcpy(&dc, dh.data(), 8);
|
||||
std::memcpy(&dr, dh.data() + 8, 8);
|
||||
mit->second.deferred_comp_offset = def_hdr_off + 16;
|
||||
mit->second.deferred_comp_size = dc;
|
||||
mit->second.deferred_raw_size = dr;
|
||||
}
|
||||
}
|
||||
viewAll();
|
||||
host_->requestFrame();
|
||||
@@ -3577,18 +3548,22 @@ void ViewportCore::loadDeferredMetadataWeb(std::uint32_t model_id,
|
||||
auto it = models_gpu_.find(model_id);
|
||||
if (it == models_gpu_.end()) { if (done) done(false); return; }
|
||||
ModelGpuData& m = it->second;
|
||||
if (m.deferred_meta_loaded || m.deferred_meta_bytes == 0) {
|
||||
if (m.deferred_meta_loaded || m.deferred_comp_size == 0) {
|
||||
m.deferred_meta_loaded = true;
|
||||
if (done) done(true);
|
||||
return;
|
||||
}
|
||||
webReadRangesAsync(m.web_source_id, 0, {{m.deferred_meta_offset, m.deferred_meta_bytes}},
|
||||
[this, model_id, done](bool ok, std::vector<std::uint8_t>&& buf) {
|
||||
const std::uint64_t raw_size = m.deferred_raw_size;
|
||||
webReadRangesAsync(m.web_source_id, 0, {{m.deferred_comp_offset, m.deferred_comp_size}},
|
||||
[this, model_id, raw_size, done](bool ok, std::vector<std::uint8_t>&& cz) {
|
||||
auto mit = models_gpu_.find(model_id);
|
||||
if (mit == models_gpu_.end()) { if (done) done(false); return; }
|
||||
std::vector<std::uint8_t> buf(static_cast<std::size_t>(raw_size));
|
||||
SidecarData tmp;
|
||||
if (!ok || !parseSidecarDeferred(buf.data(), buf.size(), tmp)) {
|
||||
Log::warn() << "loadDeferredMetadataWeb: read/parse failed";
|
||||
if (!ok ||
|
||||
!SidecarCompress::decompress(cz.data(), cz.size(), buf.data(), buf.size()) ||
|
||||
!parseSidecarDeferred(buf.data(), buf.size(), tmp)) {
|
||||
Log::warn() << "loadDeferredMetadataWeb: read/decompress/parse failed";
|
||||
if (done) done(false);
|
||||
return;
|
||||
}
|
||||
@@ -3678,8 +3653,11 @@ void ViewportCore::streamingByteProgress(std::uint64_t& total_bytes,
|
||||
for (const auto& [mid, m] : models_gpu_) {
|
||||
if (m.hidden) continue;
|
||||
for (const auto& c : m.chunks) {
|
||||
const std::uint64_t bytes =
|
||||
c.vertex_byte_size + c.index_count * sizeof(std::uint32_t);
|
||||
// Report COMPRESSED bytes — what actually crosses the network. Fall
|
||||
// back to raw for direct (in-memory) loads that have no blobs.
|
||||
const std::uint64_t bytes = (c.v_comp_size + c.i_comp_size > 0)
|
||||
? c.v_comp_size + c.i_comp_size
|
||||
: c.vertex_byte_size + c.index_count * sizeof(std::uint32_t);
|
||||
total_bytes += bytes;
|
||||
if (c.contribution_visible_count > 0) {
|
||||
needed_bytes += bytes;
|
||||
@@ -3719,10 +3697,9 @@ void ViewportCore::finalizeModel(std::uint32_t model_id) {
|
||||
// applyStreamedChunk loop below).
|
||||
StreamingSidecar metadata;
|
||||
metadata.meta = std::move(s);
|
||||
metadata.vertex_section_offset = 0;
|
||||
metadata.vertex_total_bytes = metadata.meta.vertices.size();
|
||||
metadata.index_section_offset = 0;
|
||||
metadata.index_total_count = metadata.meta.indices.size();
|
||||
// Direct load: geometry is already in memory (uploaded below), streamed
|
||||
// from nothing — leave file_path empty so the streaming worker skips it.
|
||||
metadata.geometry_section_offset = 0;
|
||||
metadata.file_path.clear();
|
||||
|
||||
std::vector<std::uint8_t> raw_vertices = std::move(metadata.meta.vertices);
|
||||
|
||||
@@ -46,8 +46,18 @@ if(WITH_MESH_OPTIMIZER)
|
||||
target_compile_definitions(test_lod_builder PRIVATE -DWITH_MESH_OPTIMIZER)
|
||||
endif()
|
||||
|
||||
# SidecarCompress: zstd compress/decompress wrappers. Links the system libzstd.
|
||||
# SidecarCache now (de)compresses geometry + metadata, so every test that
|
||||
# compiles SidecarCache.cpp needs SidecarCompress.cpp + libzstd too.
|
||||
find_library(ZSTD_LIBRARY NAMES zstd libzstd)
|
||||
add_ifcviewer_unit_test(test_sidecar_compress
|
||||
SOURCES ${IFCVIEWER_SRC}/SidecarCompress.cpp
|
||||
LIBS ${ZSTD_LIBRARY}
|
||||
)
|
||||
|
||||
add_ifcviewer_unit_test(test_sidecar_cache
|
||||
SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp
|
||||
SOURCES ${IFCVIEWER_SRC}/SidecarCache.cpp ${IFCVIEWER_SRC}/SidecarCompress.cpp
|
||||
LIBS ${ZSTD_LIBRARY}
|
||||
)
|
||||
|
||||
# StreamingLoader: metadata-only read + range readers + the pure buffer-based
|
||||
@@ -57,6 +67,8 @@ add_ifcviewer_unit_test(test_streaming_loader
|
||||
SOURCES
|
||||
${IFCVIEWER_SRC}/StreamingLoader.cpp
|
||||
${IFCVIEWER_SRC}/SidecarCache.cpp
|
||||
${IFCVIEWER_SRC}/SidecarCompress.cpp
|
||||
LIBS ${ZSTD_LIBRARY}
|
||||
)
|
||||
|
||||
add_ifcviewer_unit_test(test_instanced_geometry)
|
||||
@@ -74,6 +86,8 @@ add_ifcviewer_unit_test(test_sidecar_layout
|
||||
${IFCVIEWER_SRC}/SidecarLayout.cpp
|
||||
${IFCVIEWER_SRC}/ChunkPlanner.cpp
|
||||
${IFCVIEWER_SRC}/SidecarCache.cpp
|
||||
${IFCVIEWER_SRC}/SidecarCompress.cpp
|
||||
LIBS ${ZSTD_LIBRARY}
|
||||
)
|
||||
|
||||
# InstanceCompose: matrix composition + cross-model object_id lookup.
|
||||
|
||||
@@ -118,6 +118,9 @@ SidecarData buildFixture() {
|
||||
e.name_offset = 1; e.name_length = 4; // "Wall"
|
||||
e.type_offset = 6; e.type_length = 4; // "Slab"
|
||||
}
|
||||
// v16 stores geometry per-chunk (compressed), so a fixture with geometry
|
||||
// needs a chunk TOC covering its meshes for write/read to round-trip.
|
||||
sd.chunks = { {0, 2} };
|
||||
return sd;
|
||||
}
|
||||
|
||||
@@ -155,8 +158,8 @@ bool sidecarDataEqual(const SidecarData& a, const SidecarData& b) {
|
||||
TEST_CASE("MeshInfo and InstanceCpu have stable layouts (sidecar wire format)", "[sidecar]") {
|
||||
REQUIRE(sizeof(MeshInfo) == 56);
|
||||
REQUIRE(sizeof(InstanceGpu) == 80);
|
||||
REQUIRE(SIDECAR_VERSION == 15);
|
||||
REQUIRE(sizeof(SidecarChunk) == 8);
|
||||
REQUIRE(SIDECAR_VERSION == 16);
|
||||
REQUIRE(sizeof(SidecarChunk) == 56);
|
||||
REQUIRE(SIDECAR_MAGIC == 0x49465657u);
|
||||
}
|
||||
|
||||
|
||||
@@ -0,0 +1,68 @@
|
||||
/********************************************************************************
|
||||
* *
|
||||
* This file is part of IfcOpenShell. *
|
||||
* *
|
||||
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the Lesser GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3.0 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* IfcOpenShell is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* Lesser GNU General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the Lesser GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
********************************************************************************/
|
||||
|
||||
#include "SidecarCompress.h"
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
|
||||
#include <cstdint>
|
||||
#include <vector>
|
||||
|
||||
TEST_CASE("zstd round-trips arbitrary bytes", "[compress]") {
|
||||
// Structured data like the sidecar carries (repeated matrices, patterned
|
||||
// indices) — should both round-trip AND actually shrink.
|
||||
std::vector<std::uint8_t> raw;
|
||||
for (int i = 0; i < 20000; ++i) {
|
||||
raw.push_back(std::uint8_t(i & 0xFF));
|
||||
raw.push_back(std::uint8_t((i >> 8) & 0x07)); // low-entropy high byte
|
||||
raw.push_back(0);
|
||||
raw.push_back(0xAA);
|
||||
}
|
||||
|
||||
auto packed = SidecarCompress::compress(raw.data(), raw.size(), 19);
|
||||
REQUIRE_FALSE(packed.empty());
|
||||
REQUIRE(packed.size() < raw.size()); // it compressed
|
||||
|
||||
std::vector<std::uint8_t> out(raw.size());
|
||||
REQUIRE(SidecarCompress::decompress(packed.data(), packed.size(),
|
||||
out.data(), out.size()));
|
||||
REQUIRE(out == raw);
|
||||
}
|
||||
|
||||
TEST_CASE("decompress rejects a wrong raw size / garbage", "[compress]") {
|
||||
std::vector<std::uint8_t> raw(1024, 0x42);
|
||||
auto packed = SidecarCompress::compress(raw.data(), raw.size(), 3);
|
||||
REQUIRE_FALSE(packed.empty());
|
||||
|
||||
// Wrong declared raw size must fail, not silently truncate.
|
||||
std::vector<std::uint8_t> too_small(512);
|
||||
REQUIRE_FALSE(SidecarCompress::decompress(packed.data(), packed.size(),
|
||||
too_small.data(), too_small.size()));
|
||||
|
||||
// Garbage input fails cleanly.
|
||||
std::vector<std::uint8_t> junk = { 1, 2, 3, 4, 5, 6, 7, 8 };
|
||||
std::vector<std::uint8_t> dst(1024);
|
||||
REQUIRE_FALSE(SidecarCompress::decompress(junk.data(), junk.size(),
|
||||
dst.data(), dst.size()));
|
||||
}
|
||||
|
||||
TEST_CASE("empty round-trips to empty", "[compress]") {
|
||||
std::vector<std::uint8_t> dst;
|
||||
REQUIRE(SidecarCompress::decompress(nullptr, 0, dst.data(), 0));
|
||||
}
|
||||
@@ -18,6 +18,7 @@
|
||||
********************************************************************************/
|
||||
|
||||
#include "SidecarCache.h"
|
||||
#include "SidecarCompress.h"
|
||||
#include "StreamingLoader.h"
|
||||
|
||||
#include <catch2/catch_test_macros.hpp>
|
||||
@@ -85,12 +86,15 @@ SidecarData buildFixture() {
|
||||
sd.elements[i].ifc_id = int32_t(1000 + i);
|
||||
sd.elements[i].parent_id = (i == 0) ? -1 : int32_t(100);
|
||||
}
|
||||
// v16 stores geometry per-chunk (compressed); a fixture with geometry needs
|
||||
// a chunk TOC covering its meshes (one chunk per mesh here).
|
||||
sd.chunks = { {0, 1}, {1, 1} };
|
||||
return sd;
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bulk",
|
||||
TEST_CASE("readSidecarMetadataOnly returns metadata, skips bulk geometry",
|
||||
"[streaming]") {
|
||||
fs::path dir = makeScratchDir("metaonly");
|
||||
fs::path ifc = dir / "model.ifc";
|
||||
@@ -100,19 +104,19 @@ TEST_CASE("readSidecarMetadataOnly returns metadata + section offsets, skips bul
|
||||
auto meta = readSidecarMetadataOnly(ifc.string());
|
||||
REQUIRE(meta.has_value());
|
||||
|
||||
// Bulk sections are skipped, not loaded.
|
||||
// Bulk geometry is skipped, not loaded.
|
||||
REQUIRE(meta->meta.vertices.empty());
|
||||
REQUIRE(meta->meta.indices.empty());
|
||||
|
||||
// Offsets locate the two skipped sections. The vertex section starts
|
||||
// right after the 16-byte head.
|
||||
REQUIRE(meta->vertex_section_offset == SIDECAR_HEAD_BYTES);
|
||||
REQUIRE(meta->vertex_total_bytes == sd.vertices.size());
|
||||
REQUIRE(meta->index_total_count == sd.indices.size());
|
||||
REQUIRE(meta->index_section_offset ==
|
||||
SIDECAR_HEAD_BYTES + sd.vertices.size() + 4);
|
||||
// v16: the compressed geometry section starts right after the 20-byte head.
|
||||
REQUIRE(meta->geometry_section_offset == SIDECAR_HEAD_BYTES);
|
||||
// Chunk TOC carries compressed blob locators for each chunk.
|
||||
REQUIRE(meta->meta.chunks.size() == sd.chunks.size());
|
||||
REQUIRE(meta->meta.chunks[0].v_comp_size > 0);
|
||||
// The deferred (property) block locator is recorded for on-demand fetch.
|
||||
REQUIRE(meta->deferred_comp_size > 0);
|
||||
|
||||
// Tail metadata round-trips.
|
||||
// Metadata round-trips.
|
||||
REQUIRE(meta->meta.meshes.size() == sd.meshes.size());
|
||||
REQUIRE(meta->meta.instances.size() == sd.instances.size());
|
||||
REQUIRE(meta->meta.elements.size() == sd.elements.size());
|
||||
@@ -140,100 +144,90 @@ TEST_CASE("readSidecarMetadataOnly rejects missing / corrupt files", "[streaming
|
||||
REQUIRE_FALSE(readSidecarMetadataOnly(bad.string()).has_value());
|
||||
}
|
||||
|
||||
TEST_CASE("readSidecarVertexRanges scatters byte ranges in input order", "[streaming]") {
|
||||
fs::path dir = makeScratchDir("vranges");
|
||||
TEST_CASE("readChunkGeometryCompressed decompresses a chunk's blobs", "[streaming]") {
|
||||
fs::path dir = makeScratchDir("chunkgeom");
|
||||
fs::path ifc = dir / "model.ifc";
|
||||
SidecarData sd = buildFixture();
|
||||
REQUIRE(writeSidecar(ifc.string(), sd));
|
||||
auto meta = readSidecarMetadataOnly(ifc.string());
|
||||
REQUIRE(meta.has_value());
|
||||
REQUIRE(meta->meta.chunks.size() == 2);
|
||||
|
||||
// Two section-relative ranges given out of file order; the destination
|
||||
// must preserve input order (second mesh's bytes first, then first).
|
||||
// Chunk 0 = mesh 0: vertices [0, 2*stride), indices {0,1,2}.
|
||||
const auto& c0 = meta->meta.chunks[0];
|
||||
const uint64_t stride = INSTANCED_VERTEX_STRIDE_BYTES;
|
||||
std::vector<std::pair<uint64_t, uint64_t>> ranges = {
|
||||
{2 * stride, 2 * stride}, // last 2 vertices
|
||||
{0, 2 * stride}, // first 2 vertices
|
||||
};
|
||||
std::vector<uint8_t> out;
|
||||
REQUIRE(readSidecarVertexRanges(ifc.string(), meta->vertex_section_offset,
|
||||
ranges, out));
|
||||
REQUIRE(out.size() == 4 * stride);
|
||||
REQUIRE(std::memcmp(out.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0);
|
||||
REQUIRE(std::memcmp(out.data() + 2 * stride, sd.vertices.data(), 2 * stride) == 0);
|
||||
std::vector<uint8_t> vbytes;
|
||||
std::vector<uint32_t> idx;
|
||||
REQUIRE(readChunkGeometryCompressed(
|
||||
ifc.string(), meta->geometry_section_offset,
|
||||
c0.v_comp_off, c0.v_comp_size, c0.v_raw_size,
|
||||
c0.i_comp_off, c0.i_comp_size, c0.i_raw_size, vbytes, idx));
|
||||
REQUIRE(vbytes.size() == 2 * stride);
|
||||
REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data(), 2 * stride) == 0);
|
||||
REQUIRE(idx == std::vector<uint32_t>({0, 1, 2}));
|
||||
|
||||
// Chunk 1 = mesh 1: indices {1,2,3}.
|
||||
const auto& c1 = meta->meta.chunks[1];
|
||||
REQUIRE(readChunkGeometryCompressed(
|
||||
ifc.string(), meta->geometry_section_offset,
|
||||
c1.v_comp_off, c1.v_comp_size, c1.v_raw_size,
|
||||
c1.i_comp_off, c1.i_comp_size, c1.i_raw_size, vbytes, idx));
|
||||
REQUIRE(idx == std::vector<uint32_t>({1, 2, 3}));
|
||||
REQUIRE(std::memcmp(vbytes.data(), sd.vertices.data() + 2 * stride, 2 * stride) == 0);
|
||||
}
|
||||
|
||||
TEST_CASE("readSidecarIndexRanges reads u32 index ranges", "[streaming]") {
|
||||
fs::path dir = makeScratchDir("iranges");
|
||||
fs::path ifc = dir / "model.ifc";
|
||||
SidecarData sd = buildFixture();
|
||||
REQUIRE(writeSidecar(ifc.string(), sd));
|
||||
auto meta = readSidecarMetadataOnly(ifc.string());
|
||||
REQUIRE(meta.has_value());
|
||||
|
||||
std::vector<std::pair<uint64_t, uint64_t>> ranges = {{3, 3}}; // indices[3..6)
|
||||
std::vector<uint32_t> out;
|
||||
REQUIRE(readSidecarIndexRanges(ifc.string(), meta->index_section_offset,
|
||||
ranges, out));
|
||||
REQUIRE(out == std::vector<uint32_t>({1, 2, 3}));
|
||||
}
|
||||
|
||||
TEST_CASE("parseSidecarHead validates magic / version / length", "[streaming]") {
|
||||
TEST_CASE("parseSidecarHead validates magic / version, reads geom length", "[streaming]") {
|
||||
uint8_t head[SIDECAR_HEAD_BYTES] = {};
|
||||
uint32_t magic = SIDECAR_MAGIC, version = SIDECAR_VERSION, endian = SIDECAR_ENDIAN;
|
||||
uint32_t nvb = 4096;
|
||||
uint64_t geom = 123456;
|
||||
std::memcpy(head + 0, &magic, 4);
|
||||
std::memcpy(head + 4, &version, 4);
|
||||
std::memcpy(head + 8, &endian, 4);
|
||||
std::memcpy(head + 12, &nvb, 4);
|
||||
std::memcpy(head + 12, &geom, 8);
|
||||
|
||||
uint32_t got = 0;
|
||||
uint64_t got = 0;
|
||||
REQUIRE(parseSidecarHead(head, sizeof(head), got));
|
||||
REQUIRE(got == 4096);
|
||||
REQUIRE(got == 123456);
|
||||
|
||||
// Short buffer.
|
||||
REQUIRE_FALSE(parseSidecarHead(head, SIDECAR_HEAD_BYTES - 1, got));
|
||||
|
||||
// Wrong magic.
|
||||
uint8_t bad[SIDECAR_HEAD_BYTES];
|
||||
std::memcpy(bad, head, sizeof(bad));
|
||||
bad[0] ^= 0xFF;
|
||||
REQUIRE_FALSE(parseSidecarHead(bad, sizeof(bad), got));
|
||||
}
|
||||
|
||||
TEST_CASE("v15 critical/deferred metadata split round-trips + rejects truncation",
|
||||
"[streaming]") {
|
||||
fs::path dir = makeScratchDir("v15split");
|
||||
TEST_CASE("v16 deferred block: fetch via locator, decompress, parse", "[streaming]") {
|
||||
fs::path dir = makeScratchDir("v16def");
|
||||
fs::path ifc = dir / "model.ifc";
|
||||
SidecarData sd = buildFixture();
|
||||
sd.chunks = { {0, 1}, {1, 1} }; // a TOC, so the critical block carries chunks
|
||||
REQUIRE(writeSidecar(ifc.string(), sd));
|
||||
|
||||
// readSidecarMetadataOnly (desktop) reads BOTH blocks + records the locator.
|
||||
auto meta = readSidecarMetadataOnly(ifc.string());
|
||||
REQUIRE(meta.has_value());
|
||||
REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical
|
||||
REQUIRE(meta->meta.chunks.size() == sd.chunks.size()); // critical
|
||||
REQUIRE(meta->meta.elements.size() == sd.elements.size()); // deferred
|
||||
REQUIRE(meta->meta.string_table == sd.string_table); // deferred
|
||||
REQUIRE(meta->critical_meta_bytes > 0);
|
||||
REQUIRE(meta->meta.meshes.size() == sd.meshes.size()); // critical
|
||||
REQUIRE(meta->meta.chunks.size() == sd.chunks.size());
|
||||
REQUIRE(meta->meta.elements.size() == sd.elements.size()); // desktop reads deferred too
|
||||
REQUIRE(meta->deferred_comp_size > 0);
|
||||
|
||||
// Pull the raw critical block via the recorded locator and parse it alone —
|
||||
// exactly what the web loader does before painting.
|
||||
// The on-demand path (web) fetches the compressed deferred frame via the
|
||||
// recorded locator and decompresses it — verify that round-trips.
|
||||
FILE* f = std::fopen((dir / "model.ifcview").string().c_str(), "rb");
|
||||
REQUIRE(f);
|
||||
std::vector<uint8_t> crit(size_t(meta->critical_meta_bytes));
|
||||
std::fseek(f, long(meta->critical_meta_offset), SEEK_SET);
|
||||
REQUIRE(std::fread(crit.data(), 1, crit.size(), f) == crit.size());
|
||||
std::vector<uint8_t> cz(size_t(meta->deferred_comp_size));
|
||||
std::fseek(f, long(meta->deferred_comp_offset), SEEK_SET);
|
||||
REQUIRE(std::fread(cz.data(), 1, cz.size(), f) == cz.size());
|
||||
std::fclose(f);
|
||||
|
||||
SidecarData c;
|
||||
REQUIRE(parseSidecarCritical(crit.data(), crit.size(), c));
|
||||
REQUIRE(c.meshes.size() == sd.meshes.size());
|
||||
REQUIRE(c.chunks.size() == sd.chunks.size());
|
||||
REQUIRE(c.elements.empty()); // the critical block has no property data
|
||||
std::vector<uint8_t> raw(size_t(meta->deferred_raw_size));
|
||||
REQUIRE(SidecarCompress::decompress(cz.data(), cz.size(), raw.data(), raw.size()));
|
||||
SidecarData d;
|
||||
REQUIRE(parseSidecarDeferred(raw.data(), raw.size(), d));
|
||||
REQUIRE(d.elements.size() == sd.elements.size());
|
||||
REQUIRE(d.string_table == sd.string_table);
|
||||
|
||||
SidecarData chopped;
|
||||
REQUIRE_FALSE(parseSidecarCritical(crit.data(), crit.size() - 1, chopped));
|
||||
REQUIRE_FALSE(parseSidecarDeferred(raw.data(), raw.size() - 1, chopped));
|
||||
}
|
||||
|
||||
TEST_CASE("planSidecarReadRanges coalesces adjacent ranges, keeps far ones split",
|
||||
|
||||
Reference in New Issue
Block a user