mirror of
https://github.com/IfcOpenShell/IfcOpenShell.git
synced 2026-08-09 17:31:45 +00:00
wgpu streaming (1/4): metadata-only sidecar reader
First foundational piece for task #16. WgpuStreamingLoader exposes: - readSidecarMetadataOnly(path): reads v13 header + mesh dict + instance dict + georef + elements + string table from disk. Skips the bulky vertex and index byte sections, recording their on-disk offsets so they can be range-read later (per-chunk, on demand). The file handle is closed before return. - readSidecarVertexChunk / readSidecarIndexChunk: open + fseek + fread for a byte range. Synchronous; intended to be called from a worker thread for true async streaming or the main thread for stage-1 on-demand load. No format change yet — operates on existing v13 sidecars. v14 with an explicit per-chunk TOC arrives in a follow-up; this layer abstracts the chunk boundaries so the upgrade stays internal. No integration with existing applyCachedModel — that's commit 3/4. Build verifies the API compiles and links into IfcViewerWgpu. Commits in this series: 1/4: metadata-only reader (THIS) 2/4: per-chunk residency state on WgpuModelGpuData 3/4: --streaming opt-in path through applyCachedModel 4/4: per-frame chunk-on-visible loader (the OOM fix) Co-Authored-By: Claude Opus 4.7 <noreply@anthropic.com>
This commit is contained in:
Dion Moult
@@ -0,0 +1,170 @@
|
||||
/********************************************************************************
|
||||
* *
|
||||
* This file is part of IfcOpenShell. *
|
||||
* *
|
||||
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the Lesser GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3.0 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* IfcOpenShell is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* Lesser GNU General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the Lesser GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
********************************************************************************/
|
||||
|
||||
// v13 sidecar layout (matched against SidecarCache.cpp):
|
||||
//
|
||||
// SidecarHeader (12 bytes)
|
||||
// uint32 num_vertex_bytes
|
||||
// uint8[num_vertex_bytes] vertex data <-- streaming skips
|
||||
// uint32 num_indices
|
||||
// uint32[num_indices] index data <-- streaming skips
|
||||
// uint32 num_meshes + MeshInfo[] <-- streaming reads
|
||||
// uint32 num_instances + InstanceCpu[] <-- streaming reads
|
||||
// uint32 has_coord_op + double[16] + 2× double <-- streaming reads
|
||||
// uint32 num_elements + PackedElementInfo[] <-- streaming reads
|
||||
// uint32 string_table_bytes + char[] <-- streaming reads
|
||||
//
|
||||
// Streaming reader returns offsets to the two skipped sections so chunks
|
||||
// can be range-read on demand. File handle is closed before return.
|
||||
|
||||
#include "WgpuStreamingLoader.h"
|
||||
|
||||
#include <cstdio>
|
||||
|
||||
namespace {
|
||||
|
||||
struct SidecarHeaderRaw {
|
||||
uint32_t magic;
|
||||
uint32_t version;
|
||||
uint32_t endian;
|
||||
};
|
||||
|
||||
template<typename T>
|
||||
bool readVec(FILE* f, std::vector<T>& v) {
|
||||
uint32_t n;
|
||||
if (std::fread(&n, 4, 1, f) != 1) return false;
|
||||
v.resize(n);
|
||||
if (n > 0 && std::fread(v.data(), sizeof(T), n, f) != n) return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
std::string sidecarPath(const std::string& ifc_path) {
|
||||
std::string p = ifc_path;
|
||||
while (!p.empty() && (p.back() == '/' || p.back() == '\\')) p.pop_back();
|
||||
auto slash = p.find_last_of("/\\");
|
||||
auto dot = p.find_last_of('.');
|
||||
std::string stem = (dot != std::string::npos &&
|
||||
(slash == std::string::npos || dot > slash))
|
||||
? p.substr(0, dot)
|
||||
: p;
|
||||
return stem + ".ifcview";
|
||||
}
|
||||
|
||||
} // namespace
|
||||
|
||||
std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_path) {
|
||||
const std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = std::fopen(path.c_str(), "rb");
|
||||
if (!f) return std::nullopt;
|
||||
|
||||
auto fail = [&]() -> std::optional<StreamingSidecar> {
|
||||
std::fclose(f);
|
||||
return std::nullopt;
|
||||
};
|
||||
|
||||
SidecarHeaderRaw hdr;
|
||||
if (std::fread(&hdr, sizeof(hdr), 1, f) != 1) return fail();
|
||||
if (hdr.magic != SIDECAR_MAGIC) return fail();
|
||||
if (hdr.version != SIDECAR_VERSION) return fail();
|
||||
if (hdr.endian != SIDECAR_ENDIAN) return fail();
|
||||
|
||||
StreamingSidecar out;
|
||||
out.file_path = path;
|
||||
|
||||
// Vertex section: read count, record offset of data, seek past.
|
||||
uint32_t num_vertex_bytes = 0;
|
||||
if (std::fread(&num_vertex_bytes, 4, 1, f) != 1) return fail();
|
||||
out.vertex_section_offset = uint64_t(std::ftell(f));
|
||||
out.vertex_total_bytes = num_vertex_bytes;
|
||||
if (std::fseek(f, long(num_vertex_bytes), SEEK_CUR) != 0) return fail();
|
||||
|
||||
// Index section: same dance, in u32 units.
|
||||
uint32_t num_indices = 0;
|
||||
if (std::fread(&num_indices, 4, 1, f) != 1) return fail();
|
||||
out.index_section_offset = uint64_t(std::ftell(f));
|
||||
out.index_total_count = num_indices;
|
||||
if (std::fseek(f, long(num_indices) * 4, SEEK_CUR) != 0) return fail();
|
||||
|
||||
// Mesh dict + instance dict — small, load into meta.
|
||||
if (!readVec(f, out.meta.meshes)) return fail();
|
||||
if (!readVec(f, out.meta.instances)) return fail();
|
||||
|
||||
// v11 georef block (148 bytes total).
|
||||
if (std::fread(&out.meta.has_coordinate_operation, 4, 1, f) != 1) return fail();
|
||||
if (std::fread(out.meta.coordinate_operation_meters,
|
||||
sizeof(double), 16, f) != 16) return fail();
|
||||
if (std::fread(&out.meta.project_length_to_meters,
|
||||
sizeof(double), 1, f) != 1) return fail();
|
||||
if (std::fread(&out.meta.map_unit_to_meters,
|
||||
sizeof(double), 1, f) != 1) return fail();
|
||||
|
||||
// Element table + string table.
|
||||
if (!readVec(f, out.meta.elements)) return fail();
|
||||
uint32_t stbl_len = 0;
|
||||
if (std::fread(&stbl_len, 4, 1, f) != 1) return fail();
|
||||
out.meta.string_table.resize(stbl_len);
|
||||
if (stbl_len > 0 &&
|
||||
std::fread(out.meta.string_table.data(), 1, stbl_len, f) != stbl_len)
|
||||
return fail();
|
||||
|
||||
std::fclose(f);
|
||||
return out;
|
||||
}
|
||||
|
||||
bool readSidecarVertexChunk(const std::string& ifc_path,
|
||||
uint64_t vertex_section_offset,
|
||||
uint64_t chunk_byte_offset,
|
||||
uint64_t chunk_byte_size,
|
||||
std::vector<uint8_t>& out_bytes) {
|
||||
if (chunk_byte_size == 0) { out_bytes.clear(); return true; }
|
||||
|
||||
const std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = std::fopen(path.c_str(), "rb");
|
||||
if (!f) return false;
|
||||
if (std::fseek(f, long(vertex_section_offset + chunk_byte_offset), SEEK_SET) != 0) {
|
||||
std::fclose(f);
|
||||
return false;
|
||||
}
|
||||
out_bytes.resize(size_t(chunk_byte_size));
|
||||
const size_t got = std::fread(out_bytes.data(), 1, size_t(chunk_byte_size), f);
|
||||
std::fclose(f);
|
||||
return got == size_t(chunk_byte_size);
|
||||
}
|
||||
|
||||
bool readSidecarIndexChunk(const std::string& ifc_path,
|
||||
uint64_t index_section_offset,
|
||||
uint64_t chunk_first_index,
|
||||
uint64_t chunk_index_count,
|
||||
std::vector<uint32_t>& out_indices) {
|
||||
if (chunk_index_count == 0) { out_indices.clear(); return true; }
|
||||
|
||||
const std::string path = sidecarPath(ifc_path);
|
||||
FILE* f = std::fopen(path.c_str(), "rb");
|
||||
if (!f) return false;
|
||||
const uint64_t byte_offset = index_section_offset + chunk_first_index * 4u;
|
||||
if (std::fseek(f, long(byte_offset), SEEK_SET) != 0) {
|
||||
std::fclose(f);
|
||||
return false;
|
||||
}
|
||||
out_indices.resize(size_t(chunk_index_count));
|
||||
const size_t got = std::fread(out_indices.data(), sizeof(uint32_t),
|
||||
size_t(chunk_index_count), f);
|
||||
std::fclose(f);
|
||||
return got == size_t(chunk_index_count);
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
/********************************************************************************
|
||||
* *
|
||||
* This file is part of IfcOpenShell. *
|
||||
* *
|
||||
* IfcOpenShell is free software: you can redistribute it and/or modify *
|
||||
* it under the terms of the Lesser GNU General Public License as published by *
|
||||
* the Free Software Foundation, either version 3.0 of the License, or *
|
||||
* (at your option) any later version. *
|
||||
* *
|
||||
* IfcOpenShell is distributed in the hope that it will be useful, *
|
||||
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
|
||||
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
|
||||
* Lesser GNU General Public License for more details. *
|
||||
* *
|
||||
* You should have received a copy of the Lesser GNU General Public License *
|
||||
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
|
||||
* *
|
||||
********************************************************************************/
|
||||
|
||||
#ifndef WGPUSTREAMINGLOADER_H
|
||||
#define WGPUSTREAMINGLOADER_H
|
||||
|
||||
#include "SidecarCache.h"
|
||||
|
||||
#include <cstdint>
|
||||
#include <optional>
|
||||
#include <string>
|
||||
#include <vector>
|
||||
|
||||
// Metadata-only sidecar load — the foundation for streaming. Reads the v13
|
||||
// header + mesh dict + instance dict + georef + element table from disk but
|
||||
// *skips* the bulky vertex and index byte sections, leaving file offsets +
|
||||
// sizes for later random-access reads.
|
||||
//
|
||||
// On a typical real-scene sidecar this returns in milliseconds even when the
|
||||
// full readSidecar would block on hundreds of MB of vertex bytes. Lets the
|
||||
// renderer set up cull / instance state immediately and load vertex chunks
|
||||
// on demand as they become frustum-visible.
|
||||
//
|
||||
// Backwards-compatible with v13 sidecars on disk (the format isn't changing
|
||||
// in this step — we're just reading less of it). v14 with an explicit
|
||||
// per-chunk TOC arrives in a follow-up; this layer abstracts the chunk
|
||||
// boundaries so the upgrade is internal.
|
||||
struct StreamingSidecar {
|
||||
// Everything except vertices + indices — same shape as SidecarData but
|
||||
// with empty vertices / indices vectors. The renderer uses meshes /
|
||||
// instances / georef / elements immediately.
|
||||
SidecarData meta;
|
||||
|
||||
// Byte offsets in the on-disk file where the vertex and index sections
|
||||
// start (after their 4-byte count headers). Pair with vertex_total_bytes
|
||||
// / index_total_bytes for the section length; per-chunk reads slice
|
||||
// arbitrary ranges within these.
|
||||
uint64_t vertex_section_offset = 0;
|
||||
uint64_t vertex_total_bytes = 0;
|
||||
uint64_t index_section_offset = 0;
|
||||
uint64_t index_total_count = 0; // u32 indices, NOT bytes
|
||||
|
||||
// Resolved on-disk path so subsequent chunk reads can re-open / seek.
|
||||
std::string file_path;
|
||||
};
|
||||
|
||||
// Read just the metadata + section offsets. Returns nullopt on any I/O or
|
||||
// version error (same failure modes as readSidecar). The file is closed
|
||||
// before return — callers re-open for per-chunk reads.
|
||||
std::optional<StreamingSidecar> readSidecarMetadataOnly(const std::string& ifc_path);
|
||||
|
||||
// Read a byte range from a sidecar's vertex section. `chunk_byte_offset` is
|
||||
// RELATIVE to vertex_section_offset (i.e. 0 = first vertex byte). Returns
|
||||
// false on I/O error or out-of-range request.
|
||||
//
|
||||
// Synchronous; intended to be called from a worker thread for async
|
||||
// streaming or from the main thread for stage-1 on-demand load.
|
||||
bool readSidecarVertexChunk(const std::string& ifc_path,
|
||||
uint64_t vertex_section_offset,
|
||||
uint64_t chunk_byte_offset,
|
||||
uint64_t chunk_byte_size,
|
||||
std::vector<uint8_t>& out_bytes);
|
||||
|
||||
// Read a u32-index range. `chunk_first_index` is RELATIVE to the start of
|
||||
// the index section (i.e. 0 = first u32 index). `chunk_index_count` is in
|
||||
// indices (multiply by 4 internally).
|
||||
bool readSidecarIndexChunk(const std::string& ifc_path,
|
||||
uint64_t index_section_offset,
|
||||
uint64_t chunk_first_index,
|
||||
uint64_t chunk_index_count,
|
||||
std::vector<uint32_t>& out_indices);
|
||||
|
||||
#endif // WGPUSTREAMINGLOADER_H
|
||||
Reference in New Issue
Block a user