From 584504dcdca17df8505f5129a5b9a8adb05c8441 Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Mon, 29 Jun 2026 17:43:53 +1000 Subject: [PATCH] ifcviewer-web: stream user sidecars via Blob.slice byte ranges (#88) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picked files are no longer copied whole into the wasm heap. The browser File object stays in JS (Module.__ifcvFile) and is read lazily through Blob.slice byte ranges, so a 200-500 MB sidecar never enters wasm linear memory — only chunk-sized slices do. Mechanism (web-only, #if __EMSCRIPTEN__): - JS glue (EM_JS): ifcvFileSize + ifcvReadRangeInto — slice [off,off+n) of the File and copy it into a caller-provided heap pointer, then call back _ifcv_on_range_done. No malloc across the boundary; C pre-sizes the destination from the read plan. - webReadRangesAsync: reuses planSidecarReadRanges to coalesce a range set into Blob.slice reads (1 MB gap — each slice is an async hop), scatters them into a destination laid out in input order, and fires a continuation when the whole set lands. An in-flight map keyed by id survives unordered_map rehash (scratch buffers are heap-owned). - loadSidecarFromBlobWeb: async metadata load — head (16 B) -> index count -> tail-to-EOF -> parseSidecarHead/Tail -> applyCachedModel, then tags the model streaming_from_blob and frames it. - driveStreamingLoads: blob-sourced models route to beginWebChunkLoad (async vertex+index range reads -> applyStreamedChunk in the callback), holding is_loading until the bytes arrive. The embedded MEMFS sample keeps the synchronous fopen path. shell.html stashes the File and calls _load_sidecar_from_blob_c instead of FS.writeFile'ing the whole thing; EXPORTED_RUNTIME_METHODS=['FS'] dropped. Desktop is untouched (the new members + driveStreamingLoads branch are all emscripten-guarded). Web links clean; desktop rebuilds; 107/107 unit tests pass. Co-Authored-By: Claude Opus 4.8 --- src/ifcviewer-web/CMakeLists.txt | 31 ++-- src/ifcviewer-web/main_web.cpp | 37 ++--- src/ifcviewer-web/shell.html | 36 ++--- src/ifcviewer/ModelGpuData.h | 5 + src/ifcviewer/ViewportCore.cpp | 257 +++++++++++++++++++++++++++++++ src/ifcviewer/ViewportCore.h | 17 ++ 6 files changed, 323 insertions(+), 60 deletions(-) diff --git a/src/ifcviewer-web/CMakeLists.txt b/src/ifcviewer-web/CMakeLists.txt index fdcaab0598..13166c78e7 100644 --- a/src/ifcviewer-web/CMakeLists.txt +++ b/src/ifcviewer-web/CMakeLists.txt @@ -97,31 +97,26 @@ target_link_options(IfcViewerWeb PRIVATE # that starves the device promise (observed: ~10s delay in Firefox). "-sEXIT_RUNTIME=0" # Expose the C entry points to JS. _raf_tick_c drives the RAF loop - # (shell.html); _load_uploaded_model_c loads a user-picked sidecar - # that JS has written into MEMFS. EMSCRIPTEN_KEEPALIVE alone keeps - # the symbols in the binary but doesn't add them to Module. - "-sEXPORTED_FUNCTIONS=['_main','_raf_tick_c','_load_uploaded_model_c']" - # FS lets shell.html's file-browse handler write the picked file's - # bytes into MEMFS before calling _load_uploaded_model_c (which - # fopen()s the virtual path). Drag-drop is intentionally not used — - # it depends on an X11 drag source (file manager), which a minimal - # WM may not provide; the native file chooser is WM-independent. - "-sEXPORTED_RUNTIME_METHODS=['FS']" + # (shell.html); _load_sidecar_from_blob_c loads a user-picked File via + # byte-range Blob.slice reads; _ifcv_on_range_done is the completion + # callback the JS range reader invokes when a slice has landed in the + # heap. EMSCRIPTEN_KEEPALIVE alone keeps the symbols in the binary but + # doesn't add them to Module. + "-sEXPORTED_FUNCTIONS=['_main','_raf_tick_c','_load_sidecar_from_blob_c','_ifcv_on_range_done']" # Streaming + chunked geometry want a heap that can grow as buffers # arrive. 256 MB initial, 2 GB ceiling (matches the wasm32 pointer # cap; --shared64 / MEMORY64 would lift this later if we need it). "-sALLOW_MEMORY_GROWTH=1" "-sINITIAL_MEMORY=268435456" # 256 MB "-sMAXIMUM_MEMORY=2147483648" # 2 GB - # FETCH lets emscripten_fetch issue HTTP Range requests for the - # sidecar byte-range loader. Not used yet by the scaffold but - # needed by the upcoming #27 web streaming I/O backend. + # FETCH lets emscripten_fetch issue HTTP Range requests. The local + # file path (#88) reads byte ranges via Blob.slice and does NOT need + # this; it's retained for the remote-URL Range backend (follow-up). "-sFETCH=1" - # Bundle a small sample sidecar into Emscripten's MEMFS so the - # scaffold can prove the load path end-to-end without needing - # emscripten_fetch + COOP/COEP wiring. The @ separator mounts the - # file at the virtual path the wasm uses to fopen() it. Replaced - # by an emscripten_fetch + Range backend in #88. + # Bundle a small sample sidecar into Emscripten's MEMFS so the page + # renders something on first load without a user pick. The @ separator + # mounts the file at the virtual path the wasm fopen()s. User-picked + # files instead stream via Blob.slice byte ranges (load_sidecar_from_blob_c). "--embed-file=${CMAKE_CURRENT_SOURCE_DIR}/sample.ifcview@/sample.ifcview" # Shell template wraps the JS output in our canvas page. "--shell-file=${CMAKE_CURRENT_SOURCE_DIR}/shell.html" diff --git a/src/ifcviewer-web/main_web.cpp b/src/ifcviewer-web/main_web.cpp index 5a9cc7b0a4..144d6b196b 100644 --- a/src/ifcviewer-web/main_web.cpp +++ b/src/ifcviewer-web/main_web.cpp @@ -41,10 +41,6 @@ namespace { // WebViewportHost selector below. constexpr const char* kCanvasSelector = "#viewer-canvas"; -// MEMFS path the file-browse handler writes the picked sidecar to, and -// that load_uploaded_model_c reads back. Must match shell.html. -constexpr const char* kUploadPath = "/uploads/model.ifcview"; - struct AppState { WebViewportHost host{ kCanvasSelector }; ViewportCore core{ &host }; @@ -156,26 +152,21 @@ extern "C" EMSCRIPTEN_KEEPALIVE void raf_tick_c(void* user) { } } -// Called from shell.html's file-browse handler after it has written the -// picked file's bytes into MEMFS at kUploadPath. Replaces whatever is -// currently loaded (the embedded sample on first use, or a prior upload) -// with the new sidecar and frames it. Geometry becomes resident over the -// next frames via render()'s inline driveStreamingLoads. Exported to JS -// via EXPORTED_FUNCTIONS in CMakeLists.txt. -extern "C" EMSCRIPTEN_KEEPALIVE void load_uploaded_model_c() { +// Called from shell.html's file-browse handler after it has stashed the +// picked File on Module.__ifcvFile. Replaces whatever is currently loaded +// (the embedded sample on first use, or a prior pick) with the new sidecar. +// Byte-range (#88): the whole file is NOT copied into the wasm heap — the +// metadata is read via Blob.slice and chunk bytes stream per-chunk, so a +// 500 MB sidecar stays in the browser File object. Asynchronous: this +// returns immediately and the model frames itself from the JS completion +// callback. Exported to JS via EXPORTED_FUNCTIONS in CMakeLists.txt. +extern "C" EMSCRIPTEN_KEEPALIVE void load_sidecar_from_blob_c() { if (!g_app || !g_app->ready) return; // resetScene drops the previous model's GPU resources so a fresh load - // replaces rather than accumulates (loadSidecarFromPath appends). + // replaces rather than accumulates (loadSidecar* appends). g_app->core.resetScene(); - const unsigned int mid = g_app->core.loadSidecarFromPath(kUploadPath); - if (mid == 0) { - Log::warn() << "ifcviewer-web: uploaded model load failed"; - return; - } - g_app->core.viewAll(); - g_app->host.requestFrame(); - Log::info() << "ifcviewer-web: loaded uploaded model (id " << mid << ")"; + g_app->core.loadSidecarFromBlobWeb(); } int main(int /*argc*/, char** /*argv*/) { @@ -199,8 +190,10 @@ int main(int /*argc*/, char** /*argv*/) { g_app->core.buildPickPipeline(); // Load the embedded sample sidecar (mounted into MEMFS via - // --embed-file in CMakeLists.txt). Replaced by an - // emscripten_fetch + Range backend in #88. + // --embed-file in CMakeLists.txt). The sample stays on the + // synchronous MEMFS read; user-picked files go through the + // Blob.slice byte-range path (load_sidecar_from_blob_c) so large + // sidecars never enter the wasm heap. if (!g_app->core.loadSidecarFromPath("/sample.ifcview")) { Log::warn() << "ifcviewer-web: sample sidecar load failed"; } diff --git a/src/ifcviewer-web/shell.html b/src/ifcviewer-web/shell.html index ad64f1aa88..2438149eac 100644 --- a/src/ifcviewer-web/shell.html +++ b/src/ifcviewer-web/shell.html @@ -88,35 +88,31 @@ statusEl.classList.add('error'); } - // File-browse loading. The picked file's bytes are written into MEMFS - // (the same virtual FS the wasm fopen()s) and then load_uploaded_model_c - // is invoked to read + render it. No drag-drop: that needs an X11 drag - // source (a file manager), which a minimal WM may not provide; the - // native file chooser this button opens is WM-independent. + // File-browse loading (#88, byte-range). The picked File object is stashed + // on Module.__ifcvFile and load_sidecar_from_blob_c reads it lazily via + // Blob.slice — the file is NOT copied into the wasm heap, so a 500 MB + // sidecar stays in the browser File object and only chunk-sized slices + // ever cross into wasm. No drag-drop: that needs an X11 drag source (a + // file manager), which a minimal WM may not provide; the native file + // chooser this button opens is WM-independent. var openBtn = document.getElementById('open-btn'); var fileInput = document.getElementById('file-input'); openBtn.addEventListener('click', function() { fileInput.click(); }); fileInput.addEventListener('change', function(ev) { var f = ev.target.files && ev.target.files[0]; if (!f) return; - if (!Module.FS || !Module._load_uploaded_model_c) { + if (!Module._load_sidecar_from_blob_c) { statusEl.textContent += 'viewer not ready yet — wait for WebGPU init\n'; return; } - var reader = new FileReader(); - reader.onload = function() { - try { - var bytes = new Uint8Array(reader.result); - try { Module.FS.mkdir('/uploads'); } catch (e) { /* already exists */ } - Module.FS.writeFile('/uploads/model.ifcview', bytes); - Module._load_uploaded_model_c(); - } catch (e) { - statusEl.textContent += 'load failed: ' + e + '\n'; - statusEl.classList.add('error'); - } - fileInput.value = ''; // let the same file be re-picked - }; - reader.readAsArrayBuffer(f); + try { + Module.__ifcvFile = f; // kept alive for lazy Blob.slice reads + Module._load_sidecar_from_blob_c(); + } catch (e) { + statusEl.textContent += 'load failed: ' + e + '\n'; + statusEl.classList.add('error'); + } + fileInput.value = ''; // let the same file be re-picked });