From 0f1b00769ea68e78df87fe0b12d3eaf01c7468ba Mon Sep 17 00:00:00 2001 From: Dion Moult Date: Thu, 29 Feb 2024 15:58:55 +1100 Subject: [PATCH] Chunk all code related to chunking which can now be done in Python --- src/ifcgeom/IfcGeomIteratorImplementation.h | 146 +------------ src/ifcgeom_schema_agnostic/IfcGeomIterator.h | 4 - src/ifcgeom_schema_agnostic/IfcGeomTree.h | 201 ------------------ .../IteratorImplementation.h | 3 - src/ifcgeom_schema_agnostic/chunk.h | 19 -- src/ifcwrap/IfcPython.i | 73 ------- 6 files changed, 1 insertion(+), 445 deletions(-) delete mode 100644 src/ifcgeom_schema_agnostic/chunk.h diff --git a/src/ifcgeom/IfcGeomIteratorImplementation.h b/src/ifcgeom/IfcGeomIteratorImplementation.h index b3bff90ac6..5cf70522f9 100644 --- a/src/ifcgeom/IfcGeomIteratorImplementation.h +++ b/src/ifcgeom/IfcGeomIteratorImplementation.h @@ -153,18 +153,6 @@ namespace IfcGeom { BRepElement* current_shape_model; SerializedElement* current_serialization; - // Structures for chunking - int offset = 0; - int colour_offset = 0; - std::vector chunked_guids; - std::vector chunked_guid_ids; - std::vector chunked_verts; - std::vector chunked_faces; - std::vector chunked_materials; - std::vector chunked_material_ids; - std::vector> chunked_colours; - std::vector> colours; - // A container and iterator for IfcBuildingElements for the current IfcRepresentation referenced by *representation_iterator IfcSchema::IfcProduct::list::ptr ifcproducts; IfcSchema::IfcProduct::list::it ifcproduct_iterator; @@ -772,7 +760,7 @@ namespace IfcGeom { if (rt == GeometrySerializer::READ_TRIANGULATION) { cache_->write((IfcGeom::TriangulationElement*) element); } else { - // cache_->write((IfcGeom::BRepElement*)element); + cache_->write((IfcGeom::BRepElement*)element); } } #endif @@ -952,138 +940,6 @@ namespace IfcGeom { } } - void apply_matrix_to_flat_verts(const std::vector& flat_list, const std::vector& matrix, std::vector& result) { - result.clear(); - result.reserve(flat_list.size()); - - for (size_t i = 0; i < flat_list.size(); i += 3) { - double x = flat_list[i]; - double y = flat_list[i + 1]; - double z = flat_list[i + 2]; - result.push_back(x * matrix[0] + y * matrix[3] + z * matrix[6] + matrix[9]); - result.push_back(x * matrix[1] + y * matrix[4] + z * matrix[7] + matrix[10]); - result.push_back(x * matrix[2] + y * matrix[5] + z * matrix[8] + matrix[11]); - } - } - - bool process_chunk() - { - if (colours.size() == 0) { - colours.push_back({1, 1, 1, 1}); // Fallback "no material" colour - chunked_colours.push_back({1, 1, 1, 1}); - } - - const float tolerance = 0.01f; // Tolerance value for comparison - int chunk_size = 10000; - - // Only works for multithreaded right now - TriangulationElement* ret = 0; - ret = dynamic_cast(*task_result_iterator_); - - const std::vector& matrix = ret->transformation().matrix().data(); - const std::vector& geometry_verts = ret->geometry().verts(); - const std::vector& geometry_faces = ret->geometry().faces(); - const auto& materials = ret->geometry().materials(); - const auto& material_ids = ret->geometry().material_ids(); - - chunked_guids.push_back(ret->guid()); - int guid_ids_size = chunked_guid_ids.size(); - if (guid_ids_size > 0) { - chunked_guid_ids.push_back(chunked_guid_ids[guid_ids_size - 1] + geometry_faces.size() / 3); - } else { - chunked_guid_ids.push_back(geometry_faces.size() / 3); - } - - std::vector verts; - apply_matrix_to_flat_verts(geometry_verts, matrix, verts); - - // Redo with resize and pushback to prevent unneccessary reallocations - std::vector faces = geometry_faces; - for (size_t i = 0; i < faces.size(); ++i) { - faces[i] += offset; - } - - chunked_verts.insert(chunked_verts.end(), verts.begin(), verts.end()); - chunked_faces.insert(chunked_faces.end(), faces.begin(), faces.end()); - - std::vector material_keys; - for (const auto& material : materials) { - float alpha = 1.0; - if (material.hasTransparency() && material.transparency() > 0) { - alpha = 1.0 - material.transparency(); - } - - int i = 0; - bool is_existing_colour = false; - for (const auto& colour : colours) { - if (std::abs(colour[0] - static_cast(material.diffuse()[0])) < tolerance - && std::abs(colour[1] - static_cast(material.diffuse()[1])) < tolerance - && std::abs(colour[2] - static_cast(material.diffuse()[2])) < tolerance - && std::abs(colour[3] - alpha) < tolerance) { - is_existing_colour = true; - break; - } - i++; - } - - if ( ! is_existing_colour) { - colours.push_back({material.diffuse()[0], material.diffuse()[1], material.diffuse()[2], alpha}); - chunked_colours.push_back({material.diffuse()[0], material.diffuse()[1], material.diffuse()[2], alpha}); - } - - int chunked_index = 0; - auto it = std::find(chunked_materials.begin(), chunked_materials.end(), i); - if (it == chunked_materials.end()) { - // material not used so far in chunk - chunked_index = chunked_materials.size(); - chunked_materials.push_back(i); - } else { - // material already in chunk - chunked_index = std::distance(chunked_materials.begin(), it); - } - material_keys.push_back(chunked_index); - } - - if (materials.size() > 0) { - std::vector mat_ids(material_ids.size()); - for (int i = 0; i mat_ids(faces.size() / 3, 0); - chunked_material_ids.insert(chunked_material_ids.end(), mat_ids.begin(), mat_ids.end()); - } - - offset += verts.size() / 3; - - if (offset > chunk_size) { - return true; - } - - return false; - } - - chunk get_chunk() - { - chunk result = { - std::move(chunked_guids), - std::move(chunked_guid_ids), - std::move(chunked_verts), - std::move(chunked_faces), - std::move(chunked_materials), - std::move(chunked_material_ids), - std::move(chunked_colours), - }; - chunked_verts.clear(); - chunked_faces.clear(); - chunked_materials.clear(); - chunked_material_ids.clear(); - chunked_colours.clear(); - offset = 0; - return result; - } - /// Gets the representation of the current geometrical entity. Element* get() { diff --git a/src/ifcgeom_schema_agnostic/IfcGeomIterator.h b/src/ifcgeom_schema_agnostic/IfcGeomIterator.h index 8adf54c585..4f45c108a1 100644 --- a/src/ifcgeom_schema_agnostic/IfcGeomIterator.h +++ b/src/ifcgeom_schema_agnostic/IfcGeomIterator.h @@ -59,7 +59,6 @@ #define IFCGEOMITERATOR_H #include "../ifcgeom_schema_agnostic/IteratorImplementation.h" -#include "../ifcgeom_schema_agnostic/chunk.h" // The infamous min & max Win32 #defines can leak here from OCE depending on the build configuration #ifdef min @@ -139,9 +138,6 @@ namespace IfcGeom { BRepElement* get_native() { return implementation_->get_native(); } - bool process_chunk() { return implementation_->process_chunk(); } - IfcGeom::chunk get_chunk() { return implementation_->get_chunk(); } - const Element* get_object(int id) { return implementation_->get_object(id); } IfcUtil::IfcBaseClass* create() { return implementation_->create(); } diff --git a/src/ifcgeom_schema_agnostic/IfcGeomTree.h b/src/ifcgeom_schema_agnostic/IfcGeomTree.h index bc898488f2..6d8373087c 100644 --- a/src/ifcgeom_schema_agnostic/IfcGeomTree.h +++ b/src/ifcgeom_schema_agnostic/IfcGeomTree.h @@ -86,21 +86,6 @@ namespace IfcGeom { std::array p2; }; - struct h5_shape { - std::vector verts; - std::vector faces; - std::vector materials; - std::vector material_ids; - // Guids only used in chunked results - std::vector guids; - std::vector guid_ids; - }; - - struct chunked_model { - std::vector> materials; - std::vector elements; - }; - namespace { // Approximates the distance `other` protrudes into `volume` by finding the @@ -1884,192 +1869,6 @@ namespace IfcGeom { return hex_str; } - chunked_model load_h5() { - H5::H5File file("/home/dion/cpp.h5", H5F_ACC_RDONLY); - - H5::DataSet materials_ds = file.openDataSet("materials"); - H5::DataSpace materials_s = materials_ds.getSpace(); - hsize_t dims[2]; - materials_s.getSimpleExtentDims(dims); - size_t total_materials = dims[0]; - std::vector buffer(total_materials * 4); // Buffer to hold all materials - - std::cout << "Total mats " << total_materials << std::endl; - std::vector> materials(total_materials); - materials_ds.read(buffer.data(), H5::PredType::NATIVE_FLOAT); - for (size_t i = 0; i < total_materials; ++i) { - materials[i] = std::vector(buffer.begin() + i * 4, buffer.begin() + (i + 1) * 4); - } - - H5::Group shapes_g = file.openGroup("shapes"); - - hsize_t total_shapes = shapes_g.getNumObjs(); - std::vector shapes(total_shapes); - - for (hsize_t i = 0; i < total_shapes; ++i) { - // WARNING: getObjnameByIdx is extremely slow! It makes the entire operation take 5X the time. - //std::string shapeName = shapes_g.getObjnameByIdx(i); - std::string shapeName = std::to_string(i); - H5::Group shapeGroup = shapes_g.openGroup(shapeName); - - // Read "verts" dataset - H5::DataSet vertsDataset = shapeGroup.openDataSet("verts"); - std::vector verts(vertsDataset.getSpace().getSimpleExtentNpoints()); - vertsDataset.read(verts.data(), H5::PredType::NATIVE_FLOAT); - - // Read "faces" dataset - H5::DataSet facesDataset = shapeGroup.openDataSet("faces"); - std::vector faces(facesDataset.getSpace().getSimpleExtentNpoints()); - facesDataset.read(faces.data(), H5::PredType::NATIVE_INT); - - // Read "materials" dataset (if it exists) - std::vector materials; - if (shapeGroup.exists("materials")) { - H5::DataSet materialsDataset = shapeGroup.openDataSet("materials"); - materials.resize(materialsDataset.getSpace().getSimpleExtentNpoints()); - materialsDataset.read(materials.data(), H5::PredType::NATIVE_INT); - } - - // Read "material_ids" dataset (if it exists) - std::vector material_ids; - if (shapeGroup.exists("material_ids")) { - H5::DataSet materialIdsDataset = shapeGroup.openDataSet("material_ids"); - material_ids.resize(materialIdsDataset.getSpace().getSimpleExtentNpoints()); - materialIdsDataset.read(material_ids.data(), H5::PredType::NATIVE_INT); - } - - // Store the shape data - shapes[std::stoi(shapeName)] = {std::move(verts), std::move(faces), std::move(materials), std::move(material_ids)}; - } - - const int chunk_size = 10000; - std::vector elements; - - int offset = 0; - std::vector chunked_guids; - std::vector chunked_guid_ids; - std::vector chunked_verts; - std::vector chunked_faces; - std::vector chunked_materials; - std::vector chunked_material_ids; - - chunked_verts.reserve(chunk_size * 3); - - int i = 0; - - H5::DataSet element_shape_ids_ds = file.openDataSet("element_shape_ids"); - std::vector element_shape_ids(element_shape_ids_ds.getSpace().getSimpleExtentNpoints()); - element_shape_ids_ds.read(element_shape_ids.data(), H5::PredType::NATIVE_INT); - - H5::DataSet element_global_ids_ds = file.openDataSet("element_global_ids"); - H5::DataSpace element_global_ids_s = element_global_ids_ds.getSpace(); - hsize_t element_global_ids_d[16]; - element_global_ids_s.getSimpleExtentDims(element_global_ids_d); - size_t total_element_global_ids = element_global_ids_d[0]; - size_t element_global_ids_size = 16; - std::vector element_global_ids_b(total_element_global_ids * element_global_ids_size); - element_global_ids_ds.read(element_global_ids_b.data(), H5::PredType::NATIVE_UINT8); - - H5::DataSet matrices_ds = file.openDataSet("element_matrices"); - H5::DataSpace matrices_s = matrices_ds.getSpace(); - hsize_t matrices_d[2]; - matrices_s.getSimpleExtentDims(matrices_d); - size_t total_matrices = matrices_d[0]; - size_t matrix_size = 12; - std::vector matrices_b(total_matrices * matrix_size); - matrices_ds.read(matrices_b.data(), H5::PredType::NATIVE_FLOAT); - - std::unordered_map material_map; - for (size_t i = 0; i < total_matrices; ++i) { - material_map.clear(); - - h5_shape& shape = shapes[element_shape_ids[i]]; - - std::vector element_global_id( - element_global_ids_b.begin() + i * element_global_ids_size, - element_global_ids_b.begin() + (i + 1) * element_global_ids_size); - - chunked_guids.push_back(uint8_to_b64(element_global_id)); - int guid_ids_size = chunked_guid_ids.size(); - if (guid_ids_size > 0) { - chunked_guid_ids.push_back(chunked_guid_ids[guid_ids_size - 1] + shape.faces.size() / 3); - } else { - chunked_guid_ids.push_back(shape.faces.size() / 3); - } - - std::vector matrix( - matrices_b.begin() + i * matrix_size, - matrices_b.begin() + (i + 1) * matrix_size); - - std::vector verts; - apply_matrix_to_flat_verts(shape.verts, matrix, verts); - - std::vector faces = shape.faces; - for (size_t i = 0; i < faces.size(); ++i) { - faces[i] += offset; - } - - chunked_verts.insert(chunked_verts.end(), verts.begin(), verts.end()); - chunked_faces.insert(chunked_faces.end(), faces.begin(), faces.end()); - - int material_index = 0; - for (const auto material : shape.materials) { - auto it = std::find(chunked_materials.begin(), chunked_materials.end(), material); - int chunked_index = -1; - if (it == chunked_materials.end()) { - chunked_index = chunked_materials.size(); - chunked_materials.push_back(material); - } else { - chunked_index = std::distance(chunked_materials.begin(), it); - } - material_map[material_index] = chunked_index; - material_index++; - } - - if (shape.material_ids.size() > 0) { - std::vector material_ids(shape.material_ids.size()); - for (int i = 0; i material_ids(shape.faces.size() / 3, material_map.begin()->second); - chunked_material_ids.insert(chunked_material_ids.end(), material_ids.begin(), material_ids.end()); - } - - offset += verts.size() / 3; - - if (offset > chunk_size) { - elements.push_back({ - std::move(chunked_verts), - std::move(chunked_faces), - std::move(chunked_materials), - std::move(chunked_material_ids), - std::move(chunked_guids), - std::move(chunked_guid_ids) - }); - chunked_verts.clear(); - chunked_faces.clear(); - chunked_materials.clear(); - chunked_material_ids.clear(); - offset = 0; - } - } - - if (offset > 0) { - elements.push_back({ - std::move(chunked_verts), - std::move(chunked_faces), - std::move(chunked_materials), - std::move(chunked_material_ids), - std::move(chunked_guids), - std::move(chunked_guid_ids) - }); - } - - return { materials, elements }; - } - void add_triangulation_element(IfcGeom::TriangulationElement* elem, std::string name, std::string global_id) { triangulation_elements_.push_back(elem); const auto& t = elem->product(); diff --git a/src/ifcgeom_schema_agnostic/IteratorImplementation.h b/src/ifcgeom_schema_agnostic/IteratorImplementation.h index 388538c20c..7879263264 100644 --- a/src/ifcgeom_schema_agnostic/IteratorImplementation.h +++ b/src/ifcgeom_schema_agnostic/IteratorImplementation.h @@ -25,7 +25,6 @@ #include "../ifcgeom_schema_agnostic/IfcGeomFilter.h" #include "../ifcgeom_schema_agnostic/GeometrySerializer.h" #include "../ifcgeom_schema_agnostic/IfcGeomIteratorSettings.h" -#include "../ifcgeom_schema_agnostic/chunk.h" #include @@ -72,8 +71,6 @@ namespace IfcGeom { virtual IfcUtil::IfcBaseClass* next() = 0; virtual Element* get() = 0; virtual BRepElement* get_native() = 0; - virtual bool process_chunk() = 0; - virtual IfcGeom::chunk get_chunk() = 0; virtual const Element* get_object(int id) = 0; virtual IfcUtil::IfcBaseClass* create() = 0; virtual ~IteratorImplementation() {} diff --git a/src/ifcgeom_schema_agnostic/chunk.h b/src/ifcgeom_schema_agnostic/chunk.h deleted file mode 100644 index 1f860e1d51..0000000000 --- a/src/ifcgeom_schema_agnostic/chunk.h +++ /dev/null @@ -1,19 +0,0 @@ -// Chunk.h -#ifndef CHUNK_H -#define CHUNK_H - -#include - -namespace IfcGeom { - struct chunk { - std::vector guids; - std::vector guid_ids; - std::vector verts; - std::vector faces; - std::vector materials; - std::vector material_ids; - std::vector> colours; - }; -} - -#endif diff --git a/src/ifcwrap/IfcPython.i b/src/ifcwrap/IfcPython.i index b857796ef1..060c2a731a 100644 --- a/src/ifcwrap/IfcPython.i +++ b/src/ifcwrap/IfcPython.i @@ -17,11 +17,6 @@ * * ********************************************************************************/ -%{ -#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION -#include "numpy/arrayobject.h" -%} - %begin %{ #if defined(_DEBUG) && defined(SWIG_PYTHON_INTERPRETER_NO_DEBUG) /* https://github.com/swig/swig/issues/325 */ @@ -86,7 +81,6 @@ #include "../ifcgeom_schema_agnostic/IfcGeomIterator.h" #include "../ifcgeom_schema_agnostic/Serialization.h" #include "../ifcgeom_schema_agnostic/IfcGeomTree.h" - #include "../ifcgeom_schema_agnostic/chunk.h" #include "../serializers/SvgSerializer.h" #include "../serializers/WavefrontObjSerializer.h" @@ -153,7 +147,6 @@ #include "../ifcgeom_schema_agnostic/IfcGeomIterator.h" #include "../ifcgeom_schema_agnostic/Serialization.h" #include "../ifcgeom_schema_agnostic/IfcGeomTree.h" - #include "../ifcgeom_schema_agnostic/chunk.h" #include "../serializers/SvgSerializer.h" #include "../serializers/WavefrontObjSerializer.h" @@ -211,11 +204,6 @@ %include "IfcGeomWrapper.i" %include "IfcParseWrapper.i" %include "std_vector.i" -%include "../ifcgeom_schema_agnostic/chunk.h" -%include "numpy.i" -%init %{ - import_array(); -%} namespace std { %template(float_array_3) array; @@ -225,65 +213,4 @@ namespace std { %template(StringVector) std::vector; %template(FloatVectorVector) std::vector>; %template(DoubleVectorVector) std::vector>; - %template(H5ShapeVector) std::vector; -} - -%extend IfcGeom::h5_shape { - PyObject* get_verts() { - npy_intp dims[1] = { (npy_intp)$self->verts.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_FLOAT, (void*)$self->verts.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_faces() { - npy_intp dims[1] = { (npy_intp)$self->faces.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->faces.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_materials() { - npy_intp dims[1] = { (npy_intp)$self->materials.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->materials.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_material_ids() { - npy_intp dims[1] = { (npy_intp)$self->material_ids.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->material_ids.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } -} - -%extend IfcGeom::chunk { - PyObject* get_verts() { - npy_intp dims[1] = { (npy_intp)$self->verts.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_DOUBLE, (void*)$self->verts.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_faces() { - npy_intp dims[1] = { (npy_intp)$self->faces.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->faces.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_materials() { - npy_intp dims[1] = { (npy_intp)$self->materials.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->materials.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } - - PyObject* get_material_ids() { - npy_intp dims[1] = { (npy_intp)$self->material_ids.size() }; - PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->material_ids.data()); - PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE); // Make the array read-only - return array; - } }