Chunk all code related to chunking which can now be done in Python

src/ifcgeom/IfcGeomIteratorImplementation.h

@@ -153,18 +153,6 @@ namespace IfcGeom {
 		BRepElement* current_shape_model;
 		SerializedElement* current_serialization;
 
-        // Structures for chunking
-        int offset = 0;
-        int colour_offset = 0;
-        std::vector<std::string> chunked_guids;
-        std::vector<int> chunked_guid_ids;
-        std::vector<double> chunked_verts;
-        std::vector<int> chunked_faces;
-        std::vector<int> chunked_materials;
-        std::vector<int> chunked_material_ids;
-        std::vector<std::vector<float>> chunked_colours;
-        std::vector<std::array<float, 4>> colours;
-
 		// A container and iterator for IfcBuildingElements for the current IfcRepresentation referenced by *representation_iterator
 		IfcSchema::IfcProduct::list::ptr ifcproducts;
 		IfcSchema::IfcProduct::list::it ifcproduct_iterator;
@@ -772,7 +760,7 @@ namespace IfcGeom {
 				if (rt == GeometrySerializer::READ_TRIANGULATION) {
 					cache_->write((IfcGeom::TriangulationElement*) element);
 				} else {
-					// cache_->write((IfcGeom::BRepElement*)element);
+					cache_->write((IfcGeom::BRepElement*)element);
 				}				
 			}
 #endif
@@ -952,138 +940,6 @@ namespace IfcGeom {
 			}
 		}
 
-        void apply_matrix_to_flat_verts(const std::vector<double>& flat_list, const std::vector<double>& matrix, std::vector<double>& result) {
-            result.clear();
-            result.reserve(flat_list.size());
-
-            for (size_t i = 0; i < flat_list.size(); i += 3) {
-                double x = flat_list[i];
-                double y = flat_list[i + 1];
-                double z = flat_list[i + 2];
-                result.push_back(x * matrix[0] + y * matrix[3] + z * matrix[6] + matrix[9]);
-                result.push_back(x * matrix[1] + y * matrix[4] + z * matrix[7] + matrix[10]);
-                result.push_back(x * matrix[2] + y * matrix[5] + z * matrix[8] + matrix[11]);
-            }
-        }
-
-        bool process_chunk()
-        {
-            if (colours.size() == 0) {
-                colours.push_back({1, 1, 1, 1}); // Fallback "no material" colour
-                chunked_colours.push_back({1, 1, 1, 1});
-            }
-
-            const float tolerance = 0.01f; // Tolerance value for comparison
-            int chunk_size = 10000;
-
-            // Only works for multithreaded right now
-            TriangulationElement* ret = 0;
-            ret = dynamic_cast<IfcGeom::TriangulationElement*>(*task_result_iterator_);
-
-            const std::vector<double>& matrix = ret->transformation().matrix().data();
-            const std::vector<double>& geometry_verts = ret->geometry().verts();
-            const std::vector<int>& geometry_faces = ret->geometry().faces();
-            const auto& materials = ret->geometry().materials();
-            const auto& material_ids = ret->geometry().material_ids();
-
-            chunked_guids.push_back(ret->guid());
-            int guid_ids_size = chunked_guid_ids.size();
-            if (guid_ids_size > 0) {
-                chunked_guid_ids.push_back(chunked_guid_ids[guid_ids_size - 1] + geometry_faces.size() / 3);
-            } else {
-                chunked_guid_ids.push_back(geometry_faces.size() / 3);
-            }
-
-            std::vector<double> verts;
-            apply_matrix_to_flat_verts(geometry_verts, matrix, verts);
-
-            // Redo with resize and pushback to prevent unneccessary reallocations
-            std::vector<int> faces = geometry_faces;
-            for (size_t i = 0; i < faces.size(); ++i) {
-                faces[i] += offset;
-            }
-
-            chunked_verts.insert(chunked_verts.end(), verts.begin(), verts.end());
-            chunked_faces.insert(chunked_faces.end(), faces.begin(), faces.end());
-
-            std::vector<int> material_keys;
-            for (const auto& material : materials) {
-                float alpha = 1.0;
-                if (material.hasTransparency() && material.transparency() > 0) {
-                    alpha = 1.0 - material.transparency();
-                }
-
-                int i = 0;
-                bool is_existing_colour = false;
-                for (const auto& colour : colours) {
-                    if (std::abs(colour[0] - static_cast<float>(material.diffuse()[0])) < tolerance
-                            && std::abs(colour[1] - static_cast<float>(material.diffuse()[1])) < tolerance
-                            && std::abs(colour[2] - static_cast<float>(material.diffuse()[2])) < tolerance
-                            && std::abs(colour[3] - alpha) < tolerance) {
-                        is_existing_colour = true;
-                        break;
-                    }
-                    i++;
-                }
-
-                if ( ! is_existing_colour) {
-                    colours.push_back({material.diffuse()[0], material.diffuse()[1], material.diffuse()[2], alpha});
-                    chunked_colours.push_back({material.diffuse()[0], material.diffuse()[1], material.diffuse()[2], alpha});
-                }
-
-                int chunked_index = 0;
-                auto it = std::find(chunked_materials.begin(), chunked_materials.end(), i);
-                if (it == chunked_materials.end()) {
-                    // material not used so far in chunk
-                    chunked_index = chunked_materials.size();
-                    chunked_materials.push_back(i);
-                } else {
-                    // material already in chunk
-                    chunked_index = std::distance(chunked_materials.begin(), it);
-                }
-                material_keys.push_back(chunked_index);
-            }
-
-            if (materials.size() > 0) {
-                std::vector<int> mat_ids(material_ids.size());
-                for (int i = 0; i<material_ids.size(); ++i) {
-                    mat_ids[i] = material_keys[material_ids[i]];
-                }
-                chunked_material_ids.insert(chunked_material_ids.end(), mat_ids.begin(), mat_ids.end());
-            } else {
-                std::vector<int> mat_ids(faces.size() / 3, 0);
-                chunked_material_ids.insert(chunked_material_ids.end(), mat_ids.begin(), mat_ids.end());
-            }
-
-            offset += verts.size() / 3;
-
-            if (offset > chunk_size) {
-                return true;
-            }
-
-            return false;
-        }
-
-        chunk get_chunk()
-        {
-            chunk result = {
-                std::move(chunked_guids),
-                std::move(chunked_guid_ids),
-                std::move(chunked_verts),
-                std::move(chunked_faces),
-                std::move(chunked_materials),
-                std::move(chunked_material_ids),
-                std::move(chunked_colours),
-            };
-            chunked_verts.clear();
-            chunked_faces.clear();
-            chunked_materials.clear();
-            chunked_material_ids.clear();
-            chunked_colours.clear();
-            offset = 0;
-            return result;
-        }
-
         /// Gets the representation of the current geometrical entity.
         Element* get()
         {

src/ifcgeom_schema_agnostic/IfcGeomIterator.h

@@ -59,7 +59,6 @@
 #define IFCGEOMITERATOR_H
 
 #include "../ifcgeom_schema_agnostic/IteratorImplementation.h"
-#include "../ifcgeom_schema_agnostic/chunk.h"
 
 // The infamous min & max Win32 #defines can leak here from OCE depending on the build configuration
 #ifdef min
@@ -139,9 +138,6 @@ namespace IfcGeom {
 
 		BRepElement* get_native() { return implementation_->get_native(); }
 
-		bool process_chunk() { return implementation_->process_chunk(); }
-        IfcGeom::chunk get_chunk() { return implementation_->get_chunk(); }
-
 		const Element* get_object(int id) { return implementation_->get_object(id); }
 
 		IfcUtil::IfcBaseClass* create() { return implementation_->create(); }

src/ifcgeom_schema_agnostic/IfcGeomTree.h

@@ -86,21 +86,6 @@ namespace IfcGeom {
 		std::array<double, 3> p2;
 	};
 
-    struct h5_shape {
-        std::vector<float> verts;
-        std::vector<int> faces;
-        std::vector<int> materials;
-        std::vector<int> material_ids;
-        // Guids only used in chunked results
-        std::vector<std::string> guids;
-        std::vector<int> guid_ids;
-    };
-
-    struct chunked_model {
-        std::vector<std::vector<float>> materials;
-        std::vector<h5_shape> elements;
-    };
-
 	namespace {
 
 		// Approximates the distance `other` protrudes into `volume` by finding the
@@ -1884,192 +1869,6 @@ namespace IfcGeom {
             return hex_str;
         }
 
-        chunked_model load_h5() {
-            H5::H5File file("/home/dion/cpp.h5", H5F_ACC_RDONLY);
-
-            H5::DataSet materials_ds = file.openDataSet("materials");
-            H5::DataSpace materials_s = materials_ds.getSpace();
-            hsize_t dims[2];
-            materials_s.getSimpleExtentDims(dims);
-            size_t total_materials = dims[0];
-            std::vector<float> buffer(total_materials * 4); // Buffer to hold all materials
-
-            std::cout << "Total mats " << total_materials << std::endl;
-            std::vector<std::vector<float>> materials(total_materials);
-            materials_ds.read(buffer.data(), H5::PredType::NATIVE_FLOAT);
-            for (size_t i = 0; i < total_materials; ++i) {
-                materials[i] = std::vector<float>(buffer.begin() + i * 4, buffer.begin() + (i + 1) * 4);
-            }
-
-            H5::Group shapes_g = file.openGroup("shapes");
-
-            hsize_t total_shapes = shapes_g.getNumObjs();
-            std::vector<h5_shape> shapes(total_shapes);
-
-            for (hsize_t i = 0; i < total_shapes; ++i) {
-                // WARNING: getObjnameByIdx is extremely slow! It makes the entire operation take 5X the time.
-                //std::string shapeName = shapes_g.getObjnameByIdx(i);
-                std::string shapeName = std::to_string(i);
-                H5::Group shapeGroup = shapes_g.openGroup(shapeName);
-
-                // Read "verts" dataset
-                H5::DataSet vertsDataset = shapeGroup.openDataSet("verts");
-                std::vector<float> verts(vertsDataset.getSpace().getSimpleExtentNpoints());
-                vertsDataset.read(verts.data(), H5::PredType::NATIVE_FLOAT);
-
-                // Read "faces" dataset
-                H5::DataSet facesDataset = shapeGroup.openDataSet("faces");
-                std::vector<int> faces(facesDataset.getSpace().getSimpleExtentNpoints());
-                facesDataset.read(faces.data(), H5::PredType::NATIVE_INT);
-
-                // Read "materials" dataset (if it exists)
-                std::vector<int> materials;
-                if (shapeGroup.exists("materials")) {
-                    H5::DataSet materialsDataset = shapeGroup.openDataSet("materials");
-                    materials.resize(materialsDataset.getSpace().getSimpleExtentNpoints());
-                    materialsDataset.read(materials.data(), H5::PredType::NATIVE_INT);
-                }
-
-                // Read "material_ids" dataset (if it exists)
-                std::vector<int> material_ids;
-                if (shapeGroup.exists("material_ids")) {
-                    H5::DataSet materialIdsDataset = shapeGroup.openDataSet("material_ids");
-                    material_ids.resize(materialIdsDataset.getSpace().getSimpleExtentNpoints());
-                    materialIdsDataset.read(material_ids.data(), H5::PredType::NATIVE_INT);
-                }
-
-                // Store the shape data
-                shapes[std::stoi(shapeName)] = {std::move(verts), std::move(faces), std::move(materials), std::move(material_ids)};
-            }
-
-            const int chunk_size = 10000;
-            std::vector<h5_shape> elements;
-
-            int offset = 0;
-            std::vector<std::string> chunked_guids;
-            std::vector<int> chunked_guid_ids;
-            std::vector<float> chunked_verts;
-            std::vector<int> chunked_faces;
-            std::vector<int> chunked_materials;
-            std::vector<int> chunked_material_ids;
-
-            chunked_verts.reserve(chunk_size * 3);
-
-            int i = 0;
-
-            H5::DataSet element_shape_ids_ds = file.openDataSet("element_shape_ids");
-            std::vector<int> element_shape_ids(element_shape_ids_ds.getSpace().getSimpleExtentNpoints());
-            element_shape_ids_ds.read(element_shape_ids.data(), H5::PredType::NATIVE_INT);
-
-            H5::DataSet element_global_ids_ds = file.openDataSet("element_global_ids");
-            H5::DataSpace element_global_ids_s = element_global_ids_ds.getSpace();
-            hsize_t element_global_ids_d[16];
-            element_global_ids_s.getSimpleExtentDims(element_global_ids_d);
-            size_t total_element_global_ids = element_global_ids_d[0];
-            size_t element_global_ids_size = 16;
-            std::vector<uint8_t> element_global_ids_b(total_element_global_ids * element_global_ids_size);
-            element_global_ids_ds.read(element_global_ids_b.data(), H5::PredType::NATIVE_UINT8);
-
-            H5::DataSet matrices_ds = file.openDataSet("element_matrices");
-            H5::DataSpace matrices_s = matrices_ds.getSpace();
-            hsize_t matrices_d[2];
-            matrices_s.getSimpleExtentDims(matrices_d);
-            size_t total_matrices = matrices_d[0];
-            size_t matrix_size = 12;
-            std::vector<float> matrices_b(total_matrices * matrix_size);
-            matrices_ds.read(matrices_b.data(), H5::PredType::NATIVE_FLOAT);
-
-            std::unordered_map<int, int> material_map;
-            for (size_t i = 0; i < total_matrices; ++i) {
-                material_map.clear();
-
-                h5_shape& shape = shapes[element_shape_ids[i]];
-
-                std::vector<uint8_t> element_global_id(
-                    element_global_ids_b.begin() + i * element_global_ids_size,
-                    element_global_ids_b.begin() + (i + 1) * element_global_ids_size);
-
-                chunked_guids.push_back(uint8_to_b64(element_global_id));
-                int guid_ids_size = chunked_guid_ids.size();
-                if (guid_ids_size > 0) {
-                    chunked_guid_ids.push_back(chunked_guid_ids[guid_ids_size - 1] + shape.faces.size() / 3);
-                } else {
-                    chunked_guid_ids.push_back(shape.faces.size() / 3);
-                }
-
-                std::vector<float> matrix(
-                    matrices_b.begin() + i * matrix_size,
-                    matrices_b.begin() + (i + 1) * matrix_size);
-
-                std::vector<float> verts;
-                apply_matrix_to_flat_verts(shape.verts, matrix, verts);
-
-                std::vector<int> faces = shape.faces;
-                for (size_t i = 0; i < faces.size(); ++i) {
-                    faces[i] += offset;
-                }
-
-                chunked_verts.insert(chunked_verts.end(), verts.begin(), verts.end());
-                chunked_faces.insert(chunked_faces.end(), faces.begin(), faces.end());
-
-                int material_index = 0;
-                for (const auto material : shape.materials) {
-                    auto it = std::find(chunked_materials.begin(), chunked_materials.end(), material);
-                    int chunked_index = -1;
-                    if (it == chunked_materials.end()) {
-                        chunked_index = chunked_materials.size();
-                        chunked_materials.push_back(material);
-                    } else {
-                        chunked_index = std::distance(chunked_materials.begin(), it);
-                    }
-                    material_map[material_index] = chunked_index;
-                    material_index++;
-                }
-
-                if (shape.material_ids.size() > 0) {
-                    std::vector<int> material_ids(shape.material_ids.size());
-                    for (int i = 0; i<shape.material_ids.size(); ++i) {
-                        material_ids[i] = material_map[shape.material_ids[i]];
-                    }
-                    chunked_material_ids.insert(chunked_material_ids.end(), material_ids.begin(), material_ids.end());
-                } else {
-                    std::vector<int> material_ids(shape.faces.size() / 3, material_map.begin()->second);
-                    chunked_material_ids.insert(chunked_material_ids.end(), material_ids.begin(), material_ids.end());
-                }
-
-                offset += verts.size() / 3;
-
-                if (offset > chunk_size) {
-                    elements.push_back({
-                        std::move(chunked_verts),
-                        std::move(chunked_faces),
-                        std::move(chunked_materials),
-                        std::move(chunked_material_ids),
-                        std::move(chunked_guids),
-                        std::move(chunked_guid_ids)
-                    });
-                    chunked_verts.clear();
-                    chunked_faces.clear();
-                    chunked_materials.clear();
-                    chunked_material_ids.clear();
-                    offset = 0;
-                }
-            }
-
-            if (offset > 0) {
-                elements.push_back({
-                    std::move(chunked_verts),
-                    std::move(chunked_faces),
-                    std::move(chunked_materials),
-                    std::move(chunked_material_ids),
-                    std::move(chunked_guids),
-                    std::move(chunked_guid_ids)
-                });
-            }
-
-            return { materials, elements };
-        }
-
 		void add_triangulation_element(IfcGeom::TriangulationElement* elem, std::string name, std::string global_id) {
             triangulation_elements_.push_back(elem);
             const auto& t = elem->product();

src/ifcgeom_schema_agnostic/IteratorImplementation.h

@@ -25,7 +25,6 @@
 #include "../ifcgeom_schema_agnostic/IfcGeomFilter.h"
 #include "../ifcgeom_schema_agnostic/GeometrySerializer.h"
 #include "../ifcgeom_schema_agnostic/IfcGeomIteratorSettings.h"
-#include "../ifcgeom_schema_agnostic/chunk.h"
 
 #include <gp_XYZ.hxx>
 
@@ -72,8 +71,6 @@ namespace IfcGeom {
 		virtual IfcUtil::IfcBaseClass* next() = 0;
 		virtual Element* get() = 0;
 		virtual BRepElement* get_native() = 0;
-		virtual bool process_chunk() = 0;
-		virtual IfcGeom::chunk get_chunk() = 0;
 		virtual const Element* get_object(int id) = 0;
 		virtual IfcUtil::IfcBaseClass* create() = 0;
 		virtual ~IteratorImplementation() {}

src/ifcgeom_schema_agnostic/chunk.h

@@ -1,19 +0,0 @@
-// Chunk.h
-#ifndef CHUNK_H
-#define CHUNK_H
-
-#include <vector>
-
-namespace IfcGeom {
-    struct chunk {
-        std::vector<std::string> guids;
-        std::vector<int> guid_ids;
-        std::vector<double> verts;
-        std::vector<int> faces;
-        std::vector<int> materials;
-        std::vector<int> material_ids;
-        std::vector<std::vector<float>> colours;
-    };
-}
-
-#endif

src/ifcwrap/IfcPython.i

@@ -17,11 +17,6 @@
  *                                                                              *
  ********************************************************************************/
 
-%{
-#define NPY_NO_DEPRECATED_API NPY_1_7_API_VERSION
-#include "numpy/arrayobject.h"
-%}
-
 %begin %{
 #if defined(_DEBUG) && defined(SWIG_PYTHON_INTERPRETER_NO_DEBUG)
 /* https://github.com/swig/swig/issues/325 */
@@ -86,7 +81,6 @@
 	#include "../ifcgeom_schema_agnostic/IfcGeomIterator.h"
 	#include "../ifcgeom_schema_agnostic/Serialization.h"
 	#include "../ifcgeom_schema_agnostic/IfcGeomTree.h"
-	#include "../ifcgeom_schema_agnostic/chunk.h"
 
 	#include "../serializers/SvgSerializer.h"
 	#include "../serializers/WavefrontObjSerializer.h"
@@ -153,7 +147,6 @@
 	#include "../ifcgeom_schema_agnostic/IfcGeomIterator.h"
 	#include "../ifcgeom_schema_agnostic/Serialization.h"
 	#include "../ifcgeom_schema_agnostic/IfcGeomTree.h"
-	#include "../ifcgeom_schema_agnostic/chunk.h"
 
 	#include "../serializers/SvgSerializer.h"
 	#include "../serializers/WavefrontObjSerializer.h"
@@ -211,11 +204,6 @@
 %include "IfcGeomWrapper.i"
 %include "IfcParseWrapper.i"
 %include "std_vector.i"
-%include "../ifcgeom_schema_agnostic/chunk.h"
-%include "numpy.i"
-%init %{
-    import_array();
-%}
 	
 namespace std {
   %template(float_array_3) array<double, 3>;
@@ -225,65 +213,4 @@ namespace std {
   %template(StringVector) std::vector<std::string>;
   %template(FloatVectorVector) std::vector<std::vector<float>>;
   %template(DoubleVectorVector) std::vector<std::vector<double>>;
-  %template(H5ShapeVector) std::vector<IfcGeom::h5_shape>;
-}
-
-%extend IfcGeom::h5_shape {
-    PyObject* get_verts() {
-        npy_intp dims[1] = { (npy_intp)$self->verts.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_FLOAT, (void*)$self->verts.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_faces() {
-        npy_intp dims[1] = { (npy_intp)$self->faces.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->faces.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_materials() {
-        npy_intp dims[1] = { (npy_intp)$self->materials.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->materials.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_material_ids() {
-        npy_intp dims[1] = { (npy_intp)$self->material_ids.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->material_ids.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-}
-
-%extend IfcGeom::chunk {
-    PyObject* get_verts() {
-        npy_intp dims[1] = { (npy_intp)$self->verts.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_DOUBLE, (void*)$self->verts.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_faces() {
-        npy_intp dims[1] = { (npy_intp)$self->faces.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->faces.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_materials() {
-        npy_intp dims[1] = { (npy_intp)$self->materials.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->materials.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
-
-    PyObject* get_material_ids() {
-        npy_intp dims[1] = { (npy_intp)$self->material_ids.size() };
-        PyObject* array = PyArray_SimpleNewFromData(1, dims, NPY_INT, (void*)$self->material_ids.data());
-        PyArray_CLEARFLAGS((PyArrayObject*)array, NPY_ARRAY_WRITEABLE);  // Make the array read-only
-        return array;
-    }
 }