diff --git a/src/ifcconvert/IfcConvert.cpp b/src/ifcconvert/IfcConvert.cpp index 8bb776cc73..4e7652e7af 100644 --- a/src/ifcconvert/IfcConvert.cpp +++ b/src/ifcconvert/IfcConvert.cpp @@ -61,6 +61,7 @@ void printUsage(const boost::program_options::options_description& generic_optio << " .igs IGES Initial Graphics Exchange Specification" << std::endl << " .xml XML Property definitions and decomposition tree" << std::endl << " .svg SVG Scalable Vector Graphics (2d floor plan)" << std::endl + << " .hdf HDF5 Efficient binary IFC serialization" << std::endl << std::endl << "Command line options" << std::endl << generic_options << std::endl << "Advanced options" << std::endl << geom_options << std::endl; @@ -131,6 +132,8 @@ int main(int argc, char** argv) { ("bounds", boost::program_options::value(&bounds), "Specifies the bounding rectangle, for example 512x512, to which the " "output will be scaled. Only used when converting to SVG.") + ("compress", + "Compress HDF5.") ("include", "Specifies that the entities listed after --entities are to be included") ("exclude", @@ -250,6 +253,20 @@ int main(int argc, char** argv) { } catch (...) {} write_log(); return exit_code; + } else if (output_extension == ".hdf") { + int exit_code = 1; + // try { + IfcParse::IfcFile f; + if (!f.Init(input_filename)) { + Logger::Message(Logger::LOG_ERROR, "Unable to parse .ifc file"); + } else { + f.write_hdf5(output_filename, vmap.count("compress") != 0); + exit_code = 0; + } + // } catch (...) {} + write_log(); + // std::cin.get(); + return exit_code; } IfcGeom::IteratorSettings settings; diff --git a/src/ifcexpressparser/schema_class.py b/src/ifcexpressparser/schema_class.py index 425b52ee84..899e313fbf 100644 --- a/src/ifcexpressparser/schema_class.py +++ b/src/ifcexpressparser/schema_class.py @@ -64,7 +64,11 @@ class SchemaClass: for cpp_type, collection in collections_by_type: for name in collection.keys(): statements.append('%(cpp_type)s* %(name)s_type = 0;' % locals()) - + + statements.append('#ifdef _MSC_VER' ) + statements.append('#pragma optimize("", off)') + statements.append('#endif' ) + statements.append('schema_definition* populate_schema() {') emitted_types = set() @@ -145,6 +149,10 @@ class SchemaClass: statements.extend(('}','')) + statements.append('#ifdef _MSC_VER' ) + statements.append('#pragma optimize("", on)') + statements.append('#endif' ) + statements.extend(('const schema_definition& get_schema() {', '', ' static const schema_definition* s = populate_schema();', diff --git a/src/ifcparse/Ifc2x3-schema.cpp b/src/ifcparse/Ifc2x3-schema.cpp index 0eb17076e5..c07c4d960d 100644 --- a/src/ifcparse/Ifc2x3-schema.cpp +++ b/src/ifcparse/Ifc2x3-schema.cpp @@ -1009,6 +1009,11 @@ enumeration_type* IfcWindowPanelPositionEnum_type = 0; enumeration_type* IfcWindowStyleConstructionEnum_type = 0; enumeration_type* IfcWindowStyleOperationEnum_type = 0; enumeration_type* IfcWorkControlTypeEnum_type = 0; + +#ifdef _MSC_VER +# pragma optimize( "", off ) +#endif + schema_definition* populate_schema() { IfcAbsorbedDoseMeasure_type = new type_declaration(IfcSchema::Type::IfcAbsorbedDoseMeasure, new simple_type(simple_type::real_type)); IfcAccelerationMeasure_type = new type_declaration(IfcSchema::Type::IfcAccelerationMeasure, new simple_type(simple_type::real_type)); @@ -10374,6 +10379,10 @@ schema_definition* populate_schema() { return new schema_definition("IFC2X3", declarations, true); } +#ifdef _MSC_VER +# pragma optimize( "", on ) +#endif + const schema_definition& get_schema() { static const schema_definition* s = populate_schema(); diff --git a/src/ifcparse/IfcFile.h b/src/ifcparse/IfcFile.h index c0d268931d..517e91defa 100644 --- a/src/ifcparse/IfcFile.h +++ b/src/ifcparse/IfcFile.h @@ -135,6 +135,8 @@ public: std::pair getUnit(IfcSchema::IfcUnitEnum::IfcUnitEnum); const schema_definition* schema() const { return schema_; } + + void write_hdf5(const std::string&, bool compress) const; }; } diff --git a/src/ifcparse/IfcHdf5File.cpp b/src/ifcparse/IfcHdf5File.cpp new file mode 100644 index 0000000000..ed7393ac6e --- /dev/null +++ b/src/ifcparse/IfcHdf5File.cpp @@ -0,0 +1,671 @@ +#include "IfcHdf5File.h" + +#include "H5pubconf.h" + +#ifndef H5_HAVE_FILTER_DEFLATE +#pragma message("warning: HDF5 compression support is recommended") +#endif + +class UnmetDependencyException : public std::exception {}; + +H5::DataType* IfcParse::IfcHdf5File::map_type(const IfcParse::parameter_type* pt) { + H5::DataType* h5_dt; + if (pt->as_aggregation_type()) { + h5_dt = new H5::VarLenType(map_type(pt->as_aggregation_type()->type_of_element())); + } else if (pt->as_named_type()) { + if (pt->as_named_type()->declared_type()->as_entity()) { + h5_dt = new H5::DataType(); + h5_dt->copy(*instance_reference); + } else { + IfcSchema::Type::Enum ty = pt->as_named_type()->declared_type()->type(); + if (declared_types.find(ty) == declared_types.end()) { + throw UnmetDependencyException(); + } else { + h5_dt = new H5::DataType(); + h5_dt->copy(*declared_types[ty]); + } + } + } else if (pt->as_simple_type()) { + const H5::DataType* orig = default_types[pt->as_simple_type()->declared_type()]; + h5_dt = new H5::DataType(); + h5_dt->copy(*orig); + } else { + throw UnmetDependencyException(); + } + return h5_dt; +} + +H5::CompType* create_compound(const std::vector< IfcParse::IfcHdf5File::compound_member >& members) { + size_t s = 0, o = 0; + for (auto it = members.begin(); it != members.end(); ++it) { + s += it->second->getSize(); + } + + H5::CompType* h5_dt = new H5::CompType(s); + for (auto it = members.begin(); it != members.end(); ++it) { + h5_dt->insertMember(it->first, o, *it->second); + o += it->second->getSize(); + } + + return h5_dt; +} + +H5::EnumType* create_enumeration(const std::vector& items, int offset = 0) { + size_t numbytes = 0; + size_t size = items.size(); + while (size != 0) { + size >>= 8; + numbytes ++; + } + + int i = offset; + H5::EnumType* h5_enum = new H5::EnumType(numbytes); + for (auto it = items.begin(); it != items.end(); ++it, ++i) { + h5_enum->insert(it->c_str(), &i); + } + + return h5_enum; +} + +H5::EnumType* map_enumeration(const IfcParse::enumeration_type* en) { + return create_enumeration(en->enumeration_items()); +} + +H5::DataType* IfcParse::IfcHdf5File::commit(H5::DataType* dt, const std::string& name) { + dt->commit(schema_group, name); + return dt; +} + +H5::CompType* IfcParse::IfcHdf5File::map_entity(const IfcParse::entity* e) { + std::vector attributes = e->all_attributes(); + + std::vector< IfcParse::IfcHdf5File::compound_member > h5_attributes; + h5_attributes.reserve(attributes.size() + 2); + + h5_attributes.push_back(std::make_pair(std::string("set_unset_bitmap"), &H5::PredType::NATIVE_INT16)); + h5_attributes.push_back(std::make_pair(std::string("Entity-Instance-Identifier"), &H5::PredType::NATIVE_INT32)); + + const std::vector& attributes_derived_in_subtype = e->derived(); + auto jt = attributes_derived_in_subtype.begin(); + + for (auto it = attributes.begin(); it != attributes.end(); ++it, ++jt) { + if (*jt) { + continue; + } + const std::string& name = (*it)->name(); + const bool is_optional = (*it)->optional(); + H5::DataType* type = map_type((*it)->type_of_attribute()); + h5_attributes.push_back(std::make_pair(name, type)); + } + + return create_compound(h5_attributes); +} + +void IfcParse::IfcHdf5File::init_default_types() { + { + std::vector< compound_member > members; + members.push_back( std::make_pair(std::string("_HDF5_dataset_index_"), new H5::PredType(H5::PredType::NATIVE_INT16)) ); + members.push_back( std::make_pair(std::string("_HDF5_instance_index_"), new H5::PredType(H5::PredType::NATIVE_INT32)) ); + instance_reference = static_cast(commit(create_compound(members), "_HDF_INSTANCE_REFERENCE_HANDLE_")); + } + + object_reference_type = &H5::PredType::STD_REF_OBJ; + + { + std::vector names; + names.push_back("BOOLEAN-FALSE"); + names.push_back("BOOLEAN-TRUE"); + default_types[simple_type::boolean_type] = create_enumeration(names); + } + + { + std::vector names; + names.push_back("LOGICAL-UNKNOWN"); + names.push_back("LOGICAL-FALSE"); + names.push_back("LOGICAL-TRUE"); + default_types[simple_type::logical_type] = create_enumeration(names, -1); + } + + default_types[simple_type::binary_type] = &H5::PredType::NATIVE_OPAQUE; // vlen? + default_types[simple_type::real_type] = &H5::PredType::NATIVE_DOUBLE; + default_types[simple_type::number_type] = &H5::PredType::NATIVE_DOUBLE; + default_types[simple_type::string_type] = new H5::StrType(H5::PredType::C_S1, H5T_VARIABLE); + default_types[simple_type::integer_type] = &H5::PredType::NATIVE_INT; + + default_type_names[simple_type::logical_type] = "logical"; + default_type_names[simple_type::boolean_type] = "boolean"; + default_type_names[simple_type::binary_type] = "binary"; + default_type_names[simple_type::real_type] = "real"; + default_type_names[simple_type::number_type] = "number"; + default_type_names[simple_type::string_type] = "string"; + default_type_names[simple_type::integer_type] = "integer"; + + default_cpp_type_names[IfcUtil::Argument_BOOL] = "boolean"; + default_cpp_type_names[IfcUtil::Argument_DOUBLE] = "real"; + default_cpp_type_names[IfcUtil::Argument_STRING] = "string"; + default_cpp_type_names[IfcUtil::Argument_INT] = "integer"; +} + +void IfcParse::IfcHdf5File::visit_select(const IfcParse::select_type* pt, std::set& leafs) { + for (auto it = pt->select_list().begin(); it != pt->select_list().end(); ++it) { + if ((*it)->as_select_type()) { + visit_select((*it)->as_select_type(), leafs); + } else { + leafs.insert(*it); + } + } +} + +std::string IfcParse::IfcHdf5File::flatten_aggregate_name(const IfcParse::parameter_type* at) const { + if (at->as_aggregation_type()) { + return std::string("aggregate-of-") + flatten_aggregate_name(at->as_aggregation_type()->type_of_element()); + } else if (at->as_simple_type()) { + return default_type_names.find(at->as_simple_type()->declared_type())->second; + } else if (at->as_named_type()) { + // TODO: Unwind type name + return at->as_named_type()->declared_type()->name(); + } else { + throw; + } +} + +H5::DataType* IfcParse::IfcHdf5File::map_select(const IfcParse::select_type* pt) { + std::set leafs; + visit_select(pt, leafs); + + bool all_entity_instance_refs = true; + for (auto it = leafs.begin(); it != leafs.end(); ++it) { + if (!(*it)->as_entity()) { + all_entity_instance_refs = false; + break; + } + } + + if (all_entity_instance_refs) { + return 0; + } else { + std::set member_names; + std::vector h5_attributes; + + h5_attributes.push_back(std::make_pair(std::string("select_bitmap"), &H5::PredType::NATIVE_INT8)); + h5_attributes.push_back(std::make_pair(std::string("type_path"), default_types[simple_type::string_type])); + + for (auto it = leafs.begin(); it != leafs.end(); ++it) { + std::string name; + const H5::DataType* dt = 0; + + const IfcParse::declaration* decl = (*it); + + if ((*it)->as_type_declaration()) { + while (decl->as_type_declaration()) { + const IfcParse::parameter_type* pt = decl->as_type_declaration()->declared_type(); + const IfcParse::named_type* nt = pt->as_named_type(); + const IfcParse::simple_type* st = pt->as_simple_type(); + const IfcParse::aggregation_type* at = pt->as_aggregation_type(); + if (nt) { + decl = nt->declared_type(); + } else if (st) { + name = default_type_names[st->declared_type()]; + dt = default_types[st->declared_type()]; + break; + } else if (at) { + name = flatten_aggregate_name(at); + dt = map_type(at); + break; + } + } + } + + if (!dt) { + if ((*it)->as_entity()) { + name = "instance"; + dt = instance_reference; + } else if ((*it)->as_enumeration_type()) { + name = (*it)->name(); + dt = declared_types[(*it)->type()]; + } else { + throw; + } + } + + if (member_names.find(name) != member_names.end()) { + continue; + } + member_names.insert(name); + + name += "-value"; + h5_attributes.push_back(std::make_pair(name, dt)); + } + + return create_compound(h5_attributes); + } +} + +void IfcParse::IfcHdf5File::advance(void*& ptr, size_t n) const { + ptr = ( uint8_t*) ptr + n; +} + +template +void IfcParse::IfcHdf5File::write(void*& ptr, const T& t) const { + *((T*)ptr) = t; + advance(ptr, sizeof(T)); +} + +template <> +void IfcParse::IfcHdf5File::write(void*& ptr, const std::string& s) const { + char* c = new(allocator.allocate(s.size()+1)) char [s.size()+1]; + strcpy(c, s.c_str()); + write(ptr, c); +} + +template struct uint_of_size {}; +template <> struct uint_of_size <1> { typedef uint8_t type; }; +template <> struct uint_of_size <2> { typedef uint16_t type; }; +template <> struct uint_of_size <4> { typedef uint32_t type; }; +template <> struct uint_of_size <8> { typedef uint64_t type; }; + +template struct int_of_size {}; +template <> struct int_of_size <1> { typedef int8_t type; }; +template <> struct int_of_size <2> { typedef int16_t type; }; +template <> struct int_of_size <4> { typedef int32_t type; }; +template <> struct int_of_size <8> { typedef int64_t type; }; + +template struct float_of_size {}; +template <> struct float_of_size <4> { typedef float type; }; +template <> struct float_of_size <8> { typedef double type; }; + +template +void IfcParse::IfcHdf5File::write_number_of_size(void*& ptr, size_t n, T i) const { + if (!std::numeric_limits::is_integer) { + if (n == 4) { + write(ptr, static_cast< float_of_size<4>::type > (i)); + } else if (n == 8) { + write(ptr, static_cast< float_of_size<8>::type > (i)); + } + } else if (std::numeric_limits::is_signed) { + if (n == 1) { + write(ptr, static_cast< int_of_size<1>::type > (i)); + } else if (n == 2) { + write(ptr, static_cast< int_of_size<2>::type > (i)); + } else if (n == 4) { + write(ptr, static_cast< int_of_size<4>::type > (i)); + } else if (n == 8) { + write(ptr, static_cast< int_of_size<8>::type > (i)); + } + } else { + if (n == 1) { + write(ptr, static_cast< uint_of_size<1>::type > (i)); + } else if (n == 2) { + write(ptr, static_cast< uint_of_size<2>::type > (i)); + } else if (n == 4) { + write(ptr, static_cast< uint_of_size<4>::type > (i)); + } else if (n == 8) { + write(ptr, static_cast< uint_of_size<8>::type > (i)); + } + } +} + +void IfcParse::IfcHdf5File::write_vlen_t(void*& ptr, size_t n_elements, void* vlen_data) const { + advance(ptr, HOFFSET(hvl_t, len)); + void* temp_ptr; + write_number_of_size(temp_ptr = ptr, sizeof(size_t), n_elements); + advance(ptr, HOFFSET(hvl_t, p)); + write(ptr, vlen_data); +} + +template +void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const T& ts) const { + size_t elem_size = get_datatype()->getSize(); + size_t n_elements = ts.size(); + size_t size_in_bytes = elem_size * n_elements; + void* aggr_data = allocator.allocate(size_in_bytes); + void* aggr_ptr = aggr_data; + for (T::const_iterator it = ts.begin(); it != ts.end(); ++it) { + write_number_of_size(aggr_ptr, elem_size, *it); + } + write_vlen_t(ptr, n_elements, aggr_data); +} + +template <> +void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const std::vector& ts) const { + size_t elem_size = sizeof(char*); + size_t n_elements = ts.size(); + size_t size_in_bytes = elem_size * n_elements; + void* aggr_data = allocator.allocate(size_in_bytes); + void* aggr_ptr = aggr_data; + for (std::vector::const_iterator it = ts.begin(); it != ts.end(); ++it) { + write(aggr_ptr, *it); + } + write_vlen_t(ptr, n_elements, aggr_data); +} + +template <> +void IfcParse::IfcHdf5File::write_aggregate(void*& ptr, const IfcEntityList::ptr& ts) const { + size_t elem_size = instance_reference->getSize(); + size_t n_elements = ts->size(); + size_t size_in_bytes = elem_size * n_elements; + void* aggr_data = allocator.allocate(size_in_bytes); + void* aggr_ptr = aggr_data; + for (IfcEntityList::it it = ts->begin(); it != ts->end(); ++it) { + auto ref = make_instance_reference(*it); + write_number_of_size(aggr_ptr, instance_reference->getMemberDataType(0).getSize(), ref.first); + write_number_of_size(aggr_ptr, instance_reference->getMemberDataType(1).getSize(), ref.second); + } + write_vlen_t(ptr, n_elements, aggr_data); +} + +void IfcParse::IfcHdf5File::write_schema(const IfcParse::schema_definition& schema) { + + // From h5ex_g_compact.c + // Compact groups? + H5::FileAccPropList* plist = new H5::FileAccPropList(); + plist->setLibverBounds(H5F_LIBVER_LATEST, H5F_LIBVER_LATEST); + + // H5::FileCreatPropList* plist2 = new H5::FileCreatPropList(); + // H5Pset_file_space(plist2->getId(), H5F_FILE_SPACE_VFD, (hsize_t)0); + + file = new H5::H5File(name_, H5F_ACC_TRUNC, H5::FileCreatPropList::DEFAULT, *plist); + + schema_group = file->createGroup(schema.name() + "_encoding"); + population_group = file->createGroup("population"); + + init_default_types(); + + for (auto it = schema.enumeration_types().begin(); it != schema.enumeration_types().end(); ++it) { + declared_types[(*it)->type()] = commit(map_enumeration((*it)), (*it)->name()); + } + + const std::vector& ts = schema.type_declarations(); + std::set processed; + while (processed.size() < ts.size()) { + for (auto it = ts.begin(); it != ts.end(); ++it) { + const std::string& name = (*it)->name(); + if (processed.find(*it) != processed.end()) continue; + try { + declared_types[(*it)->type()] = commit(map_type((*it)->declared_type()), (*it)->name()); + processed.insert(*it); + } catch(const UnmetDependencyException&) {} + auto pt = (*it)->declared_type(); + } + } + + for (auto it = schema.select_types().begin(); it != schema.select_types().end(); ++it) { + H5::DataType* dt = map_select(*it); + if (dt) { + declared_types[(*it)->type()] = commit(dt, (*it)->name()); + } else { + declared_types[(*it)->type()] = instance_reference; + } + } + + for (auto it = schema.entities().begin(); it != schema.entities().end(); ++it) { + declared_types[(*it)->type()] = commit(map_entity(*it), (*it)->name()); + } +}; + +std::pair IfcParse::IfcHdf5File::make_instance_reference(const IfcUtil::IfcBaseClass* instance) const { + const std::vector& es = sorted_entities.find(instance->declaration().type())->second; + + auto dataset_id = std::lower_bound(dataset_names.begin(), dataset_names.end(), instance->declaration().type()); + auto instance_id = std::lower_bound(es.begin(), es.end(), instance); // TODO: << make a vector of ids to make sure pointers are stable + + size_t dataset_offset = std::distance(dataset_names.begin(), dataset_id); + size_t instance_offset = std::distance(es.begin(), instance_id); + + return std::make_pair(dataset_offset, instance_offset); +} + +void IfcParse::IfcHdf5File::write_select(void*& ptr, IfcUtil::IfcBaseClass* instance, const H5::CompType* datatype) const { + int member_index = -1; + + if (instance->declaration().as_entity()) { + member_index = datatype->getMemberIndex("instance-value"); + size_t offset = datatype->getMemberOffset(member_index); + auto ref = make_instance_reference(instance); + void* ptr_member = (uint8_t*) ptr + offset; + write_number_of_size(ptr_member, instance_reference->getMemberDataType(0).getSize(), ref.first); + write_number_of_size(ptr_member, instance_reference->getMemberDataType(1).getSize(), ref.second); + } else { + Argument& wrapped_data = *instance->data().getArgument(0); + IfcUtil::ArgumentType ty = wrapped_data.type(); + if (default_cpp_type_names.find(ty) == default_cpp_type_names.end()) { + Logger::Message(Logger::LOG_ERROR, "Unsupported select valuation encountered", instance); + } else { + std::string member_name = default_cpp_type_names.find(ty)->second + "-value"; + member_index = datatype->getMemberIndex(member_name); + size_t offset = datatype->getMemberOffset(member_index); + size_t member_size = datatype->getMemberDataType(member_index).getSize(); + void* ptr_member = (uint8_t*) ptr + offset; + switch(wrapped_data.type()) { + case IfcUtil::Argument_BOOL: { + bool instance = wrapped_data; + write_number_of_size(ptr_member, member_size, static_cast(instance ? 1 : 0)); + break; } + case IfcUtil::Argument_DOUBLE: { + double d = wrapped_data; + write_number_of_size(ptr_member, member_size, d); + break; } + case IfcUtil::Argument_STRING: { + std::string s = wrapped_data; + write(ptr_member, s); + break; } + case IfcUtil::Argument_INT: { + int i = wrapped_data; + write_number_of_size(ptr_member, member_size, i); + break; } + default: + Logger::Message(Logger::LOG_ERROR, "Unsupported select valuation encountered", instance); + break; + } + } + } + + if (member_index == -1) { + member_index = 0; + } else { + member_index -= 2; // select_bitmap, type_path + } + + void* temp_ptr = ptr; + write_number_of_size(temp_ptr, H5::PredType::NATIVE_INT8.getSize(), member_index); + write(temp_ptr, instance->declaration().name()); + + // TODO: Should string be set to "", or keep as null? + // std::cout << datatype->getMemberIndex("string-value") << std::endl; + + advance(ptr, datatype->getSize()); +} + +void IfcParse::IfcHdf5File::write_population(const IfcFile& f, bool compress) { + std::set tys; + + for (auto it = f.begin(); it != f.end(); ++it) { + IfcSchema::Type::Enum ty = it->second->declaration().type(); + tys.insert(ty); + // This already is sorted on entity instance name + sorted_entities[ty].push_back(it->second); + } + + dataset_names.assign(tys.begin(), tys.end()); + std::sort(dataset_names.begin(), dataset_names.end()); + + for (auto it = dataset_names.begin(); it != dataset_names.end(); ++it) { + const std::vector& es = sorted_entities.find(*it)->second; + hsize_t dims = es.size(); + // don't chunk too small + // hsize_t chunk = (std::min)(std::max(es.size() / 64, (size_t)8), es.size()); + // hsize_t chunk = (std::min)( (std::max)(es.size() / 8, (size_t) 128 ) , es.size()); + hsize_t chunk = dims; + + const H5::DSetCreatPropList* plist; + // H5O_MESG_MAX_SIZE = 65536 + const bool compact = es.size() * declared_types[*it]->getSize() < (1 << 15); + if (compress && !compact) { + H5::DSetCreatPropList* plist_ = new H5::DSetCreatPropList; + plist_->setChunk(1, &chunk); + // D'oh. Order is significant, according to h5ex_d_shuffle.c + plist_->setShuffle(); + plist_->setDeflate(9); + plist = plist_; + } else if (compact) { + H5::DSetCreatPropList* plist_ = new H5::DSetCreatPropList; + // Set compact according to h5ex_d_compact.c + plist_->setLayout(H5D_COMPACT); + plist = plist_; + } else { + plist = &H5::DSetCreatPropList::DEFAULT; + } + + H5::DataSet ds = population_group.createDataSet(IfcSchema::Type::ToString(*it) + "_instances", *declared_types[*it], H5::DataSpace(1, &dims), *plist); + + void* data = allocator.allocate(declared_types[*it]->getSize() * static_cast(dims)); + + void* ptr = data; + const H5::CompType* dt = (H5::CompType*) declared_types[*it]; + int ind = 0; + + for (auto jt = es.begin(); jt != es.end(); ++jt, ++ind) { + int member_idx = 0; + + H5::DataType member_type; + + IfcAbstractEntity& dat = (*jt)->data(); + const declaration& decl = (*jt)->declaration(); + std::vector attributes = decl.as_entity()->all_attributes(); + const std::vector& attributes_derived_in_subtype = decl.as_entity()->derived(); + std::vector::const_iterator is_derived = attributes_derived_in_subtype.begin(); + + member_type = dt->getMemberDataType(member_idx++); + uint32_t set_unset_mask = 0; + void* set_unset_ptr = ptr; + size_t set_unset_size = member_type.getSize(); + // skip for now write later + advance(ptr, set_unset_size); + + member_type = dt->getMemberDataType(member_idx++); + write_number_of_size(ptr, member_type.getSize(), static_cast(dat.id())); + + for (unsigned i = 0; i < dat.getArgumentCount(); ++i, ++is_derived) { + if (*is_derived) { + continue; + } + + const IfcParse::entity::attribute* schema_attr = attributes[i]; + Argument& attr_value = *dat.getArgument(i); + member_type = dt->getMemberDataType(member_idx++); + + if (attr_value.isNull()) { + if (member_type == *default_types[simple_type::string_type]) { + // HDF5 otherwise crashes on derefencing a zero pointer for the string type + write(ptr, new(allocator.allocate(1)) char(0)); + } else { + memset(ptr, 0, member_type.getSize()); + advance(ptr, member_type.getSize()); + } + } else { + set_unset_mask |= 1 << i; + + if (member_type == *instance_reference) { + IfcUtil::IfcBaseClass* v = attr_value; + auto ref = make_instance_reference(v); + write_number_of_size(ptr, instance_reference->getMemberDataType(0).getSize(), ref.first); + write_number_of_size(ptr, instance_reference->getMemberDataType(1).getSize(), ref.second); + } else if (member_type == *default_types[simple_type::real_type]) { + double d = attr_value; + write_number_of_size(ptr, member_type.getSize(), d); + } else if (member_type == *default_types[simple_type::string_type]) { + std::string s = attr_value; + write(ptr, s); + } else if (member_type == *default_types[simple_type::integer_type]) { + int i = attr_value; + write_number_of_size(ptr, member_type.getSize(), i); + } else if (member_type == *default_types[simple_type::boolean_type] || + member_type == *default_types[simple_type::logical_type]) + { + bool b = attr_value; + write_number_of_size(ptr, member_type.getSize(), static_cast(b ? 1 : 0)); + } else if (member_type.getClass() == H5T_ENUM) { + // NB: Note that boolean and logical are also enums + // NB2: In IfcOpenShell an enum value can be read as a string + std::string s = attr_value; + const std::vector& enum_values = schema_attr->type_of_attribute()->as_named_type()->declared_type()->as_enumeration_type()->enumeration_items(); + size_t d = std::distance(enum_values.begin(), std::find(enum_values.begin(), enum_values.end(), s)); + write_number_of_size(ptr, member_type.getSize(), d); + } else if (member_type.getClass() == H5T_VLEN) { + const parameter_type* pt = schema_attr->type_of_attribute(); + while (pt->as_named_type()) { + pt = pt->as_named_type()->declared_type()->as_type_declaration()->declared_type(); + } + const named_type* nt = pt->as_aggregation_type()->type_of_element()->as_named_type(); + if (nt && nt->declared_type()->as_select_type()) { + const H5::DataType* datatype = declared_types[nt->declared_type()->type()]; + IfcEntityList::ptr es = attr_value; + if (datatype == instance_reference) { + // For a SELECTs with only ENTITY leaves, a blind instance reference type is used + write_aggregate(ptr, es); + } else { + size_t size_in_bytes = datatype->getSize(); + size_t num_elements = es->size(); + void* buffer_ptr; + // void* buffer = buffer_ptr = new uint8_t[size_in_bytes * num_elements]; + void* buffer = buffer_ptr = allocator.allocate(size_in_bytes * num_elements); + for (auto it = es->begin(); it != es->end(); ++it) { + write_select(buffer_ptr, *it, static_cast(datatype)); + } + write_vlen_t(ptr, num_elements, buffer); + } + + } else { + if (sizeof(hvl_t) != member_type.getSize()) throw; + switch(attr_value.type()) { + case IfcUtil::Argument_AGGREGATE_OF_INT: { + std::vector ds = attr_value; + write_aggregate(ptr, ds); + break; } + case IfcUtil::Argument_AGGREGATE_OF_DOUBLE: { + std::vector ds = attr_value; + write_aggregate(ptr, ds); + break; } + case IfcUtil::Argument_AGGREGATE_OF_ENTITY_INSTANCE: { + IfcEntityList::ptr es = attr_value; + write_aggregate(ptr, es); + break; } + case IfcUtil::Argument_AGGREGATE_OF_STRING: { + std::vector ss = attr_value; + write_aggregate(ptr, ss); + break; } + default: + // Can be an empty list in which case parser does not know type + if (attr_value.size() > 0) { + Logger::Message(Logger::LOG_ERROR, "Unsupported aggregate encountered", *jt); + } + memset(ptr, 0, member_type.getSize()); + ptr = (uint8_t*) ptr + member_type.getSize(); + } + } + } else if (member_type.getClass() == H5T_COMPOUND) { + memset(ptr, 0, member_type.getSize()); + + IfcUtil::ArgumentType ty = attr_value.type(); + if (ty != IfcUtil::Argument_ENTITY_INSTANCE) throw; + IfcUtil::IfcBaseClass* b = attr_value; + + write_select(ptr, attr_value, static_cast(&member_type)); + } + } + } + + write_number_of_size(set_unset_ptr, set_unset_size, set_unset_mask); + } + + ds.write(data, *declared_types[*it]); + ds.close(); + + allocator.free(); + + for (auto it = es.begin(); it != es.end(); ++it) { + static_cast(&(**it).data())->Unload(); + } + + } +} \ No newline at end of file diff --git a/src/ifcparse/IfcHdf5File.h b/src/ifcparse/IfcHdf5File.h new file mode 100644 index 0000000000..7fdfcb1706 --- /dev/null +++ b/src/ifcparse/IfcHdf5File.h @@ -0,0 +1,122 @@ +/******************************************************************************** + * * + * This file is part of IfcOpenShell. * + * * + * IfcOpenShell is free software: you can redistribute it and/or modify * + * it under the terms of the Lesser GNU General Public License as published by * + * the Free Software Foundation, either version 3.0 of the License, or * + * (at your option) any later version. * + * * + * IfcOpenShell is distributed in the hope that it will be useful, * + * but WITHOUT ANY WARRANTY; without even the implied warranty of * + * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the * + * Lesser GNU General Public License for more details. * + * * + * You should have received a copy of the Lesser GNU General Public License * + * along with this program. If not, see . * + * * + ********************************************************************************/ + +#ifndef IFCHDF5FILE_H +#define IFCHDF5FILE_H + +#include "H5Cpp.h" + +#include "../ifcparse/IfcUtil.h" +#include "../ifcparse/IfcFile.h" + +namespace IfcParse { + + class IfcHdf5File { + private: + class allocator_t { + private: + // TODO: Change this? + mutable std::vector buffers; + public: + void* allocate(size_t n) const { + return new uint8_t[n]; + } + void free() { + std::vector::const_iterator it; + for (it = buffers.begin(); it != buffers.end(); ++it) { + delete[] *it; + } + buffers.clear(); + } + ~allocator_t() { + free(); + } + }; + + allocator_t allocator; + + std::string name_; + schema_definition schema_; + + H5::H5File* file; + H5::Group schema_group; + H5::Group population_group; + + // TODO: const + H5::CompType* instance_reference; + const H5::DataType* object_reference_type; + + std::map declared_types; + // TODO: why? get_datatype() + std::map default_types; + std::map default_type_names; + std::map default_cpp_type_names; + + std::map > sorted_entities; + std::vector dataset_names; + + H5::DataType* map_type(const IfcParse::parameter_type* pt); + H5::CompType* map_entity(const IfcParse::entity* e); + H5::DataType* map_select(const IfcParse::select_type* pt); + H5::DataType* commit(H5::DataType* dt, const std::string& name); + + void init_default_types(); + void visit_select(const IfcParse::select_type* pt, std::set& leafs); + std::string flatten_aggregate_name(const IfcParse::parameter_type* at) const; + std::pair make_instance_reference(const IfcUtil::IfcBaseClass* instance) const; + void write_select(void*& ptr, IfcUtil::IfcBaseClass* instance, const H5::CompType* datatype) const; + + void write_schema(const schema_definition& schema); + void write_population(const IfcFile& f, bool compress); + + template + const H5::DataType* get_datatype() const { + return default_types.find(IfcUtil::cpp_to_schema_type::schema_type)->second; + } + + void advance(void*& ptr, size_t n) const; + + template + void write(void*& ptr, const T& t) const; + + template <> + void write(void*& ptr, const std::string& s) const; + + void write_vlen_t(void*& ptr, size_t n_elements, void* vlen_data) const; + + template + void write_number_of_size(void*& ptr, size_t n, T i) const; + + template + void write_aggregate(void*& ptr, const T& ts) const; + public: + typedef std::pair compound_member; + + IfcHdf5File(const IfcFile* f, const std::string& name, bool compress) + : name_(name) + , schema_(*f->schema()) + { + write_schema(*f->schema()); + write_population(*f, compress); + }; + }; + +} + +#endif diff --git a/src/ifcparse/IfcParse.cpp b/src/ifcparse/IfcParse.cpp index 2162a9ef82..78a57510d9 100644 --- a/src/ifcparse/IfcParse.cpp +++ b/src/ifcparse/IfcParse.cpp @@ -38,6 +38,7 @@ #include "../ifcparse/IfcFile.h" #include "../ifcparse/IfcSIPrefix.h" #include "../ifcparse/IfcSchema.h" +#include "../ifcparse/IfcHdf5File.h" using namespace IfcParse; @@ -830,6 +831,11 @@ void Entity::Load(std::vector& ids, bool seek) const { if ( ! TokenFunc::isOperator(semilocon,';') ) file->tokens->stream->Seek(old_offset); } +void Entity::Unload() { + delete args; + args = 0; +} + IfcSchema::Type::Enum Entity::type() const { return _type; } @@ -1535,4 +1541,8 @@ std::pair IfcFile::getUnit(IfcSchema::IfcUnitE } } return return_value; +} + +void IfcFile::write_hdf5(const std::string& name, bool compress) const { + IfcHdf5File(this, name, compress); } \ No newline at end of file diff --git a/src/ifcparse/IfcParse.h b/src/ifcparse/IfcParse.h index 6ba81834d0..ff96a3aa75 100644 --- a/src/ifcparse/IfcParse.h +++ b/src/ifcparse/IfcParse.h @@ -130,7 +130,7 @@ namespace IfcParse { std::vector list; void push(Argument* l); public: - ~ArgumentList(); + virtual ~ArgumentList(); void read(IfcSpfLexer* t, std::vector& ids); @@ -206,7 +206,7 @@ namespace IfcParse { IfcUtil::IfcBaseClass* entity; public: EntityArgument(const Token& t); - ~EntityArgument(); + virtual ~EntityArgument(); IfcUtil::ArgumentType type() const; @@ -251,6 +251,7 @@ namespace IfcParse { ~Entity(); IfcEntityList::ptr getInverse(IfcSchema::Type::Enum type, int attribute_index); void Load(std::vector& ids, bool seek=false) const; + void Unload(); Argument* getArgument (unsigned int i); unsigned int getArgumentCount() const; std::string toString(bool upper=false) const; diff --git a/src/ifcparse/IfcSchema.h b/src/ifcparse/IfcSchema.h index 4edc9ba6ab..5e2cab6f94 100644 --- a/src/ifcparse/IfcSchema.h +++ b/src/ifcparse/IfcSchema.h @@ -23,6 +23,7 @@ #include #include #include +#include #ifdef USE_IFC4 #include "../ifcparse/Ifc4enum.h" @@ -243,12 +244,14 @@ namespace IfcParse { const std::vector all_attributes() const { std::vector attrs; attrs.reserve(derived_.size()); - std::vector::iterator it = attrs.begin(); + // std::vector::iterator it = attrs.begin(); if (supertype_) { const std::vector supertype_attrs = supertype_->all_attributes(); - it = std::copy(supertype_attrs.begin(), supertype_attrs.end(), it); + // it = std::copy(supertype_attrs.begin(), supertype_attrs.end(), it); + std::copy(supertype_attrs.begin(), supertype_attrs.end(), std::back_inserter(attrs)); } - std::copy(attributes_.begin(), attributes_.end(), it); + // std::copy(attributes_.begin(), attributes_.end(), it); + std::copy(attributes_.begin(), attributes_.end(), std::back_inserter(attrs)); return attrs; } @@ -266,6 +269,7 @@ namespace IfcParse { std::vector type_declarations_; std::vector select_types_; std::vector enumeration_types_; + std::vector entities_; class declaration_by_name_cmp : public std::binary_function { public: @@ -284,6 +288,7 @@ namespace IfcParse { if ((**it).as_type_declaration()) type_declarations_.push_back((**it).as_type_declaration()); if ((**it).as_select_type()) select_types_.push_back((**it).as_select_type()); if ((**it).as_enumeration_type()) enumeration_types_.push_back((**it).as_enumeration_type()); + if ((**it).as_entity()) entities_.push_back((**it).as_entity()); } } @@ -307,10 +312,13 @@ namespace IfcParse { return declaration_by_name(IfcSchema::Type::ToString(name)); } - const std::vector& declarations() { return declarations_; } - const std::vector& type_declarations() { return type_declarations_; } - const std::vector& select_types() { return select_types_; } - const std::vector& enumeration_types() { return enumeration_types_; } + const std::vector& declarations() const { return declarations_; } + const std::vector& type_declarations() const { return type_declarations_; } + const std::vector& select_types() const { return select_types_; } + const std::vector& enumeration_types() const { return enumeration_types_; } + const std::vector& entities() const { return entities_; } + + const std::string& name() const { return name_; } }; } diff --git a/src/ifcparse/IfcUtil.h b/src/ifcparse/IfcUtil.h index bbacfe9461..89535c4828 100644 --- a/src/ifcparse/IfcUtil.h +++ b/src/ifcparse/IfcUtil.h @@ -130,6 +130,23 @@ namespace IfcUtil { }; bool valid_binary_string(const std::string& s); + + template struct cpp_to_schema_type; + template <> struct cpp_to_schema_type < boost::dynamic_bitset<> > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::binary_type; }; + template <> struct cpp_to_schema_type < bool > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::boolean_type; }; + template <> struct cpp_to_schema_type < int > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::integer_type; }; + template <> struct cpp_to_schema_type < double > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::real_type; }; + template <> struct cpp_to_schema_type < std::string > { static const IfcParse::simple_type::data_type schema_type = IfcParse::simple_type::string_type; }; + + template + struct schema_to_cpp_type; + template <> struct schema_to_cpp_type < IfcParse::simple_type::binary_type > { typedef boost::dynamic_bitset<> cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::boolean_type > { typedef bool cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::integer_type > { typedef int cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::logical_type > { typedef bool cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::number_type > { typedef double cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::real_type > { typedef double cpp_type; }; + template <> struct schema_to_cpp_type < IfcParse::simple_type::string_type > { typedef std::string cpp_type; }; } template