python changes for latebound schema manipulation from python

src/ifcopenshell-python/ifcopenshell/__init__.py

@@ -73,6 +73,10 @@ def create_entity(type, *args, **kwargs):
     for idx, arg in attrs:
         e[idx] = arg
     return e
-
+    
+gcroot = []
+def register_schema(schema):
+    gcroot.append(schema)
+    ifcopenshell_wrapper.register_schema(schema.schema)
 
 from .main import *

src/ifcopenshell-python/ifcopenshell/express/bootstrap.py

@@ -17,6 +17,7 @@
 #                                                                             #
 ###############################################################################
 
+import os
 import sys
 import string
 import operator
@@ -98,7 +99,7 @@ expression << (union | factor)
 grammar = OneOrMore(Group(rule))
 grammar.ignore(HASH + restOfLine)
 
-express = grammar.parseFile(sys.argv[1])
+express = grammar.parseFile(os.path.join(os.path.dirname(__file__), 'express.bnf'))
 
 def find_bytype(expr, ty, li = None):
     if li is None: li = []
@@ -191,7 +192,7 @@ from nodes import *
 
 def parse(fn):
     cache_file = fn + ".cache.dat"
-    if os.path.exists(cache_file):
+    if os.path.exists(cache_file) and os.path.getmtime(cache_file) >= os.path.getmtime(fn):
         with open(cache_file, "rb") as f:
             m = pickle.load(f)
     else:      

src/ifcopenshell-python/ifcopenshell/express/nodes.py

@@ -83,7 +83,9 @@ def format_clause(exp):
 
 class TypeDeclaration(Node):
     name = property(lambda self: self.type_id[0])
-    type = property(lambda self: self.underlying_type.any().any())
+    utype = property(lambda self: self.underlying_type.any().any())
+    type = property(lambda self: self.utype[0] if isinstance(self.utype, list) else self.utype)
+    
     def init(self):   
         
         assert hasattr(self, "TYPE")

src/ifcopenshell-python/ifcopenshell/express/schema_class.py

@@ -25,8 +25,235 @@ import templates
 
 from collections import defaultdict
 
+import ifcopenshell.ifcopenshell_wrapper as w
+
+class LateBoundSchemaInstantiator:
+    
+    def __init__(self, schema_name):
+        self.schema_name = schema_name
+        self.schema_name_title = schema_name.capitalize()                      
+        self.declarations = {}
+        self.names = []
+        # We need to make sure anonymous types are not gc'ed.
+        self.cache = []
+                      
+    def aggregation_type(self, aggr_type, bound1, bound2, decl_type):
+        self.cache.append(w.aggregation_type(getattr(w.aggregation_type, aggr_type + "_type"), bound1, bound2, decl_type))
+        return self.cache[-1]
+        
+    def simple_type(self, type):
+        self.cache.append(w.simple_type(getattr(w.simple_type, type + "_type")))
+        return self.cache[-1]
+        
+    def named_type(self, type):
+        self.cache.append(w.named_type(self.declarations[str(type)]))
+        return self.cache[-1]
+        
+    def declare(self, definition_type, name):
+        self.names.append(str(name))
+        
+    def begin_schema(self):
+        self.names.sort(key=str.lower)
+        
+    def typedef(self, name, declared_type):
+        index_in_schema = self.names.index(str(name))
+        self.declarations[str(name)] = w.type_declaration(name, index_in_schema, declared_type)
+        
+    def enumeration(self, name, enum):
+        schema_name = self.schema_name
+        index_in_schema = self.names.index(str(name))
+        self.declarations[str(name)] = w.enumeration_type(name, index_in_schema, sorted(enum.values))
+        
+    def entity(self, name, type):
+        index_in_schema = self.names.index(str(name))
+        supertype = None if len(type.supertypes) == 0 else self.declarations[str(type.supertypes[0])]
+        self.declarations[str(name)] = w.entity(name, type.abstract, index_in_schema, supertype)
+        
+    def select(self, name, type):
+        index_in_schema = self.names.index(str(name))
+        children = [self.declarations[str(v)] for v in type.values]
+        self.declarations[str(name)] = w.select_type(name, index_in_schema, children)
+        
+    def entity_attributes(self, name, attribute_definitions, is_derived):
+        attributes = []
+        for attr_name, decl_type, optional in attribute_definitions:
+            attributes.append(w.attribute(attr_name, decl_type, optional))
+        self.declarations[str(name)].set_attributes(attributes, is_derived)
+        self.cache.append(attributes)
+        
+    def inverse_attributes(self, name, inv_attrs):
+        attributes = []
+        for attr_name, aggr_type, bound1, bound2, entity_ref, attribute_entity, attribute_entity_index in inv_attrs:
+            en = self.declarations[str(entity_ref)]
+            attributes.append(w.inverse_attribute(attr_name, getattr(w.inverse_attribute, aggr_type + "_type"), bound1, bound2, en, en.attributes()[attribute_entity_index]))
+        self.declarations[str(name)].set_inverse_attributes(attributes)
+        
+    def entity_subtypes(self, name, tys):
+        self.declarations[str(name)].set_subtypes([self.declarations[str(v)] for v in tys])
+        
+    def finalize(self, can_be_instantiated_set, override_schema_name = None):
+        self.schema = w.schema_definition(override_schema_name or self.schema_name, list(self.declarations.values()), None)
+        
+
+class EarlyBoundCodeWriter:
+    
+    def __init__(self, schema_name):
+        self.schema_name = schema_name
+        self.schema_name_title = schema_name.capitalize()
+        
+        self.statements = ['',
+                      '#include "../ifcparse/IfcSchema.h"',
+                      '#include "../ifcparse/%(schema_name_title)s.h"' % self.__dict__,
+                      '',
+                      'using namespace IfcParse;',
+                      '']
+                      
+        self.names = []
+                      
+    def aggregation_type(self, aggr_type, bound1, bound2, decl_type):
+        return "new aggregation_type(aggregation_type::%(aggr_type)s_type, %(bound1)d, %(bound2)d, %(decl_type)s)" % locals()
+        
+    def simple_type(self, type):
+        return "new simple_type(simple_type::%s_type)" % type
+        
+    def named_type(self, type):
+        return "new named_type(%s_%s_type)" % (self.schema_name, type)
+        
+    def declare(self, definition_type, name):
+        schema_name = self.schema_name
+        self.statements.append('%(definition_type)s* %(schema_name)s_%(name)s_type = 0;' % locals())
+        self.names.append(name)
+        
+    def begin_schema(self):
+        self.names.sort(key=str.lower)
+        
+        self.statements.append("{factory_placeholder}")
+                      
+        self.statements.append("""
+#if defined(__clang__)
+__attribute__((optnone))
+#elif defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC push_options
+#pragma GCC optimize ("O0")
+#elif defined(_MSC_VER)
+#pragma optimize("", off)
+#endif
+        """)
+        self.statements.append('IfcParse::schema_definition* %s_populate_schema() {' % self.schema_name)
+        
+    def typedef(self, name, declared_type):
+        schema_name = self.schema_name
+        index_in_schema = self.names.index(name)
+        self.statements.append('    %(schema_name)s_%(name)s_type = new type_declaration("%(name)s", %(index_in_schema)d, %(declared_type)s);' % locals())
+        
+    def enumeration(self, name, enum):
+        schema_name = self.schema_name
+        index_in_schema = self.names.index(name)
+        self.statements.append('    {')
+        self.statements.append('        std::vector<std::string> items; items.reserve(%d);' % len(enum.values))
+        self.statements.extend(map(lambda v: '        items.push_back("%s");' % v, sorted(enum.values)))
+        self.statements.append('        %(schema_name)s_%(name)s_type = new enumeration_type("%(name)s", %(index_in_schema)d, items);' % locals())
+        self.statements.append('    }')
+        
+    def entity(self, name, type):
+        schema_name = self.schema_name
+        index_in_schema = self.names.index(name)
+        supertype = '0' if len(type.supertypes) == 0 else '%s_%s_type' % (self.schema_name, type.supertypes[0])
+        is_abstract = "true" if type.abstract else "false"
+        self.statements.append('    %(schema_name)s_%(name)s_type = new entity("%(name)s", %(is_abstract)s, %(index_in_schema)d, %(supertype)s);' % locals())
+        
+    def select(self, name, type):
+        schema_name = self.schema_name
+        index_in_schema = self.names.index(name)
+        self.statements.append('    {')
+        self.statements.append('        std::vector<const declaration*> items; items.reserve(%d);' % len(type.values))
+        self.statements.extend(map(lambda v: '        items.push_back(%s_%s_type);' % (self.schema_name, v), sorted(map(str, type.values))))
+        self.statements.append('        %(schema_name)s_%(name)s_type = new select_type("%(name)s", %(index_in_schema)d, items);' % locals())
+        self.statements.append('    }')
+        
+    def entity_attributes(self, name, attribute_definitions, is_derived):
+        schema_name = self.schema_name
+        self.statements.append('    {')
+        self.statements.append('        std::vector<const attribute*> attributes; attributes.reserve(%d);' % len(attribute_definitions))
+        for attr_name, decl_type, optional in attribute_definitions:
+            optional_cpp = str(optional).lower()
+            self.statements.append('        attributes.push_back(new attribute("%(attr_name)s", %(decl_type)s, %(optional_cpp)s));' % locals())
+        self.statements.append('        std::vector<bool> derived; derived.reserve(%d);' % len(is_derived))
+        self.statements.append('        ' + " ".join(map(lambda b: 'derived.push_back(%s);' % str(b).lower(), is_derived)))
+        self.statements.append('        %(schema_name)s_%(name)s_type->set_attributes(attributes, derived);' % locals())
+        self.statements.append('    }')
+        
+    def inverse_attributes(self, name, inv_attrs):
+        schema_name = self.schema_name
+        self.statements.append('    {')
+        self.statements.append('        std::vector<const inverse_attribute*> attributes; attributes.reserve(%d);' % len(inv_attrs))
+        for attr_name, aggr_type, bound1, bound2, entity_ref, attribute_entity, attribute_entity_index in inv_attrs:
+            self.statements.append('        attributes.push_back(new inverse_attribute("%(attr_name)s", inverse_attribute::%(aggr_type)s_type, %(bound1)d, %(bound2)d, %(schema_name)s_%(entity_ref)s_type, %(schema_name)s_%(attribute_entity)s_type->attributes()[%(attribute_entity_index)d]));' % locals())
+        self.statements.append('        %(schema_name)s_%(name)s_type->set_inverse_attributes(attributes);' % locals())
+        self.statements.append('    }')
+        
+    def entity_subtypes(self, name, tys):
+        schema_name = self.schema_name
+        self.statements.append('    {')
+        self.statements.append('        std::vector<const entity*> defs; defs.reserve(%d);' % len(tys))
+        self.statements.append(('        ' + "".join(map(lambda t: ("defs.push_back(%%(schema_name)s_%s_type);" % t), tys))) % locals())
+        self.statements.append('        %(schema_name)s_%(name)s_type->set_subtypes(defs);' % locals())
+        self.statements.append('    }')
+        
+    def finalize(self, can_be_instantiated_set):
+        schema_name = self.schema_name
+        schema_name_title = self.schema_name.capitalize()
+        
+        num_declarations = len(self.names)
+        
+        self.statements.append('')
+        self.statements.append('    std::vector<const declaration*> declarations; declarations.reserve(%(num_declarations)d);' % locals())
+        for type_name in self.names:
+            self.statements.append('    declarations.push_back(%(schema_name)s_%(type_name)s_type);' % locals())
+            
+        self.statements.append('    return new schema_definition("%(schema_name)s", declarations, new %(schema_name)s_instance_factory());' % locals())
+        
+        self.statements.extend(('}',''))
+        
+        self.statements.append("""
+#if defined(__clang__)
+#elif defined(__GNUC__) || defined(__GNUG__)
+#pragma GCC pop_options
+#elif defined(_MSC_VER)
+#pragma optimize("", on)
+#endif
+        """)
+        
+        self.statements.extend(('const schema_definition& %s::get_schema() {' % schema_name_title,
+                           '',
+                           '    static const schema_definition* s = %(schema_name)s_populate_schema();' % locals(),
+                           '    return *s;',
+                           '}','',''))
+                           
+        def can_be_instantiated(idx_name):
+            name = idx_name[1]
+            return name in can_be_instantiated_set
+                           
+        instance_mapping = """switch(data->type()->index_in_schema()) {
+            %s
+            default: throw IfcParse::IfcException(data->type()->name() + " cannot be instantiated");
+        }
+""" % "\n            ".join(map(lambda tup: ("case %%d: return new ::%s::%%s(data);" % schema_name_title) % tup, filter(can_be_instantiated, enumerate(self.names))))
+
+        self.statements[self.statements.index("{factory_placeholder}")] = """
+class %(schema_name)s_instance_factory : public IfcParse::instance_factory {
+    virtual IfcUtil::IfcBaseClass* operator()(IfcEntityInstanceData* data) const {
+        %(instance_mapping)s
+    }
+};
+""" % locals()
+
+    def __str__(self):
+        return  "\n".join(self.statements)
+
+
 class SchemaClass(codegen.Base):
-    def __init__(self, mapping):
+    def __init__(self, mapping, code=EarlyBoundCodeWriter):
         
         class UnmetDependenciesException(Exception): pass
         
@@ -35,6 +262,8 @@ class SchemaClass(codegen.Base):
 
         declared_types = []
         
+        x = code(schema_name)
+        
         def get_declared_type(type, emitted_names=None):
             if isinstance(type, nodes.SimpleType):
                 type = type.type
@@ -46,19 +275,19 @@ class SchemaClass(codegen.Base):
                 make_bound = lambda b: -1 if b == '?' else int(b)
                 bound1, bound2 = map(make_bound, (type.bounds.lower, type.bounds.upper))
                 decl_type = get_declared_type(type.type, emitted_names)
-                return "new aggregation_type(aggregation_type::%(aggr_type)s_type, %(bound1)d, %(bound2)d, %(decl_type)s)" % locals()
+                return x.aggregation_type(aggr_type, bound1, bound2, decl_type)
             elif isinstance(type, nodes.BinaryType):
-                return "new simple_type(simple_type::binary_type)"
+                return x.simple_type("binary")
             elif isinstance(type, nodes.StringType):
-                return "new simple_type(simple_type::string_type)"
+                return x.simple_type("string")
             elif isinstance(type, str):
                 if mapping.schema.is_type(type) or mapping.schema.is_entity(type):
                     if emitted_names is None or type.lower() in emitted_names:
-                        return "new named_type(%s_%s_type)" % (schema_name, type)
+                        return x.named_type(type)
                     else:
                         raise UnmetDependenciesException(type)
                 else:
-                    return "new simple_type(simple_type::%s_type)" % type
+                    return x.simple_type(type)
             else:
                 raise ValueError("No mapping for '%s'" % type)
 
@@ -79,37 +308,20 @@ class SchemaClass(codegen.Base):
             else:
                 raise Exception("No declared type for <%r>" % type)
                     
-        statements = ['',
-                      '#include "../ifcparse/IfcSchema.h"',
-                      '#include "../ifcparse/%(schema_name_title)s.h"' % locals(),
-                      '',
-                      'using namespace IfcParse;',
-                      '']
+        
 
         collections_by_type = (('entity',           mapping.schema.entities    ),
                                ('type_declaration', mapping.schema.simpletypes ),
                                ('select_type',      mapping.schema.selects     ),
                                ('enumeration_type', mapping.schema.enumerations))
 
-        for cpp_type, collection in collections_by_type:
+        for definition_type, collection in collections_by_type:
             for name in collection.keys():
-                statements.append('%(cpp_type)s* %(schema_name)s_%(name)s_type = 0;' % locals())
-                
+                x.declare(definition_type, name)
+                                
         declarations_by_index = []
         
-        statements.append("{factory_placeholder}")
-                      
-        statements.append("""
-#if defined(__clang__)
-__attribute__((optnone))
-#elif defined(__GNUC__) || defined(__GNUG__)
-#pragma GCC push_options
-#pragma GCC optimize ("O0")
-#elif defined(_MSC_VER)
-#pragma optimize("", off)
-#endif
-        """)
-        statements.append('IfcParse::schema_definition* %(schema_name)s_populate_schema() {' % locals())
+        x.begin_schema()
         
         emitted = set()
         len_to_emit = len(mapping.schema)
@@ -122,29 +334,19 @@ __attribute__((optnone))
                 # print("Unmet", repr(name))
                 return False
 
-            statements.append('    %(schema_name)s_%(name)s_type = new type_declaration("%(name)s", %%(index_in_schema_%(name)s)d, %(declared_type)s);' % locals())
+            x.typedef(name, declared_type)
             
         def write_enumeration(schema_name, name, enum):
-            statements.append('    {')
-            statements.append('        std::vector<std::string> items; items.reserve(%d);' % len(enum.values))
-            statements.extend(map(lambda v: '        items.push_back("%s");' % v, sorted(enum.values)))
-            statements.append('        %(schema_name)s_%(name)s_type = new enumeration_type("%(name)s", %%(index_in_schema_%(name)s)d, items);' % locals())
-            statements.append('    }')
+            x.enumeration(name, enum)
             
         def write_entity(schema_name, name, type):
             if len(type.supertypes) == 0 or set(map(lambda s: s.lower(), type.supertypes)) < emitted:
-                supertype = '0' if len(type.supertypes) == 0 else '%s_%s_type' % (schema_name, type.supertypes[0])
-                is_abstract = "true" if type.abstract else "false"
-                statements.append('    %(schema_name)s_%(name)s_type = new entity("%(name)s", %(is_abstract)s, %%(index_in_schema_%(name)s)d, %(supertype)s);' % locals())
+                x.entity(name, type)
             else: return False
                 
         def write_select(schema_name, name, type):
             if set(map(lambda s: str(s).lower(), type.values)) < emitted:
-                statements.append('    {')
-                statements.append('        std::vector<const declaration*> items; items.reserve(%d);' % len(type.values))
-                statements.extend(map(lambda v: '        items.push_back(%s_%s_type);' % (schema_name, v), sorted(map(str, type.values))))
-                statements.append('        %(schema_name)s_%(name)s_type = new select_type("%(name)s", %%(index_in_schema_%(name)s)d, items);' % locals())
-                statements.append('    }')
+                x.select(name, type)
             else: return False
                                 
         def write(name):
@@ -175,22 +377,16 @@ __attribute__((optnone))
         for name, type in mapping.schema.entities.items():
             derived = set(mapping.derived_in_supertype(type))
             attribute_names = list(map(operator.attrgetter('name'), mapping.arguments(type)))
-            
-            statements.append('    {')
-            statements.append('        std::vector<const attribute*> attributes; attributes.reserve(%d);' % len(type.attributes))
+            is_derived = [b in derived for b in attribute_names]
+            attribute_definitions = []
             for attr in type.attributes:
-                attr_name, optional = attr.name, str(attr.optional).lower()
                 decl_type = get_declared_type(attr.type)
-                statements.append('        attributes.push_back(new attribute("%(attr_name)s", %(decl_type)s, %(optional)s));' % locals())
-            statements.append('        std::vector<bool> derived; derived.reserve(%d);' % len(attribute_names))
-            statements.append('        ' + " ".join(map(lambda b: 'derived.push_back(%s);' % str(b in derived).lower(), attribute_names)))
-            statements.append('        %(schema_name)s_%(name)s_type->set_attributes(attributes, derived);' % locals())
-            statements.append('    }')
+                attribute_definitions.append((attr.name, decl_type, attr.optional))
+            x.entity_attributes(name, attribute_definitions, is_derived)
             
         for name, type in mapping.schema.entities.items():
             if type.inverse:
-                statements.append('    {')
-                statements.append('        std::vector<const inverse_attribute*> attributes; attributes.reserve(%d);' % len(type.inverse))
+                inv_attrs = []
                 for attr in type.inverse:
                     if attr.bounds:
                         make_bound = lambda b: -1 if b == '?' else int(b)
@@ -200,9 +396,8 @@ __attribute__((optnone))
                     attr_name, aggr_type, entity_ref = attr.name, attr.type, attr.entity
                     if aggr_type is None: aggr_type = 'unspecified'
                     attribute_entity, attribute_entity_index = find_inverse_name_and_index(entity_ref, attr.attribute)
-                    statements.append('        attributes.push_back(new inverse_attribute("%(attr_name)s", inverse_attribute::%(aggr_type)s_type, %(bound1)d, %(bound2)d, %(schema_name)s_%(entity_ref)s_type, %(schema_name)s_%(attribute_entity)s_type->attributes()[%(attribute_entity_index)d]));' % locals())
-                statements.append('        %(schema_name)s_%(name)s_type->set_inverse_attributes(attributes);' % locals())
-                statements.append('    }')
+                    inv_attrs.append((attr_name, aggr_type, bound1, bound2, entity_ref, attribute_entity, attribute_entity_index))
+                x.inverse_attributes(name, inv_attrs)
                 
         subtypes = defaultdict(list)
         
@@ -211,65 +406,14 @@ __attribute__((optnone))
                 subtypes[ty].append(name)
                 
         for name, tys in subtypes.items():
-            statements.append('    {')
-            statements.append('        std::vector<const entity*> defs; defs.reserve(%d);' % len(tys))
-            statements.append(('        ' + "".join(map(lambda t: ("defs.push_back(%%(schema_name)s_%s_type);" % t), tys))) % locals())
-            statements.append('        %(schema_name)s_%(name)s_type->set_subtypes(defs);' % locals())
-            statements.append('    }')
-            
-        statements.append('')
-        statements.append('    std::vector<const declaration*> declarations; declarations.reserve(%(num_declarations)d);' % locals())
-        for type_name in declared_types:
-            statements.append('    declarations.push_back(%(type_name)s);' % locals())
-            
-        statements.append('    return new schema_definition("%(schema_name)s", declarations, new %(schema_name)s_instance_factory());' % locals())
-        
-        statements.extend(('}',''))
-        
-        statements.append("""
-#if defined(__clang__)
-#elif defined(__GNUC__) || defined(__GNUG__)
-#pragma GCC pop_options
-#elif defined(_MSC_VER)
-#pragma optimize("", on)
-#endif
-        """)
-        
-        statements.extend(('const schema_definition& %s::get_schema() {' % schema_name_title,
-                           '',
-                           '    static const schema_definition* s = %(schema_name)s_populate_schema();' % locals(),
-                           '    return *s;',
-                           '}','',''))
-                           
-        declarations_by_index.sort(key=str.lower)
-        declarations_by_index_map = dict(("index_in_schema_%s" % j,i) for i,j in enumerate(declarations_by_index))
-        
-        def bind(s):
-            if "%" in s: return s % declarations_by_index_map
-            else: return s
-            
+            x.entity_subtypes(name, tys)
+                                      
         can_be_instantiated_set = set(list(mapping.schema.entities.keys()) + list(mapping.schema.simpletypes.keys()))
-        def can_be_instantiated(idx_name):
-            name = idx_name[1]
-            return name in can_be_instantiated_set
+        x.finalize(can_be_instantiated_set)
                            
-        instance_mapping = """switch(data->type()->index_in_schema()) {
-            %s
-            default: throw IfcParse::IfcException(data->type()->name() + " cannot be instantiated");
-        }
-""" % "\n            ".join(map(lambda tup: ("case %%d: return new ::%s::%%s(data);" % schema_name_title) % tup, filter(can_be_instantiated, enumerate(declarations_by_index))))
-
-        statements[statements.index("{factory_placeholder}")] = """
-class %(schema_name)s_instance_factory : public IfcParse::instance_factory {
-    virtual IfcUtil::IfcBaseClass* operator()(IfcEntityInstanceData* data) const {
-        %(instance_mapping)s
-    }
-};
-""" % locals()
-                           
-        self.str = "\n".join(map(bind, statements))
-        
-        self.file_name = '%s-schema.cpp'%self.schema_name
+        self.str = str(x)
+        self.file_name = '%s-schema.cpp' % self.schema_name
+        self.code = x
 
     def __repr__(self):
         return self.str