Performance improvement of IfcParse tokenizer (#82)

* Applied MSVC tabify to IfcParse.cpp/h.

* Now Argument base class has throwing implementation for each conversion operator (removed those from derived classes).

* Added NullArgument class for the case when null arguments appear after setting elements.

* Now operator tokens are handled just like any other token.

* Refactoring: Token is now a struct with good fields, not a nameless pair.

* Now end position of token is set at its construction.

* Now type of token is determined on construction, as* function simply check stored type.

* Now int-s, float-s and some etc are parsed when token is created.

* Implemented simple way to remove spaces in parsing.

* Tokens are now created without allocations.

* Call RemoveTokenSeparators once for every token.

* Added temporary std::string object in lexer (for optimization purposes).

* Reimplemented asString without allocations.

* Parsing keywords without allocations.

* Added checks for type to tokenFunc::as*** functions.

src/ifcparse/IfcParse.cpp

@@ -100,8 +100,8 @@ void init_locale() {
 // Opens the file, gets the filesize and reads a chunk in memory
 //
 IfcSpfStream::IfcSpfStream(const std::string& fn)
-    : stream(0)
-    , buffer(0)
+		: stream(0)
+		, buffer(0)
 {
 	eof = false;
 #ifdef _MSC_VER
@@ -134,8 +134,8 @@ IfcSpfStream::IfcSpfStream(const std::string& fn)
 }
 
 IfcSpfStream::IfcSpfStream(std::istream& f, int l)
-    : stream(0)
-    , buffer(0)
+		: stream(0)
+		, buffer(0)
 {
 	eof = false;
 	size = l;
@@ -151,8 +151,8 @@ IfcSpfStream::IfcSpfStream(std::istream& f, int l)
 }
 
 IfcSpfStream::IfcSpfStream(void* data, int l)
-    : stream(0)
-    , buffer(0)
+		: stream(0)
+		, buffer(0)
 {
 	eof = false;
 	size = l;
@@ -333,11 +333,11 @@ unsigned int IfcSpfLexer::skipComment() {
 //
 Token IfcSpfLexer::Next() {
 
-	if ( stream->eof ) return TokenPtr();
+	if ( stream->eof ) return NoneTokenPtr();
 
 	while (skipWhitespace() || skipComment()) {}
 	
-	if ( stream->eof ) return TokenPtr();
+	if ( stream->eof ) return NoneTokenPtr();
 	unsigned int pos = stream->Tell();
 
 	char c = stream->Peek();
@@ -345,7 +345,7 @@ Token IfcSpfLexer::Next() {
 	// If the cursor is at [()=,;$*] we know token consists of single char
 	if (c == '(' || c == ')' || c == '=' || c == ',' || c == ';' || c == '$' || c == '*') {
 		stream->Inc();
-		return TokenPtr(c);
+		return OperatorTokenPtr(this, pos, pos+1);
 	}
 
 	int len = 0;
@@ -361,142 +361,200 @@ Token IfcSpfLexer::Next() {
 		// If a string is encountered defer processing to the IfcCharacterDecoder
 		if ( c == '\'' ) decoder->dryRun();
 	}
-	if ( len ) return TokenPtr(this,pos);
-	else return TokenPtr();
+	if ( len ) return GeneralTokenPtr(this, pos, stream->Tell());
+	else return NoneTokenPtr();
 }
 
 //
 // Reads a std::string from the file at specified offset
 // Omits whitespace and comments
 //
-std::string IfcSpfLexer::TokenString(unsigned int offset) {
+void IfcSpfLexer::TokenString(unsigned int offset, std::string &buffer) {
 	const bool was_eof = stream->eof;
 	unsigned int old_offset = stream->Tell();
 	stream->Seek(offset);
-	std::string buffer;
-	buffer.reserve(128);
+	buffer.clear();
 	while ( ! stream->eof ) {
 		char c = stream->Peek();
 		if ( buffer.size() && (c == '(' || c == ')' || c == '=' || c == ',' || c == ';' || c == '/') ) break;
 		stream->Inc();
 		if ( c == ' ' || c == '\r' || c == '\n' || c == '\t' ) continue;
-		else if ( c == '\'' ) return *decoder;
+		else if ( c == '\'' ) {
+			buffer = *decoder;
+			return;
+		}
 		else buffer.push_back(c);
 	}
 	if ( was_eof ) stream->eof = true;
 	else stream->Seek(old_offset);
-	return buffer;
 }
 
-//
-// Functions for creating Tokens from an arbitary file offset.
-// The first 4 bits are reserved for Tokens of type ()=,;$*
-//
-Token IfcParse::TokenPtr(IfcSpfLexer* tokens, unsigned int offset) { return Token(tokens,offset); }
-Token IfcParse::TokenPtr(char c) { return Token((IfcSpfLexer*)0,(unsigned) c); }
-Token IfcParse::TokenPtr() { return Token((IfcSpfLexer*)0,0); }
+//Note: according to STEP standard, there may be newlines in tokens
+inline void RemoveTokenSeparators(IfcSpfStream* stream, unsigned start, unsigned end, std::string &oDestination) {
+	oDestination.clear();
+	for (unsigned i = start; i < end; i++) {
+		char c = stream->Read(i);
+		if (c == ' ' || c == '\r' || c == '\n' || c == '\t')
+			continue;
+		oDestination += c;
+	}
+}
 
-//
-// Functions to convert Tokens to binary data
-//
-bool TokenFunc::startsWith(const Token& t, char c) {
-	return t.first->stream->Read(t.second) == c;
+bool ParseInt(const char *pStart, int &val) {
+	char* pEnd;
+	long result = strtol(pStart, &pEnd, 10);
+	if (*pEnd != 0)
+		return false;
+	val = (int)result;
+	return true;
+}
+
+bool ParseFloat(const char *pStart, double &val) {
+	char* pEnd;
+#ifdef _MSC_VER
+	double result = _strtod_l(pStart, &pEnd, locale);
+#else
+	double result = strtod_l(pStart, &pEnd, locale);
+#endif
+	if (*pEnd != 0)
+		return false;
+	val = result;
+	return true;
+}
+
+bool ParseBool(const char *pStart, bool &val) {
+	if (strlen(pStart) != 3 || pStart[0] != '.' || pStart[2] != '.')
+		return false;
+	char mid = pStart[1];
+	if (!(mid == 'T' || mid == 'F'))
+		return false;
+	val = (mid == 'T');
+	return true;
+}
+
+Token IfcParse::OperatorTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
+	char first = lexer->stream->Read(start);
+	Token token(lexer, start, end, Token_OPERATOR);
+	token.value_char = first;
+	return token;
+}
+
+Token IfcParse::GeneralTokenPtr(IfcSpfLexer* lexer, unsigned start, unsigned end) {
+	Token token(lexer, start, end, Token_NONE);
+
+	//extract token into temp buffer (remove eol-s, no encoding changes)
+	std::string &tokenStr = lexer->GetTempString();
+	RemoveTokenSeparators(lexer->stream, start, end, tokenStr);
+	
+	//determine type of the token
+	char first = lexer->stream->Read(start);
+	if (first == '#') {
+		token.type = Token_IDENTIFIER;
+		if (!ParseInt(tokenStr.c_str() + 1, token.value_int))
+			throw IfcException("Identifier token as not integer");
+	}
+	else if (first == '\'')
+		token.type = Token_STRING;
+	else if (first == '.') {
+		token.type = Token_ENUMERATION;
+		if (ParseBool(tokenStr.c_str(), token.value_bool)) //bool is also enumeration
+			token.type = Token_BOOL;
+	}
+	else if (first == '"')
+		token.type = Token_BINARY;
+	else if (ParseInt(tokenStr.c_str(), token.value_int))
+		token.type = Token_INT;
+	else if (ParseFloat(tokenStr.c_str(), token.value_double))
+		token.type = Token_FLOAT;
+	else
+		token.type = Token_KEYWORD;
+
+	return token;
+}
+Token IfcParse::NoneTokenPtr() { return Token(); }
+
+
+bool TokenFunc::isOperator(const Token& t) {
+	return t.type == Token_OPERATOR;
 }
 
 bool TokenFunc::isOperator(const Token& t, char op) {
-	return (!t.first) && (!op || (unsigned)op == t.second);
+	return t.type == Token_OPERATOR && t.value_char == op;
 }
 
 bool TokenFunc::isIdentifier(const Token& t) {
-	return ! isOperator(t) && startsWith(t, '#');
+	return t.type == Token_IDENTIFIER;
 }
 
 bool TokenFunc::isString(const Token& t) {
-	return ! isOperator(t) && startsWith(t, '\'');
+	return t.type == Token_STRING;
 }
 
 bool TokenFunc::isEnumeration(const Token& t) {
-	return ! isOperator(t) && startsWith(t, '.');
+	return t.type == Token_ENUMERATION || t.type == Token_BOOL;
 }
 
 bool TokenFunc::isBinary(const Token& t) {
-	return ! isOperator(t) && startsWith(t, '"');
+	return t.type == Token_BINARY;
 }
 
 bool TokenFunc::isKeyword(const Token& t) {
-	// bool is a subtype of enumeration, no need to test for that
-	return !isOperator(t) && !isIdentifier(t) && !isString(t) && !isEnumeration(t) && !isInt(t) && !isFloat(t) && !isBinary(t);
+	return t.type == Token_KEYWORD;
 }
 
 bool TokenFunc::isInt(const Token& t) {
-	if (isOperator(t) || isString(t) || isEnumeration(t)) {
-		return false;
-	}
-	const std::string str = asString(t);
-	const char* start = str.c_str();
-	char* end;
-	/*long result =*/ strtol(start,&end,10);
-	return ((end - start) == (ptrdiff_t)str.length());
+	return t.type == Token_INT;
 }
 
 bool TokenFunc::isBool(const Token& t) {
-	if (!isEnumeration(t)) return false;
-	const std::string str = asString(t);
-	return str == "T" || str == "F";
+	return t.type == Token_BOOL;
 }
 
 bool TokenFunc::isFloat(const Token& t) {
-	if (isOperator(t) || isString(t) || isEnumeration(t)) {
-		return false;
-	}
-	const std::string str = asString(t);
-	const char* start = str.c_str();
-	char* end;
-#ifdef _MSC_VER
-	/*double result =*/ _strtod_l(start,&end,locale);
-#else
-	double result = strtod_l(start,&end,locale);
-#endif
-	return ((end - start) == (ptrdiff_t)str.length());
+	return t.type == Token_FLOAT || t.type == Token_INT;
 }
 
 int TokenFunc::asInt(const Token& t) {
-	const std::string str = asString(t);
-	// In case of an ENTITY_INSTANCE_NAME skip the leading #
-	const char* start = str.c_str() + (isIdentifier(t) ? 1 : 0);
-	char* end;
-	long result = strtol(start,&end,10);
-	if ( start == end ) throw IfcException("Token is not an integer or identifier");
-	return (int) result;
+	if (t.type != Token_INT)
+		throw IfcException("Token is not an integer");
+	return t.value_int;
+}
+
+int TokenFunc::asIdentifier(const Token& t) {
+	if (t.type != Token_IDENTIFIER)
+		throw IfcException("Token is not an identifier");
+	return t.value_int;
 }
 
 bool TokenFunc::asBool(const Token& t) {
-	const std::string str = asString(t);
-	return str == "T";
+	if (t.type != Token_BOOL)
+		throw IfcException("Token is not a boolean");
+	return t.value_bool;
 }
 
 double TokenFunc::asFloat(const Token& t) {
-	const std::string str = asString(t);
-	const char* start = str.c_str();
-	char* end;
-#ifdef _MSC_VER
-	double result = _strtod_l(start,&end,locale);
-#else
-	double result = strtod_l(start,&end,locale);
-#endif
-	if ( start == end ) throw IfcException("Token is not a real");
-	return result;
+	if (t.type != Token_FLOAT)
+		throw IfcException("Token is not a float");
+	return t.value_double;
+}
+
+const std::string &TokenFunc::asStringRef(const Token& t) {
+	std::string &str = t.lexer->GetTempString();
+	t.lexer->TokenString(t.startPos, str);
+	if (isString(t) || isEnumeration(t) || isBinary(t)) {
+		//remove start+end characters in-place
+		str.pop_back();
+		str.erase(str.begin());
+	}
+	return str;
 }
 
 std::string TokenFunc::asString(const Token& t) {
-	if ( isOperator(t,'$') ) return "";
-	else if ( isOperator(t) ) throw IfcException("Token is not a string");
-	std::string str = t.first->TokenString(t.second);
-	return isString(t) || isEnumeration(t) || isBinary(t) ? str.substr(1,str.size()-2) : str;
+	return asStringRef(t);
 }
 
 boost::dynamic_bitset<> TokenFunc::asBinary(const Token& t) {
-	const std::string str = asString(t);
+	const std::string &str = asStringRef(t);
 	if (str.size() < 1) {
 		throw IfcException("Token is not a valid binary sequence");
 	}
@@ -526,8 +584,9 @@ boost::dynamic_bitset<> TokenFunc::asBinary(const Token& t) {
 }
 
 std::string TokenFunc::toString(const Token& t) {
-	if ( isOperator(t) ) return std::string ( (char*) &t.second	, 1 );
-	else return t.first->TokenString(t.second);
+	std::string result;
+	t.lexer->TokenString(t.startPos, result);
+	return result;
 }
 
 
@@ -536,11 +595,11 @@ TokenArgument::TokenArgument(const Token& t) {
 }
 
 EntityArgument::EntityArgument(const Token& t) {
-	IfcParse::IfcFile* file = t.first->file;
+	IfcParse::IfcFile* file = t.lexer->file;
 	if (file->create_latebound_entities()) {
-		entity = new IfcLateBoundEntity(new Entity(0, file, t.second));
+		entity = new IfcLateBoundEntity(new Entity(0, file, t.startPos));
 	} else {
-		entity = IfcSchema::SchemaEntity(new Entity(0, file, t.second));
+		entity = IfcSchema::SchemaEntity(new Entity(0, file, t.startPos));
 	}
 }
 
@@ -551,7 +610,7 @@ EntityArgument::EntityArgument(const Token& t) {
 void ArgumentList::read(IfcSpfLexer* t, std::vector<unsigned int>& ids) {
 	//IfcParse::IfcFile* file = t->file;
 	Token next = t->Next();
-	while( next.second || next.first ) {
+	while( next.startPos || next.lexer ) {
 		if ( TokenFunc::isOperator(next,',') ) {
 			// do nothing
 		} else if ( TokenFunc::isOperator(next,')') ) {
@@ -562,7 +621,7 @@ void ArgumentList::read(IfcSpfLexer* t, std::vector<unsigned int>& ids) {
 			push(alist);
 		} else {
 			if ( TokenFunc::isIdentifier(next) ) {
-				ids.push_back(TokenFunc::asInt(next));
+				ids.push_back(TokenFunc::asIdentifier(next));
 			} if ( TokenFunc::isKeyword(next) ) {
 				t->Next();
 				try {
@@ -631,12 +690,6 @@ std::vector< std::vector<T> > read_aggregate_of_aggregate_as_vector2(const std::
 //
 // Functions for casting the ArgumentList to other types
 //
-ArgumentList::operator int() const { throw IfcException("Argument is not an integer"); }
-ArgumentList::operator bool() const { throw IfcException("Argument is not a boolean"); }
-ArgumentList::operator double() const { throw IfcException("Argument is not a number"); }
-ArgumentList::operator std::string() const { throw IfcException("Argument is not a string"); }
-ArgumentList::operator boost::dynamic_bitset<>() const { throw IfcException("Argument is not a binary"); }
-
 ArgumentList::operator std::vector<double>() const {
 	return read_aggregate_as_vector<double>(list);
 }
@@ -653,8 +706,6 @@ ArgumentList::operator std::vector<boost::dynamic_bitset<> >() const {
 	return read_aggregate_as_vector<boost::dynamic_bitset<> >(list);
 }
 
-ArgumentList::operator IfcUtil::IfcBaseClass*() const { throw IfcException("Argument is not an entity instance"); }
-
 ArgumentList::operator IfcEntityList::ptr() const {
 	IfcEntityList::ptr l ( new IfcEntityList() );
 	std::vector<Argument*>::const_iterator it;
@@ -699,7 +750,7 @@ Argument* ArgumentList::operator [] (unsigned int i) const {
 
 void ArgumentList::set(unsigned int i, Argument* argument) {
 	while (size() < i) {
-		push(new TokenArgument(Token(static_cast<IfcSpfLexer*>(0), '$')));
+		push(new NullArgument());
 	}
 	if (i < size()) {
 		delete list[i];
@@ -762,15 +813,7 @@ TokenArgument::operator bool() const { return TokenFunc::asBool(token); }
 TokenArgument::operator double() const { return TokenFunc::asFloat(token); }
 TokenArgument::operator std::string() const { return TokenFunc::asString(token); }
 TokenArgument::operator boost::dynamic_bitset<>() const { return TokenFunc::asBinary(token); }
-TokenArgument::operator std::vector<double>() const { throw IfcException("Argument is not a list of floats"); }
-TokenArgument::operator std::vector<int>() const { throw IfcException("Argument is not a list of ints"); }
-TokenArgument::operator std::vector<std::string>() const { throw IfcException("Argument is not a list of strings"); }
-TokenArgument::operator std::vector<boost::dynamic_bitset<> >() const { throw IfcException("Argument is not a list of binaries"); }
-TokenArgument::operator IfcUtil::IfcBaseClass*() const { return token.first->file->entityById(TokenFunc::asInt(token)); }
-TokenArgument::operator IfcEntityList::ptr() const { throw IfcException("Argument is not a list of entity instances"); }
-TokenArgument::operator std::vector< std::vector<int> >() const { throw IfcException("Argument is not a list of list of ints"); }
-TokenArgument::operator std::vector< std::vector<double> >() const { throw IfcException("Argument is not a list of list of floats"); }
-TokenArgument::operator IfcEntityListList::ptr() const { throw IfcException("Argument is not a list of list of entity instances"); }
+TokenArgument::operator IfcUtil::IfcBaseClass*() const { return token.lexer->file->entityById(TokenFunc::asIdentifier(token)); }
 unsigned int TokenArgument::size() const { return 1; }
 Argument* TokenArgument::operator [] (unsigned int /*i*/) const { throw IfcException("Argument is not a list of attributes"); }
 std::string TokenArgument::toString(bool upper) const { 
@@ -789,20 +832,7 @@ IfcUtil::ArgumentType EntityArgument::type() const {
 //
 // Functions for casting the EntityArgument to other types
 //
-EntityArgument::operator int() const { throw IfcException("Argument is not an integer"); }
-EntityArgument::operator bool() const { throw IfcException("Argument is not a boolean"); }
-EntityArgument::operator double() const { throw IfcException("Argument is not a number"); }
-EntityArgument::operator boost::dynamic_bitset<>() const { throw IfcException("Argument is not a binary"); }
-EntityArgument::operator std::string() const { throw IfcException("Argument is not a string"); }
-EntityArgument::operator std::vector<double>() const { throw IfcException("Argument is not a list of floats"); }
-EntityArgument::operator std::vector<int>() const { throw IfcException("Argument is not a list of ints"); }
-EntityArgument::operator std::vector<std::string>() const { throw IfcException("Argument is not a list of strings"); }
-EntityArgument::operator std::vector<boost::dynamic_bitset<> >() const { throw IfcException("Argument is not a list of binaries"); }
 EntityArgument::operator IfcUtil::IfcBaseClass*() const { return entity; }
-EntityArgument::operator IfcEntityList::ptr() const { throw IfcException("Argument is not a list of entity instances"); }
-EntityArgument::operator std::vector< std::vector<int> >() const { throw IfcException("Argument is not a list of list of ints"); }
-EntityArgument::operator std::vector< std::vector<double> >() const { throw IfcException("Argument is not a list of list of floats"); }
-EntityArgument::operator IfcEntityListList::ptr() const { throw IfcException("Argument is not a list of list of entity instances"); }
 unsigned int EntityArgument::size() const { return 1; }
 Argument* EntityArgument::operator [] (unsigned int /*i*/) const { throw IfcException("Argument is not a list of arguments"); }
 std::string EntityArgument::toString(bool upper) const { 
@@ -819,8 +849,8 @@ Entity::Entity(unsigned int i, IfcFile* f) : args(0), _id(i) {
 	file = f;
 	Token datatype = f->tokens->Next();
 	if ( ! TokenFunc::isKeyword(datatype)) throw IfcException("Unexpected token while parsing entity");
-	_type = IfcSchema::Type::FromString(TokenFunc::asString(datatype));
-	offset = datatype.second;
+	_type = IfcSchema::Type::FromString(TokenFunc::asStringRef(datatype));
+	offset = datatype.startPos;
 }
 
 //
@@ -861,7 +891,7 @@ void Entity::Load(std::vector<unsigned int>& ids, bool seek) const {
 		file->tokens->stream->Seek(offset);
 		Token datatype = file->tokens->Next();
 		if ( ! TokenFunc::isKeyword(datatype)) throw IfcException("Unexpected token while parsing entity");
-		_type = IfcSchema::Type::FromString(TokenFunc::asString(datatype));
+		_type = IfcSchema::Type::FromString(TokenFunc::asStringRef(datatype));
 	}
 	Token open = file->tokens->Next();
 	args = new ArgumentList();
@@ -975,8 +1005,8 @@ bool IfcFile::Init(IfcParse::IfcSpfStream* s) {
 		Logger::Message(Logger::LOG_ERROR, std::string("File schema encountered different from expected '") + IfcSchema::Identifier + "'");
 	}
 
-	Token token = TokenPtr();
-	Token previous = TokenPtr();
+	Token token = NoneTokenPtr();
+	Token previous = NoneTokenPtr();
 
 	unsigned int currentId = 0;
 	lastId = 0;
@@ -1045,13 +1075,13 @@ bool IfcFile::Init(IfcParse::IfcSpfStream* s) {
 			currentId = 0;
 		} else {
 			try { token = tokens->Next(); }
-			catch (... ) { token = TokenPtr(); }
+			catch (... ) { token = NoneTokenPtr(); }
 		}
 
-		if ( ! (token.second || token.first) ) break;
+		if ( ! (token.startPos || token.lexer) ) break;
 		
-		if ( (previous.second || previous.first) && TokenFunc::isIdentifier(previous) ) {
-			int id = TokenFunc::asInt(previous);
+		if ( (previous.startPos || previous.lexer) && TokenFunc::isIdentifier(previous) ) {
+			int id = TokenFunc::asIdentifier(previous);
 			if ( TokenFunc::isOperator(token,'=') ) {
 				currentId = id;
 			} else if (entity) {

src/ifcparse/IfcParse.h

@@ -60,7 +60,34 @@ namespace IfcParse {
 	class IfcFile;
 	class IfcSpfLexer;
 
-	typedef std::pair<IfcSpfLexer*, unsigned> Token;
+	enum TokenType {
+		Token_NONE,
+		Token_STRING,
+		Token_IDENTIFIER,
+		Token_OPERATOR,
+		Token_ENUMERATION,
+		Token_KEYWORD,
+		Token_INT,
+		Token_BOOL,
+		Token_FLOAT,
+		Token_BINARY
+	};
+
+	struct Token {
+		IfcSpfLexer* lexer; //TODO: remove it from here
+		unsigned startPos, endPos;
+		TokenType type;
+		union {
+			bool value_bool;      //types: BOOL
+			char value_char;      //types: OPERATOR
+			int value_int;        //types: INT, IDENTIFIER
+			double value_double;  //types: FLOAT
+		};
+
+		Token() : lexer(0), startPos(0), endPos(0), type(Token_NONE) {}
+		Token(IfcSpfLexer* _lexer, unsigned _startPos, unsigned _endPos, TokenType _type)
+			: lexer(_lexer), startPos(_startPos), endPos(_endPos), type(_type) {}
+	};
 
 	/// Provides functions to convert Tokens to binary data
 	/// Tokens are merely offsets to where they can be read in the file
@@ -75,7 +102,9 @@ namespace IfcParse {
 		/// Returns whether the token can be interpreted as an identifier
 		static bool isIdentifier(const Token& t);
 		/// Returns whether the token can be interpreted as a syntactical operator
-		static bool isOperator(const Token& t, char op = 0);
+		static bool isOperator(const Token& t);
+		/// Returns whether the token is a given operator
+		static bool isOperator(const Token& t, char op);
 		/// Returns whether the token can be interpreted as an enumerated value
 		static bool isEnumeration(const Token& t);
 		/// Returns whether the token can be interpreted as a datatype name
@@ -90,12 +119,16 @@ namespace IfcParse {
 		static bool isBinary(const Token& t);
 		/// Returns the token interpreted as an integer
 		static int asInt(const Token& t);
+		/// Returns the token interpreted as an identifier
+		static int asIdentifier(const Token& t);
 		/// Returns the token interpreted as an boolean (.T. or .F.)
 		static bool asBool(const Token& t);
 		/// Returns the token as a floating point number
 		static double asFloat(const Token& t);
 		/// Returns the token as a string (without the dot or apostrophe)
 		static std::string asString(const Token& t);
+		/// Returns the token as a string in internal buffer (for optimization purposes)
+		static const std::string &asStringRef(const Token& t);
 		/// Returns the token as a string (without the dot or apostrophe)
 		static boost::dynamic_bitset<> asBinary(const Token& t);
 		/// Returns a string representation of the token (including the dot or apostrophe)
@@ -106,23 +139,26 @@ namespace IfcParse {
 	// Functions for creating Tokens from an arbitary file offset
 	// The first 4 bits are reserved for Tokens of type ()=,;$*
 	//
-	Token TokenPtr(IfcSpfLexer* tokens, unsigned int offset);
-	Token TokenPtr(char c);	
-	Token TokenPtr();
+	Token OperatorTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
+	Token GeneralTokenPtr(IfcSpfLexer* tokens, unsigned start, unsigned end);
+	Token NoneTokenPtr();
 
 	/// A stream of tokens to be read from a IfcSpfStream.
 	class IfcParse_EXPORT IfcSpfLexer {
 	private:
 		IfcCharacterDecoder* decoder;
+		//storage for temporary string without allocation
+		mutable std::string _tempString;
 		unsigned int skipWhitespace();
 		unsigned int skipComment();
 	public:
+		std::string &GetTempString() const { return _tempString; }
 		IfcSpfStream* stream;
 		IfcFile* file;
 		IfcSpfLexer(IfcSpfStream* s, IfcFile* f);
 		Token Next();
 		~IfcSpfLexer();
-		std::string TokenString(unsigned int offset);
+		void TokenString(unsigned int offset, std::string &result);
 	};
 
 	/// Argument of type list, e.g.
@@ -139,13 +175,6 @@ namespace IfcParse {
 
 		IfcUtil::ArgumentType type() const;
 		
-		operator int() const;
-		operator bool() const;
-		operator double() const;
-		operator std::string() const;
-		operator boost::dynamic_bitset<>() const;
-		operator IfcUtil::IfcBaseClass*() const;
-
 		operator std::vector<int>() const;
 		operator std::vector<double>() const;
 		operator std::vector<std::string>() const;
@@ -165,6 +194,19 @@ namespace IfcParse {
 		std::string toString(bool upper=false) const;
 	};
 
+
+	/// Argument being null, e.g. '$'
+	///              == ===
+	class IfcParse_EXPORT NullArgument : public Argument {
+	public:
+		NullArgument() {}
+		IfcUtil::ArgumentType type() const { return IfcUtil::Argument_NULL; }
+		bool isNull() const { return true; }
+		unsigned int size() const { return 1; }
+		Argument* operator [] (unsigned int /*i*/) const { throw IfcException("Argument is not a list of attributes"); }
+		std::string toString(bool /*upper=false*/) const { return "$"; }
+	};
+
 	/// Argument of type scalar or string, e.g.
 	/// #1=IfcVector(#2,1.0);
 	///              == ===
@@ -184,16 +226,6 @@ namespace IfcParse {
 		operator boost::dynamic_bitset<>() const;
 		operator IfcUtil::IfcBaseClass*() const;
 
-		operator std::vector<int>() const;
-		operator std::vector<double>() const;
-		operator std::vector<std::string>() const;
-		operator std::vector<boost::dynamic_bitset<> >() const;
-		operator IfcEntityList::ptr() const;
-		
-		operator std::vector< std::vector<int> >() const;
-		operator std::vector< std::vector<double> >() const;
-		operator IfcEntityListList::ptr() const;
-
 		bool isNull() const;
 		unsigned int size() const;
 
@@ -213,23 +245,8 @@ namespace IfcParse {
 
 		IfcUtil::ArgumentType type() const;
 
-		operator int() const;
-		operator bool() const;
-		operator double() const;
-		operator std::string() const;
-		operator boost::dynamic_bitset<>() const;
 		operator IfcUtil::IfcBaseClass*() const;
 
-		operator std::vector<int>() const;
-		operator std::vector<double>() const;
-		operator std::vector<std::string>() const;
-		operator std::vector<boost::dynamic_bitset<> >() const;
-		operator IfcEntityList::ptr() const;
-
-		operator std::vector< std::vector<int> >() const;
-		operator std::vector< std::vector<double> >() const;
-		operator IfcEntityListList::ptr() const;
-
 		bool isNull() const;
 		unsigned int size() const;
 

src/ifcparse/IfcUtil.cpp

@@ -76,6 +76,24 @@ unsigned int IfcUtil::IfcBaseType::getArgumentCount() const { return 1; }
 Argument* IfcUtil::IfcBaseType::getArgument(unsigned int i) const { return entity->getArgument(i); }
 const char* IfcUtil::IfcBaseType::getArgumentName(unsigned int i) const { if (i == 0) { return "wrappedValue"; } else { throw IfcParse::IfcAttributeOutOfRangeException("Argument index out of range"); } }
 
+
+//Note: some of these methods are overloaded in derived classes
+Argument::operator int() const { throw IfcParse::IfcException("Argument is not an integer"); }
+Argument::operator bool() const { throw IfcParse::IfcException("Argument is not a boolean"); }
+Argument::operator double() const { throw IfcParse::IfcException("Argument is not a number"); }
+Argument::operator std::string() const { throw IfcParse::IfcException("Argument is not a string"); }
+Argument::operator boost::dynamic_bitset<>() const { throw IfcParse::IfcException("Argument is not a binary"); }
+Argument::operator IfcUtil::IfcBaseClass*() const { throw IfcParse::IfcException("Argument is not an entity instance"); }
+Argument::operator std::vector<double>() const { throw IfcParse::IfcException("Argument is not a list of floats"); }
+Argument::operator std::vector<int>() const { throw IfcParse::IfcException("Argument is not a list of ints"); }
+Argument::operator std::vector<std::string>() const { throw IfcParse::IfcException("Argument is not a list of strings"); }
+Argument::operator std::vector<boost::dynamic_bitset<> >() const { throw IfcParse::IfcException("Argument is not a list of binaries"); }
+Argument::operator IfcEntityList::ptr() const { throw IfcParse::IfcException("Argument is not a list of entity instances"); }
+Argument::operator std::vector< std::vector<int> >() const { throw IfcParse::IfcException("Argument is not a list of list of ints"); }
+Argument::operator std::vector< std::vector<double> >() const { throw IfcParse::IfcException("Argument is not a list of list of floats"); }
+Argument::operator IfcEntityListList::ptr() const { throw IfcParse::IfcException("Argument is not a list of list of entity instances"); }
+
+
 static const char* const argument_type_string[] = {
 	"NULL",
 	"DERIVED",

src/ifcparse/IfcUtil.h

@@ -283,22 +283,22 @@ namespace IfcParse {
 
 class IfcParse_EXPORT Argument {
 public:
-	virtual operator int() const = 0;
-	virtual operator bool() const = 0;
-	virtual operator double() const = 0;
-	virtual operator std::string() const = 0;
-	virtual operator boost::dynamic_bitset<>() const = 0;
-	virtual operator IfcUtil::IfcBaseClass*() const = 0;
+	virtual operator int() const;
+	virtual operator bool() const;
+	virtual operator double() const;
+	virtual operator std::string() const;
+	virtual operator boost::dynamic_bitset<>() const;
+	virtual operator IfcUtil::IfcBaseClass*() const;
 
-	virtual operator std::vector<int>() const = 0;
-	virtual operator std::vector<double>() const = 0;
-	virtual operator std::vector<std::string>() const = 0;
-	virtual operator std::vector<boost::dynamic_bitset<> >() const = 0;
-	virtual operator IfcEntityList::ptr() const = 0;
+	virtual operator std::vector<int>() const;
+	virtual operator std::vector<double>() const;
+	virtual operator std::vector<std::string>() const;
+	virtual operator std::vector<boost::dynamic_bitset<> >() const;
+	virtual operator IfcEntityList::ptr() const;
 
-	virtual operator std::vector< std::vector<int> >() const = 0;
-	virtual operator std::vector< std::vector<double> >() const = 0;
-	virtual operator IfcEntityListList::ptr() const = 0;
+	virtual operator std::vector< std::vector<int> >() const;
+	virtual operator std::vector< std::vector<double> >() const;
+	virtual operator IfcEntityListList::ptr() const;
 
 	virtual bool isNull() const = 0;
 	virtual unsigned int size() const = 0;