Make real number parsing and serializing locale-independent, as suggested by Ian Clevy.

This commit is contained in:
Thomas Krijnen
2015-02-03 13:53:36 +00:00
parent 5f677baf63
commit 2da6f5428d
2 changed files with 52 additions and 4 deletions
+48 -2
View File
@@ -38,6 +38,27 @@
using namespace IfcParse; using namespace IfcParse;
// A static locale for the real number parser. strtod() is locale-dependent, causing issues
// in locales that have ',' as a decimal separator. Therefore the non standard _strtod_l() /
// strtod_l() is used and a reference to the "C" locale is obtained here. The alternative is
// to use std::istringstream::imbue(std::locale::classic()), but there are subtleties in
// parsing in MSVC2010 and it appears to be much slower.
#ifdef _MSC_VER
static _locale_t locale = (_locale_t) 0;
void init_locale() {
if (locale == (_locale_t) 0) {
locale = _create_locale(LC_NUMERIC, "C");
}
}
#else
static locale_t locale = (locale_t) 0;
void init_locale() {
if (locale == (_locale_t) 0) {
locale = newlocale(LC_NUMERIC_MASK, "C", (locale_t) 0);
}
}
#endif
// //
// Opens the file, gets the filesize and reads a chunk in memory // Opens the file, gets the filesize and reads a chunk in memory
// //
@@ -333,22 +354,28 @@ Token IfcParse::TokenPtr() { return Token((IfcSpfLexer*)0,0); }
bool TokenFunc::startsWith(const Token& t, char c) { bool TokenFunc::startsWith(const Token& t, char c) {
return t.first->stream->Read(t.second) == c; return t.first->stream->Read(t.second) == c;
} }
bool TokenFunc::isOperator(const Token& t, char op) { bool TokenFunc::isOperator(const Token& t, char op) {
return (!t.first) && (!op || op == t.second); return (!t.first) && (!op || op == t.second);
} }
bool TokenFunc::isIdentifier(const Token& t) { bool TokenFunc::isIdentifier(const Token& t) {
return ! isOperator(t) && startsWith(t, '#'); return ! isOperator(t) && startsWith(t, '#');
} }
bool TokenFunc::isString(const Token& t) { bool TokenFunc::isString(const Token& t) {
return ! isOperator(t) && startsWith(t, '\''); return ! isOperator(t) && startsWith(t, '\'');
} }
bool TokenFunc::isEnumeration(const Token& t) { bool TokenFunc::isEnumeration(const Token& t) {
return ! isOperator(t) && startsWith(t, '.'); return ! isOperator(t) && startsWith(t, '.');
} }
bool TokenFunc::isKeyword(const Token& t) { bool TokenFunc::isKeyword(const Token& t) {
// bool is a subtype of enumeration, no need to test for that // bool is a subtype of enumeration, no need to test for that
return !isOperator(t) && !isIdentifier(t) && !isString(t) && !isEnumeration(t) && !isInt(t) && !isFloat(t); return !isOperator(t) && !isIdentifier(t) && !isString(t) && !isEnumeration(t) && !isInt(t) && !isFloat(t);
} }
bool TokenFunc::isInt(const Token& t) { bool TokenFunc::isInt(const Token& t) {
if (isOperator(t)) return false; if (isOperator(t)) return false;
const std::string str = asString(t); const std::string str = asString(t);
@@ -357,19 +384,26 @@ bool TokenFunc::isInt(const Token& t) {
long result = strtol(start,&end,10); long result = strtol(start,&end,10);
return ((end - start) == str.length()); return ((end - start) == str.length());
} }
bool TokenFunc::isBool(const Token& t) { bool TokenFunc::isBool(const Token& t) {
if (!isEnumeration(t)) return false; if (!isEnumeration(t)) return false;
const std::string str = asString(t); const std::string str = asString(t);
return str == "T" || str == "F"; return str == "T" || str == "F";
} }
bool TokenFunc::isFloat(const Token& t) { bool TokenFunc::isFloat(const Token& t) {
if (isOperator(t)) return false; if (isOperator(t)) return false;
const std::string str = asString(t); const std::string str = asString(t);
const char* start = str.c_str(); const char* start = str.c_str();
char* end; char* end;
double result = strtod(start,&end); #ifdef _MSC_VER
double result = _strtod_l(start,&end,locale);
#else
double result = strtod_l(start,&end,locale);
#endif
return ((end - start) == str.length()); return ((end - start) == str.length());
} }
int TokenFunc::asInt(const Token& t) { int TokenFunc::asInt(const Token& t) {
const std::string str = asString(t); const std::string str = asString(t);
// In case of an ENTITY_INSTANCE_NAME skip the leading # // In case of an ENTITY_INSTANCE_NAME skip the leading #
@@ -379,24 +413,32 @@ int TokenFunc::asInt(const Token& t) {
if ( start == end ) throw IfcException("Token is not an integer or identifier"); if ( start == end ) throw IfcException("Token is not an integer or identifier");
return (int) result; return (int) result;
} }
bool TokenFunc::asBool(const Token& t) { bool TokenFunc::asBool(const Token& t) {
const std::string str = asString(t); const std::string str = asString(t);
return str == "T"; return str == "T";
} }
double TokenFunc::asFloat(const Token& t) { double TokenFunc::asFloat(const Token& t) {
const std::string str = asString(t); const std::string str = asString(t);
const char* start = str.c_str(); const char* start = str.c_str();
char* end; char* end;
double result = strtod(start,&end); #ifdef _MSC_VER
double result = _strtod_l(start,&end,locale);
#else
double result = strtod_l(start,&end,locale);
#endif
if ( start == end ) throw IfcException("Token is not a real"); if ( start == end ) throw IfcException("Token is not a real");
return result; return result;
} }
std::string TokenFunc::asString(const Token& t) { std::string TokenFunc::asString(const Token& t) {
if ( isOperator(t,'$') ) return ""; if ( isOperator(t,'$') ) return "";
else if ( isOperator(t) ) throw IfcException("Token is not a string"); else if ( isOperator(t) ) throw IfcException("Token is not a string");
std::string str = t.first->TokenString(t.second); std::string str = t.first->TokenString(t.second);
return isString(t) || isEnumeration(t) ? str.substr(1,str.size()-2) : str; return isString(t) || isEnumeration(t) ? str.substr(1,str.size()-2) : str;
} }
std::string TokenFunc::toString(const Token& t) { std::string TokenFunc::toString(const Token& t) {
if ( isOperator(t) ) return std::string ( (char*) &t.second , 1 ); if ( isOperator(t) ) return std::string ( (char*) &t.second , 1 );
else return t.first->TokenString(t.second); else return t.first->TokenString(t.second);
@@ -769,6 +811,10 @@ bool IfcFile::Init(void* data, int len) {
return IfcFile::Init(new IfcSpfStream(data,len)); return IfcFile::Init(new IfcSpfStream(data,len));
} }
bool IfcFile::Init(IfcParse::IfcSpfStream* s) { bool IfcFile::Init(IfcParse::IfcSpfStream* s) {
// Initialize a "C" locale for locale-independent
// number parsing. See comment above on line 41.
init_locale();
stream = s; stream = s;
if (!stream->valid) { if (!stream->valid) {
return false; return false;
+4 -2
View File
@@ -17,7 +17,8 @@
* * * *
********************************************************************************/ ********************************************************************************/
#include <iomanip> #include <iomanip>
#include <locale>
#include "../ifcparse/IfcParse.h" #include "../ifcparse/IfcParse.h"
#include "../ifcparse/IfcWrite.h" #include "../ifcparse/IfcWrite.h"
@@ -204,7 +205,8 @@ private:
// REAL = [ SIGN ] DIGIT { DIGIT } "." { DIGIT } [ "E" [ SIGN ] DIGIT { DIGIT } ] . // REAL = [ SIGN ] DIGIT { DIGIT } "." { DIGIT } [ "E" [ SIGN ] DIGIT { DIGIT } ] .
std::string format_double(const double& d) { std::string format_double(const double& d) {
std::ostringstream oss; std::ostringstream oss;
oss << std::setprecision(16) << d; oss.imbue(std::locale::classic());
oss << std::setprecision(15) << d;
const std::string str = oss.str(); const std::string str = oss.str();
oss.str(""); oss.str("");
std::string::size_type e = str.find('e'); std::string::size_type e = str.find('e');