Glassvision/IfcOpenShell
1
0
Fork 0
mirror of https://github.com/IfcOpenShell/IfcOpenShell.git synced 2026-08-14 11:24:19 +00:00
Code Issues Packages Projects Releases Wiki Activity

Ensure correct parsing and decoding of strings [#3395016]

Browse Source
This commit is contained in:
Thomas Krijnen
2011-09-25 09:53:22 +00:00
parent 767f7b4090
commit f7abadc081
13 changed files with 438 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/ifcparse/IfcCharacterDecoder.cpp
+234
View File
@@ -0,0 +1,234 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
/********************************************************************************
* *
* Implementation of character decoding as described in ISO 10303-21 table 2 and *
* table 4 *
* *
********************************************************************************/
#include <string>
#include <sstream>
#include <iomanip>
#include "../ifcparse/IfcCharacterDecoder.h"
#include "../ifcparse/IfcException.h"
#include "../ifcparse/IfcFile.h"
#define FIRST_SOLIDUS (1 << 1)
#define PAGE (1 << 2)
#define ALPHABET (1 << 3)
#define SECOND_SOLIDUS (1 << 4)
#define ALPHABET_DEFINITION (1 << 5)
#define APOSTROPHE (1 << 6)
#define ARBITRARY (1 << 7)
#define EXTENDED2 (1 << 8)
#define EXTENDED4 (1 << 9)
#define HEX(N) (1 << (9+N))
#define THIRD_SOLIDUS (1 << 18)
#define ENDEXTENDED_X (1 << 19)
#define ENDEXTENDED_0 (1 << 20)
#define FOURTH_SOLIDUS (1 << 21)
#define IGNORED_DIRECTIVE (1 << 22)
// FIXME: These probably need to be less forgiving in terms of wrongly defined sequences
#define EXPECTS_ALPHABET(S) (S & FIRST_SOLIDUS)
#define EXPECTS_PAGE(S) (S & FIRST_SOLIDUS)
#define EXPECTS_ARBITRARY(S) (S & FIRST_SOLIDUS)
#define EXPECTS_N_OR_F(S) (S & FIRST_SOLIDUS && !(S & ARBITRARY) )
#define EXPECTS_ARBITRARY2(S) (S & ARBITRARY && !(S & SECOND_SOLIDUS))
#define EXPECTS_ALPHABET_DEFINITION(S) (S & FIRST_SOLIDUS && S & ALPHABET)
#define EXPECTS_SOLIDUS(S) (S & ALPHABET_DEFINITION || S & PAGE || S & ARBITRARY || S & EXTENDED2 || S & EXTENDED4 || S & ENDEXTENDED_0 || S & IGNORED_DIRECTIVE || (S & EXTENDED4 && S & HEX(8)) || (S & EXTENDED2 && S & HEX(4)))
#define EXPECTS_CHARACTER(S) (S & PAGE && S & SECOND_SOLIDUS)
#define EXPECTS_HEX(S) (S & HEX(1) || S & HEX(3) || S & HEX(5) || S & HEX(6) || S & HEX(7) || (S & ARBITRARY && S & SECOND_SOLIDUS) || (S & EXTENDED2 && S & HEX(2)) || (S & EXTENDED4 && S & HEX(4)) )
#define EXPECTS_ENDEXTENDED_X(S) (S & THIRD_SOLIDUS)
#define EXPECTS_ENDEXTENDED_0(S) (S & ENDEXTENDED_X)
#define IS_VALID_ALPHABET_DEFINITION(C) (C >= 0x40 && C <= 0x4A)
#define IS_HEXADECIMAL(C) ((C >= 0x30 && C <= 0x39 ) || (C >= 0x41 && C <= 0x46 ))
#define HEX_TO_INT(C) ((C >= 0x30 && C <= 0x39 ) ? C - 0x30 : (C+10) - 0x41)
#define CLEAR_HEX(C) (C &= ~(HEX(1)&HEX(2)&HEX(3)&HEX(4)&HEX(5)&HEX(6)&HEX(7)&HEX(8)))
using namespace IfcParse;
void IfcCharacterDecoder::addChar(std::stringstream& s,const UChar32& ch) {
if ( destination ) {
char* extraction_buffer = new char[4];
UnicodeString(ch).extract(extraction_buffer,4,destination,status);
s << extraction_buffer;
delete extraction_buffer;
} else {
std::stringstream s2;
s2 << "\\u" << std::hex << std::setw(4) << std::setfill('0') << (int) ch;
s << s2.str();
}
}
IfcCharacterDecoder::IfcCharacterDecoder(IfcParse::File* f) {
file = f;
if ( ! destination && mode == UTF8 ) {
destination = ucnv_open("utf-8", &status);
} else if ( ! destination && mode == LATIN ) {
destination = ucnv_open("iso-8859-1", &status);
}
}
IfcCharacterDecoder::~IfcCharacterDecoder() {
if ( destination ) ucnv_close(destination);
if ( converter ) ucnv_close(converter);
}
IfcCharacterDecoder::operator std::string() {
unsigned int parse_state = 0;
std::stringstream s;
s.put('\'');
char current_char;
int codepage = 1;
unsigned int hex = 0;
unsigned int hex_count = 0;
while ( current_char = file->Peek() ) {
if ( EXPECTS_CHARACTER(parse_state) ) {
if ( previous_codepage != codepage ) {
if ( converter ) ucnv_close(converter);
char encoder[11] = {'i','s','o','-','8','8','5','9','-',codepage + 0x30};
converter = ucnv_open(encoder, &status);
}
const char characters[2] = { current_char + 0x80 };
const char* char_array = &characters[0];
UChar32 ch = ucnv_getNextUChar(converter,&char_array,char_array+1,&status);
addChar(s,ch);
parse_state = 0;
} else if ( current_char == '\'' && ! parse_state ) {
parse_state = APOSTROPHE;
} else if ( current_char == '\\' && ! parse_state ) {
parse_state = FIRST_SOLIDUS;
} else if ( current_char == '\\' && EXPECTS_SOLIDUS(parse_state) ) {
if ( parse_state & ALPHABET_DEFINITION ||
parse_state & IGNORED_DIRECTIVE ||
parse_state & ENDEXTENDED_0 ) parse_state = hex = hex_count = 0;
else if ( parse_state & HEX(3) ) parse_state += THIRD_SOLIDUS;
else parse_state += SECOND_SOLIDUS;
} else if ( current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state) ) {
parse_state += ENDEXTENDED_X;
} else if ( current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state) ) {
parse_state += ENDEXTENDED_0;
} else if ( current_char == 'X' && EXPECTS_ARBITRARY(parse_state) ) {
parse_state += ARBITRARY;
} else if ( current_char == '2' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2;
} else if ( current_char == '4' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2 + EXTENDED4;
} else if ( current_char == 'P' && EXPECTS_ALPHABET(parse_state) ) {
parse_state += ALPHABET;
} else if ( (current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state) ) {
parse_state += IGNORED_DIRECTIVE;
} else if ( IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state) ) {
codepage = current_char - 0x40;
parse_state += ALPHABET_DEFINITION;
} else if ( current_char == 'S' && EXPECTS_PAGE(parse_state) ) {
parse_state += PAGE;
} else if ( IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state) ) {
hex <<= 4;
parse_state += HEX((++hex_count));
hex += HEX_TO_INT(current_char);
if ( (hex_count == 2 && !(parse_state & EXTENDED2)) ||
(hex_count == 4 && !(parse_state & EXTENDED4)) ||
(hex_count == 8) ) {
addChar(s,(UChar32) hex);
if ( hex_count == 2 ) parse_state = 0;
else CLEAR_HEX(parse_state);
hex = hex_count = 0;
}
} else if ( parse_state && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE)
) ) {
if ( parse_state == APOSTROPHE && current_char != '\'' ) break;
throw IfcException("Invalid character encountered");
} else {
parse_state = hex = hex_count = 0;
s.put(current_char);
}
file->Inc();
}
s.put('\'');
return s.str();
}
void IfcCharacterDecoder::dryRun() {
unsigned int parse_state = 0;
char current_char;
unsigned int hex_count = 0;
while ( current_char = file->Peek() ) {
if ( EXPECTS_CHARACTER(parse_state) ) {
parse_state = 0;
} else if ( current_char == '\'' && ! parse_state ) {
parse_state = APOSTROPHE;
} else if ( current_char == '\\' && ! parse_state ) {
parse_state = FIRST_SOLIDUS;
} else if ( current_char == '\\' && EXPECTS_SOLIDUS(parse_state) ) {
if ( parse_state & ALPHABET_DEFINITION ||
parse_state & IGNORED_DIRECTIVE ||
parse_state & ENDEXTENDED_0 ) parse_state = hex_count = 0;
else if ( parse_state & HEX(3) ) parse_state += THIRD_SOLIDUS;
else parse_state += SECOND_SOLIDUS;
} else if ( current_char == 'X' && EXPECTS_ENDEXTENDED_X(parse_state) ) {
parse_state += ENDEXTENDED_X;
} else if ( current_char == '0' && EXPECTS_ENDEXTENDED_0(parse_state) ) {
parse_state += ENDEXTENDED_0;
} else if ( current_char == 'X' && EXPECTS_ARBITRARY(parse_state) ) {
parse_state += ARBITRARY;
} else if ( current_char == '2' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2;
} else if ( current_char == '4' && EXPECTS_ARBITRARY2(parse_state) ) {
parse_state += EXTENDED2 + EXTENDED4;
} else if ( current_char == 'P' && EXPECTS_ALPHABET(parse_state) ) {
parse_state += ALPHABET;
} else if ( (current_char == 'N' || current_char == 'F') && EXPECTS_N_OR_F(parse_state) ) {
parse_state += IGNORED_DIRECTIVE;
} else if ( IS_VALID_ALPHABET_DEFINITION(current_char) && EXPECTS_ALPHABET_DEFINITION(parse_state) ) {
parse_state += ALPHABET_DEFINITION;
} else if ( current_char == 'S' && EXPECTS_PAGE(parse_state) ) {
parse_state += PAGE;
} else if ( IS_HEXADECIMAL(current_char) && EXPECTS_HEX(parse_state) ) {
parse_state += HEX((++hex_count));
if ( (hex_count == 2 && !(parse_state & EXTENDED2)) ||
(hex_count == 4 && !(parse_state & EXTENDED4)) ||
(hex_count == 8) ) {
if ( hex_count == 2 ) parse_state = 0;
else CLEAR_HEX(parse_state);
hex_count = 0;
}
} else if ( parse_state && !(
(current_char == '\\' && parse_state == FIRST_SOLIDUS) ||
(current_char == '\'' && parse_state == APOSTROPHE)
) ) {
if ( parse_state == APOSTROPHE && current_char != '\'' ) break;
throw IfcException("Invalid character encountered");
} else {
parse_state = hex_count = 0;
}
file->Inc();
}
}
UConverter* IfcCharacterDecoder::destination = 0;
UConverter* IfcCharacterDecoder::converter = 0;
int IfcCharacterDecoder::previous_codepage = -1;
UErrorCode IfcCharacterDecoder::status = U_ZERO_ERROR;
IfcCharacterDecoder::ConversionMode IfcCharacterDecoder::mode = IfcCharacterDecoder::JSON;
src/ifcparse/IfcCharacterDecoder.h
+58
View File
@@ -0,0 +1,58 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
/********************************************************************************
* *
* Implementation of character decoding as described in ISO 10303-21 table 2 and *
* table 4 *
* *
********************************************************************************/
#ifndef IFCCHARACTERDECODER_H
#define IFCCHARACTERDECODER_H
#include <string>
#include <sstream>
#include "unicode/ucnv.h"
#include "../ifcparse/IfcFile.h"
namespace IfcParse {
class IfcCharacterDecoder {
private:
IfcParse::File* file;
static UConverter* destination;
static UConverter* converter;
static int previous_codepage;
static UErrorCode status;
void addChar(std::stringstream& s,const UChar32& ch);
public:
enum ConversionMode {UTF8,LATIN,JSON,PYTHON};
static ConversionMode mode;
IfcCharacterDecoder(IfcParse::File* file);
~IfcCharacterDecoder();
void dryRun();
operator std::string();
};
}
#endif
src/ifcparse/IfcException.h
+37
View File
@@ -0,0 +1,37 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
#ifndef IFCEXCEPTION_H
#define IFCEXCEPTION_H
#include <exception>
#include <string>
namespace IfcParse {
class IfcException : public std::exception {
private:
std::string error;
public:
IfcException(std::string e);
~IfcException () throw ();
const char* what() const throw();
};
}
#endif
src/ifcparse/IfcFile.h
+61
View File
@@ -0,0 +1,61 @@
/********************************************************************************
* *
* This file is part of IfcOpenShell. *
* *
* IfcOpenShell is free software: you can redistribute it and/or modify *
* it under the terms of the Lesser GNU General Public License as published by *
* the Free Software Foundation, either version 3.0 of the License, or *
* (at your option) any later version. *
* *
* IfcOpenShell is distributed in the hope that it will be useful, *
* but WITHOUT ANY WARRANTY; without even the implied warranty of *
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the *
* Lesser GNU General Public License for more details. *
* *
* You should have received a copy of the Lesser GNU General Public License *
* along with this program. If not, see <http://www.gnu.org/licenses/>. *
* *
********************************************************************************/
/********************************************************************************
* *
* Reads a file in chunks of BUF_SIZE and provides functions to access its *
* contents randomly and character by character *
* *
********************************************************************************/
#ifndef IFCFILE_H
#define IFCFILE_H
#include <fstream>
#include <string>
const int BUF_SIZE = (32 * 1024 * 1024);
namespace IfcParse {
class File {
private:
std::ifstream stream;
char* buffer;
unsigned int ptr;
unsigned int len;
unsigned int offset;
void ReadBuffer(bool inc=true);
bool paging;
public:
bool valid;
bool eof;
unsigned int size;
File(const std::string& fn);
File(std::istream& f, int len);
char Peek();
char Read(unsigned int offset);
void Inc();
void Close();
void Seek(unsigned int offset);
unsigned int Tell();
};
}
#endif
src/ifcparse/IfcParse.cpp
+20 -2
View File
@@ -20,8 +20,11 @@
#include <stdio.h>
#include <stdlib.h>
#include "../ifcparse/IfcCharacterDecoder.h"
#include "../ifcparse/IfcParse.h"
#include "../ifcparse/IfcException.h"
#include "../ifcparse/IfcUtil.h"
#include "../ifcparse/IfcFile.h"
using namespace IfcParse;
@@ -134,10 +137,17 @@ void File::Inc() {
if ( paging ) ReadBuffer();
else eof = true;
}
const char current = File::Peek();
if ( current == '\n' || current == '\r' ) File::Inc();
}
Tokens::Tokens(IfcParse::File *f) {
file = f;
decoder = new IfcCharacterDecoder(f);
}
Tokens::~Tokens() {
delete decoder;
}
//
@@ -186,7 +196,7 @@ Token Tokens::Next() {
// Keep track of whether cursor is inside a string or comment
if ( inComment && p == '*' && c == '/' ) inComment = false;
else if ( !inString && !inComment && p == '/' && c == '*' ) inComment = true;
else if ( !inComment && c == '\'' ) inString = ! inString;
else if ( !inComment && c == '\'' ) decoder->dryRun();
p = c;
}
@@ -214,7 +224,7 @@ std::string Tokens::TokenString(unsigned int offset) {
if ( !inComment ) buffer.push_back(c);
if ( inComment && p == '*' && c == '/' ) inComment = false;
else if ( !inString && !inComment && p == '/' && c == '*' ) inComment = true;
else if ( !inComment && c == '\'' ) inString = ! inString;
else if ( !inComment && c == '\'' ) return *decoder;
p = c;
}
file->Seek(old_offset);
@@ -450,6 +460,14 @@ ArgumentPtr Entity::getArgument(unsigned int i) {
return (*args)[i];
}
unsigned int Entity::getArgumentCount() {
if ( ! args ) {
std::vector<unsigned int> ids;
Load(ids, true);
}
return args->Size();
}
//
// Load the ArgumentList
//
src/ifcparse/IfcParse.h
+8 -41
View File
@@ -34,13 +34,12 @@
#include <fstream>
#include <cstring>
#include <map>
#include <exception>
#include "../ifcparse/SharedPointer.h"
#include "../ifcparse/IfcCharacterDecoder.h"
#include "../ifcparse/IfcUtil.h"
#include "../ifcparse/Ifc2x3.h"
const int BUF_SIZE = (32 * 1024 * 1024);
#include "../ifcparse/IfcFile.h"
namespace IfcParse {
@@ -49,33 +48,6 @@ namespace IfcParse {
typedef Entity* EntityPtr;
typedef SHARED_PTR<Entities> EntitiesPtr;
//
// Reads a file in chunks of BUF_SIZE and provides
// functions to randomly access its contents
//
class File {
private:
std::ifstream stream;
char* buffer;
unsigned int ptr;
unsigned int len;
unsigned int offset;
void ReadBuffer(bool inc=true);
bool paging;
public:
bool valid;
bool eof;
unsigned int size;
File(const std::string& fn);
File(std::istream& f, int len);
char Peek();
char Read(unsigned int offset);
void Inc();
void Close();
void Seek(unsigned int offset);
unsigned int Tell();
};
typedef unsigned int Token;
//
@@ -113,9 +85,11 @@ namespace IfcParse {
class Tokens {
private:
File* file;
IfcCharacterDecoder* decoder;
public:
Tokens(File* f);
Token Next();
~Tokens();
std::string TokenString(unsigned int offset);
};
@@ -154,8 +128,9 @@ namespace IfcParse {
//
class TokenArgument : public Argument {
private:
Token token;
public:
Token token;
TokenArgument(Token t);
operator int() const;
operator bool() const;
@@ -219,21 +194,13 @@ namespace IfcParse {
IfcEntities getInverse(Ifc2x3::Type::Enum c, int i, const std::string& a);
void Load(std::vector<unsigned int>& ids, bool seek=false);
ArgumentPtr getArgument (unsigned int i);
unsigned int getArgumentCount();
std::string toString();
std::string datatype();
Ifc2x3::Type::Enum type() const;
bool is(Ifc2x3::Type::Enum v) const;
unsigned int id();
};
class IfcException : public std::exception {
private:
std::string error;
public:
IfcException(std::string e);
~IfcException () throw ();
const char* what() const throw();
};
}
typedef IfcUtil::IfcSchemaEntity IfcEntity;
@@ -248,8 +215,8 @@ typedef std::map<unsigned int,unsigned int> MapOffsetById;
//
class Ifc {
private:
static MapEntitiesByType bytype;
static MapEntityById byid;
static MapEntitiesByType bytype;
static MapEntitiesByRef byref;
static MapOffsetById offsets;
static unsigned int lastId;
src/ifcparse/IfcUtil.h
+1
View File
@@ -143,6 +143,7 @@ public:
virtual IfcEntities getInverse(Ifc2x3::Type::Enum c, int i, const std::string& a) = 0;
virtual std::string datatype() = 0;
virtual ArgumentPtr getArgument (unsigned int i) = 0;
virtual unsigned int getArgumentCount() = 0;
virtual ~IfcAbstractEntity() {};
virtual Ifc2x3::Type::Enum type() const = 0;
virtual bool is(Ifc2x3::Type::Enum v) const = 0;