diff options
author | Matthias Schiffer <matthias@gamezock.de> | 2009-09-27 19:58:24 +0200 |
---|---|---|
committer | Matthias Schiffer <matthias@gamezock.de> | 2009-09-27 19:58:24 +0200 |
commit | b40ba0cf91603b695f1f2380cbd39966a458f22f (patch) | |
tree | 1fec48ddc59eb1392fac38495b230e4b2cbf7528 /src/Core/Tokenizer.cpp | |
parent | e1d8490f0654a3da0b900407d80d91d8d0da68c8 (diff) | |
download | mad-b40ba0cf91603b695f1f2380cbd39966a458f22f.tar mad-b40ba0cf91603b695f1f2380cbd39966a458f22f.zip |
Use Unicode-aware String class instead of std::string
Diffstat (limited to 'src/Core/Tokenizer.cpp')
-rw-r--r-- | src/Core/Tokenizer.cpp | 14 |
1 files changed, 7 insertions, 7 deletions
diff --git a/src/Core/Tokenizer.cpp b/src/Core/Tokenizer.cpp index 93c853a..9e0553e 100644 --- a/src/Core/Tokenizer.cpp +++ b/src/Core/Tokenizer.cpp @@ -24,11 +24,11 @@ namespace Mad { namespace Core { -const UnicodeString Tokenizer::delimiters(" \t\n\"'\\"); +const String Tokenizer::delimiters(" \t\n\"'\\"); -std::vector<UnicodeString> Tokenizer::split(const UnicodeString &str) { - std::vector<UnicodeString> ret; +std::vector<String> Tokenizer::split(const String &str) { + std::vector<String> ret; for(boost::int32_t s = 0; s < str.length();) { boost::int32_t index = str.findFirstOf(delimiters, s); @@ -55,16 +55,16 @@ std::vector<UnicodeString> Tokenizer::split(const UnicodeString &str) { return ret; } -bool Tokenizer::tokenize(const UnicodeString &str, std::vector<UnicodeString> &out) { - std::vector<UnicodeString> splitString = split(str); +bool Tokenizer::tokenize(const String &str, std::vector<String> &out) { + std::vector<String> splitString = split(str); bool singleQuotes = false, doubleQuotes = false, escape = false; - UnicodeString token; + String token; bool forceToken = false; out.clear(); - for(std::vector<UnicodeString>::iterator s = splitString.begin(); s != splitString.end(); ++s) { + for(std::vector<String>::iterator s = splitString.begin(); s != splitString.end(); ++s) { token += *s; escape = false; |