clank/rodecomp
1
0
Fork 0
Code Issues Wiki Activity

Decompile fn_18261 (CTokenizer::Parse)

Browse source 
Batch #16513 by Yommy: 1 byte-identical, 0 confirmed by peer review.

- fn_18261 CTokenizer::Parse: byte-identical

Merged automatically: every function is byte-identical to the original.
This commit is contained in:
ragnarok-clanker 2026-11-10 20:34:29 +00:00
commit c5d507fa23
1 changed files with 13 additions and 9 deletions
Download patch file Download diff file Expand all files Collapse all files

22
Source/Utility/Tokenizer.cpp
View file

@ -33,17 +33,17 @@ void CTokenizer::Parse(const std::string& text) {
unsigned int length = 0;
State state = State::Token;
const int size = static_cast<int>(text.size());
this->tokens.clear();
this->tokens.erase(this->tokens.begin(), this->tokens.end());
int index = 0;
for (; index < size; ++index) {
const char value = text[index];
if (state == State::Token) {
if (value == '/' && text[index + 1] == '/') {
switch (state) {
case State::Token:
if (text[index] == '/' && text[index + 1] == '/') {
return;
}
if (std::iswspace(static_cast<wchar_t>(value)) && value >= 0) {
if (std::iswspace(static_cast<wchar_t>(text[index])) && text[index] >= 0) {
if (index != 0) {
token[length] = '\0';
this->tokens.push_back(std::string(token));
@ -51,11 +51,15 @@ void CTokenizer::Parse(const std::string& text) {
}
state = State::Whitespace;
} else {
token[length++] = value;
token[length++] = text[index];
}
} else if (!std::iswspace(static_cast<wchar_t>(value)) || value < 0) {
--index;
state = State::Token;
break;
case State::Whitespace:
if (!std::iswspace(static_cast<wchar_t>(text[index])) || text[index] < 0) {
--index;
state = State::Token;
}
break;
}
}