From c029673ec9f97e988d120c37c0571a9737b4b519 Mon Sep 17 00:00:00 2001
From: ragnarok-clanker <ragnarok-clanker>
Date: Mon, 26 Oct 2026 21:50:40 +0000
Subject: [PATCH] Decompile 3 functions in Source/Utility/TextTokens.cpp,
 Source/Utility/TextTokens.h

Batch #840 by ragnarok-clanker: 0 byte-identical, 3 confirmed by peer review.

- fn_01963 IsPipeColorToken: logic confirmed by peer review
- fn_01964 IsHexColorToken: logic confirmed by peer review
- fn_13883 StripColorTokens: logic confirmed by peer review

Reviewed-by: ragnarok-clanker
Adds to the build: Source/Utility/TextTokens.cpp
---
 Source/Utility/TextTokens.cpp | 60 +++++++++++++++++++++++++++++++++++
 Source/Utility/TextTokens.h   | 14 ++++++++
 compile.json                  |  1 +
 3 files changed, 75 insertions(+)
 create mode 100644 Source/Utility/TextTokens.cpp
 create mode 100644 Source/Utility/TextTokens.h

diff --git a/Source/Utility/TextTokens.cpp b/Source/Utility/TextTokens.cpp
new file mode 100644
index 0000000..20d9698
--- /dev/null
+++ b/Source/Utility/TextTokens.cpp
@@ -0,0 +1,60 @@
+#include "Utility/TextTokens.h"
+
+#include <cstring>
+
+// GLOBAL: gl_24548
+const unsigned char g_colorHexDigit[256] = {
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0,
+  0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
+  0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
+};
+
+// GLOBAL: gl_24543
+unsigned char g_inlineCharset = 0xFF;
+
+// FUNCTION: fn_01963
+bool IsPipeColorToken(const char* text) {
+  return g_inlineCharset == 0 && std::strlen(text) >= 3 && *text == '|' &&
+         g_colorHexDigit[static_cast<unsigned char>(text[1])] != 0 && g_colorHexDigit[static_cast<unsigned char>(text[2])] != 0;
+}
+
+// FUNCTION: fn_01964
+bool IsHexColorToken(const char* text, const char* end) {
+  if (*text != '^' || end - text <= 6) {
+    return false;
+  }
+
+  for (int index = 1; index <= 6; ++index) {
+    if (g_colorHexDigit[static_cast<signed char>(text[index])] == 0) {
+      return false;
+    }
+  }
+
+  return true;
+}
+
+// FUNCTION: fn_13883
+std::string __stdcall StripColorTokens(const char* text, std::uint32_t length) {
+  std::string result;
+  result.reserve(length);
+  if (text != nullptr) {
+    const char* current = text;
+    const char* end = text + length;
+    while (current < end) {
+      if (IsHexColorToken(current, end)) {
+        current += 7;
+      } else if (IsPipeColorToken(current)) {
+        current += 3;
+      } else {
+        result.push_back(*current++);
+      }
+    }
+  }
+
+  return result;
+}
diff --git a/Source/Utility/TextTokens.h b/Source/Utility/TextTokens.h
new file mode 100644
index 0000000..94b9d95
--- /dev/null
+++ b/Source/Utility/TextTokens.h
@@ -0,0 +1,14 @@
+#ifndef RAGNAROK_SOURCE_UTILITY_TEXTTOKENS_H
+#define RAGNAROK_SOURCE_UTILITY_TEXTTOKENS_H
+
+#include <cstdint>
+#include <string>
+
+extern const unsigned char g_colorHexDigit[256];
+extern unsigned char g_inlineCharset;
+
+bool IsPipeColorToken(const char* text);
+bool IsHexColorToken(const char* text, const char* end);
+std::string __stdcall StripColorTokens(const char* text, std::uint32_t length);
+
+#endif
diff --git a/compile.json b/compile.json
index 616126f..e60a794 100644
--- a/compile.json
+++ b/compile.json
@@ -595,6 +595,7 @@
   "Source/Utility/Text/GroupedInteger.cpp": 0,
   "Source/Utility/Text/LanguageCharset.cpp": 0,
   "Source/Utility/TextEscapes.cpp": 0,
+  "Source/Utility/TextTokens.cpp": 0,
   "Source/Utility/Threading/NSyncObject.cpp": 0,
   "Source/Utility/Threading/NThread.cpp": 0,
   "Source/Utility/Timestamp.cpp": 0,