Something went wrong. Try again.
OpenMW is an open implementation of the Morrowind engine
Something went wrong. Try again.
9.5 kB · 258 lines
C++
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259#include "utf8.hpp"
#include <format>
namespace{ constexpr std::string_view UTF8PATT = "[%z\x01-\x7F\xC2-\xF4][\x80-\xBF]*"; // %z is deprecated in Lua5.2 // constexpr uint32_t MAXUTF = 0x7FFFFFFFu; constexpr uint32_t MAXUNICODE = 0x10FFFFu;
inline bool isNilOrNone(const sol::stack_proxy arg) { return (arg.get_type() == sol::type::lua_nil || arg.get_type() == sol::type::none); }
inline std::int64_t getInteger(const sol::stack_proxy arg, const size_t n, std::string_view name) { double integer; if (!arg.is<double>()) throw std::runtime_error(std::format("bad argument #{} to '{}' (number expected, got {})", n, name, sol::type_name(arg.lua_state(), arg.get_type())));
if (std::modf(arg, &integer) != 0) throw std::runtime_error( std::format("bad argument #{} to '{}' (number has no integer representation)", n, name));
return static_cast<std::int64_t>(integer); }
// If the input 'pos' is negative, it is treated as counting from the end of the string, // where -1 represents the last character position, -2 represents the second-to-last position, // and so on. If 'pos' is non-negative, it is used as-is. inline void relativePosition(int64_t& pos, const size_t len) { if (pos < 0) pos = std::max<int64_t>(0, pos + len + 1); }
inline void codepointToUTF8(char32_t codepoint, std::string& str) { if (codepoint <= 0x7Fu) { str.push_back(static_cast<char>(codepoint)); } else if (codepoint <= 0x7FFu) { str.push_back(static_cast<char>(0xC0 | ((codepoint & 0x7C0) >> 6))); str.push_back(static_cast<char>(0x80 | (codepoint & 0x3F))); } else if (codepoint <= 0xFFFFu) { str.push_back(static_cast<char>(0xE0 | ((codepoint & 0xF000) >> 12))); str.push_back(static_cast<char>(0x80 | ((codepoint & 0xFC0) >> 6))); str.push_back(static_cast<char>(0x80 | (codepoint & 0x3F))); } else if (codepoint <= MAXUNICODE) { str.push_back(static_cast<char>(0xF0 | ((codepoint & 0x1C0000) >> 18))); str.push_back(static_cast<char>(0x80 | ((codepoint & 0x3F000) >> 12))); str.push_back(static_cast<char>(0x80 | ((codepoint & 0xFC0) >> 6))); str.push_back(static_cast<char>(0x80 | (codepoint & 0x3F))); } else throw std::runtime_error("Invalid codepoint"); }
// returns: first - character pos in bytes, second - character codepoint std::pair<int64_t, int64_t> decodeNextUTF8Character(std::string_view s, std::vector<int64_t>& posByte) { const int64_t pos = posByte.back() - 1; const unsigned char ch = static_cast<unsigned char>(s[pos]); int64_t codepoint = -1; size_t byteSize = 0;
if ((ch & 0b10000000) == 0) { codepoint = ch; byteSize = 1; } else if ((ch & 0b11100000) == 0b11000000) { codepoint = ch & 0b00011111; byteSize = 2; } else if ((ch & 0b11110000) == 0b11100000) { codepoint = ch & 0b00001111; byteSize = 3; } else if ((ch & 0b11111000) == 0b11110000) { codepoint = ch & 0b00000111; byteSize = 4; }
// construct codepoint for non-ascii for (size_t i = 1; i < byteSize; ++i) { // if not a continuation byte if ((pos + i) >= s.size() || (static_cast<unsigned char>(s[pos + i]) & 0b11000000) != 0b10000000) { return std::make_pair(0, -1); } codepoint = (codepoint << 6) | (static_cast<unsigned char>(s[pos + i]) & 0b00111111); }
std::pair<size_t, int64_t> res = std::make_pair(posByte.back(), codepoint);
posByte.push_back(posByte.back() + byteSize); /* the next character (if exists) starts at this byte */
return res; }
}
namespace LuaUtf8{ sol::table initUtf8Package(sol::state_view& lua) { sol::table utf8(lua, sol::create);
utf8["charpattern"] = UTF8PATT;
utf8["char"] = [](const sol::variadic_args args) -> std::string { std::string result{}; for (size_t i = 0; i < args.size(); ++i) { int64_t codepoint = getInteger(args[i], (i + 1), "char"); if (codepoint < 0 || codepoint > MAXUNICODE) throw std::runtime_error(std::format("bad argument #{} to 'char' (value out of range)", i + 1));
codepointToUTF8(static_cast<char32_t>(codepoint), result); } return result; };
utf8["codes"] = [](std::string_view s) { return sol::as_function( [s, posByte = std::vector<int64_t>{ 1 }]() mutable -> sol::optional<std::pair<int64_t, int64_t>> { if (posByte.back() <= static_cast<int64_t>(s.size())) { const auto pair = decodeNextUTF8Character(s, posByte); if (pair.second == -1) throw std::runtime_error(std::format("Invalid UTF-8 code at position {}", posByte.size()));
return pair; } return sol::nullopt; }); };
utf8["len"] = [](std::string_view s, const sol::variadic_args args) -> std::variant<size_t, std::pair<sol::object, int64_t>> { const size_t len = s.size(); int64_t iv = isNilOrNone(args[0]) ? 1 : getInteger(args[0], 2, "len"); int64_t fv = isNilOrNone(args[1]) ? -1 : getInteger(args[1], 3, "len");
relativePosition(iv, len); relativePosition(fv, len);
if (iv <= 0) throw std::runtime_error("bad argument #2 to 'len' (initial position out of bounds)"); if (fv > static_cast<int64_t>(len)) throw std::runtime_error("bad argument #3 to 'len' (final position out of bounds)");
if (len == 0) return len;
std::vector<int64_t> posByte = { iv };
while (posByte.back() <= fv) { if (decodeNextUTF8Character(s, posByte).second == -1) return std::pair(sol::lua_nil, posByte.back()); } return posByte.size() - 1; };
utf8["codepoint"] = [](std::string_view s, const sol::variadic_args args) -> sol::as_returns_t<std::vector<int64_t>> { size_t len = s.size(); int64_t iv = isNilOrNone(args[0]) ? 1 : getInteger(args[0], 2, "codepoint"); int64_t fv = isNilOrNone(args[1]) ? iv : getInteger(args[1], 3, "codepoint");
relativePosition(iv, len); relativePosition(fv, len);
if (iv <= 0) throw std::runtime_error("bad argument #2 to 'codepoint' (initial position out of bounds)"); if (fv > static_cast<int64_t>(len)) throw std::runtime_error("bad argument #3 to 'codepoint' (final position out of bounds)");
if (iv > fv) return sol::as_returns(std::vector<int64_t>{}); /* empty interval; return nothing */
std::vector<int64_t> posByte = { iv }; std::vector<int64_t> codepoints;
while (posByte.back() <= fv) { codepoints.push_back(decodeNextUTF8Character(s, posByte).second); if (codepoints.back() == -1) throw std::runtime_error(std::format("Invalid UTF-8 code at position {}", posByte.size())); }
return sol::as_returns(std::move(codepoints)); };
utf8["offset"] = [](std::string_view s, const int64_t n, const sol::variadic_args args) -> sol::optional<int64_t> { size_t len = s.size(); int64_t iv;
if (isNilOrNone(args[0])) { if (n >= 0) iv = 1; else iv = s.size() + 1; } else iv = getInteger(args[0], 3, "offset");
std::vector<int64_t> posByte = { 1 };
relativePosition(iv, len);
if (iv > static_cast<int64_t>(len) + 1) throw std::runtime_error("bad argument #3 to 'offset' (position out of bounds)");
while (posByte.back() <= static_cast<int64_t>(len)) decodeNextUTF8Character(s, posByte);
for (auto it = posByte.begin(); it != posByte.end(); ++it) { if (*it == iv) { if (n <= 0 && it + n >= posByte.begin()) return *(it + n); if (n > 0 && it + n - 1 < posByte.end()) return *(it + n - 1); break; } else if (*it > iv) /* a continuation byte */ { if (n == 0) return *(it - 1); /* special case */ else throw std::runtime_error("initial position is a continuation byte"); } }
return sol::nullopt; };
return utf8; }}