Something went wrong. Try again.
OpenMW is an open implementation of the Morrowind engine
Something went wrong. Try again.
9.6 kB · 333 lines
C++
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334#include "lexer.hpp"
#include <cctype>#include <cmath>#include <cstdlib>#include <format>
#include <components/misc/strings/algorithm.hpp>
namespace Fx{ namespace Lexer { Lexer::Lexer(std::string_view buffer) : mHead(buffer.data()) , mTail(mHead + buffer.length()) , mAbsolutePos(0) , mColumn(0) , mLine(0) , mBuffer(buffer) , mLastToken(Eof{}) { }
Token Lexer::next() { if (mLookahead) { auto token = *mLookahead; drop(); return token; }
mLastToken = scanToken();
return mLastToken; }
Token Lexer::peek() { if (!mLookahead) mLookahead = scanToken();
return *mLookahead; }
void Lexer::drop() { mLookahead = std::nullopt; }
std::optional<std::string_view> Lexer::jump() { bool multi = false; bool single = false; auto start = mHead; std::size_t level = 1;
mLastJumpBlock.mLine = mLine;
if (head() == '}') { mLastJumpBlock.mContent = {}; return mLastJumpBlock.mContent; }
for (; mHead != mTail; advance()) { if (head() == '\n') { mLine++; mColumn = 0; if (single) { single = false; continue; } } else if (multi && head() == '*' && peekChar('/')) { multi = false; advance(); continue; } else if (multi || single) { continue; } else if (head() == '/' && peekChar('/')) { single = true; advance(); continue; } else if (head() == '/' && peekChar('*')) { multi = true; advance(); continue; }
if (head() == '{') level++; else if (head() == '}') level--;
if (level == 0) { mHead--; mLine--; auto sv = std::string_view{ start, static_cast<std::string_view::size_type>(mHead + 1 - start) }; mLastJumpBlock.mContent = sv; return sv; } }
mLastJumpBlock = {}; return std::nullopt; }
Lexer::Block Lexer::getLastJumpBlock() const { return mLastJumpBlock; }
[[noreturn]] void Lexer::error(std::string_view msg) { throw LexerException(std::format("Line {} Col {}. {}", mLine + 1, mColumn, msg)); }
void Lexer::advance() { mAbsolutePos++; mHead++; mColumn++; }
unsigned char Lexer::head() { return *mHead; }
bool Lexer::peekChar(char c) { if (mHead == mTail) return false; return *(mHead + 1) == c; }
Token Lexer::scanToken() { while (true) { if (mHead == mTail) return { Eof{} };
if (head() == '\n') { mLine++; mColumn = 0; }
if (!std::isspace(head())) break;
advance(); }
if (head() == '\"') return scanStringLiteral();
if (std::isalpha(head())) return scanLiteral();
if (std::isdigit(head()) || head() == '.' || head() == '-') return scanNumber();
switch (head()) { case '=': advance(); return { Equal{} }; case '{': advance(); return { Open_bracket{} }; case '}': advance(); return { Close_bracket{} }; case '(': advance(); return { Open_Parenthesis{} }; case ')': advance(); return { Close_Parenthesis{} }; case '\"': advance(); return { Quote{} }; case ':': advance(); return { Colon{} }; case ';': advance(); return { SemiColon{} }; case '|': advance(); return { VBar{} }; case ',': advance(); return { Comma{} }; default: error(std::format("unexpected token <{:c}>", head())); } }
Token Lexer::scanLiteral() { auto start = mHead; advance();
while (mHead != mTail && (std::isalnum(head()) || head() == '_')) advance();
std::string_view value{ start, static_cast<std::string_view::size_type>(mHead - start) };
if (value == "shared") return Shared{}; if (value == "technique") return Technique{}; if (value == "render_target") return Render_Target{}; if (value == "vertex") return Vertex{}; if (value == "fragment") return Fragment{}; if (value == "compute") return Compute{}; if (value == "sampler_1d") return Sampler_1D{}; if (value == "sampler_2d") return Sampler_2D{}; if (value == "sampler_3d") return Sampler_3D{}; if (value == "uniform_bool") return Uniform_Bool{}; if (value == "uniform_float") return Uniform_Float{}; if (value == "uniform_int") return Uniform_Int{}; if (value == "uniform_vec2") return Uniform_Vec2{}; if (value == "uniform_vec3") return Uniform_Vec3{}; if (value == "uniform_vec4") return Uniform_Vec4{}; if (value == "true") return True{}; if (value == "false") return False{}; if (value == "vec2") return Vec2{}; if (value == "vec3") return Vec3{}; if (value == "vec4") return Vec4{};
return Literal{ value }; }
Token Lexer::scanStringLiteral() { advance(); // consume quote auto start = mHead;
bool terminated = false;
for (; mHead != mTail; advance()) { if (head() == '\"') { terminated = true; advance(); break; } }
if (!terminated) error("unterminated string");
return String{ { start, static_cast<std::string_view::size_type>(mHead - start - 1) } }; }
Token Lexer::scanNumber() { double buffer; char* endPtr = nullptr; buffer = std::strtod(mHead, &endPtr); if (endPtr == nullptr || endPtr == mHead) error("critical error while parsing number");
bool isDefinitelyAFloat = false; // GLSL allows floats to end on f/F if (endPtr != mTail && Misc::StringUtils::toLower(*endPtr) == 'f') { isDefinitelyAFloat = true; ++endPtr; }
std::string_view literal(mHead, endPtr); mHead = endPtr;
// Disallow -inf, -nan, and values that cannot be represented as doubles if (!std::isfinite(buffer)) return Literal{ literal }; // Disallow hex notation (not allowed in GLSL so confusing to partially allow) if (Misc::StringUtils::ciStartsWith(literal, "0x") || Misc::StringUtils::ciStartsWith(literal, "-0x")) return Literal{ literal };
constexpr std::string_view floatCharacters = ".eE"; if (!isDefinitelyAFloat && literal.find_first_of(floatCharacters) == std::string_view::npos) { // This is supposed to be an int, but that doesn't mean it can fit in one int intValue = static_cast<int>(buffer); if (intValue != buffer) error("number out of range"); return Integer{ intValue }; } float floatValue = static_cast<float>(buffer); if (!std::isfinite(floatValue)) error("number out of range"); return Float{ floatValue }; } }}