Something went wrong. Try again.
Reactos
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549/* * PROJECT: ReactOS TXT to NLS Converter * LICENSE: GPL-2.0-or-later (https://spdx.org/licenses/GPL-2.0-or-later) * FILE: sdk/tools/txt2nls/main.c * COPYRIGHT: Copyright 2021 Jérôme Gardou <jerome.gardou@reactos.org> */
#include <iostream>#include <fstream>#include <limits>#include <vector>#include <cstring>#include <string>#include <cstdint>
static const char whitespaces[] = " \t\f\v\n\r";static long line_number = -1;
#pragma pack(push, 1)#define MAXIMUM_LEADBYTES 12struct NLS_FILE_HEADER{ uint16_t HeaderSize; uint16_t CodePage; uint16_t MaximumCharacterSize; uint16_t DefaultChar; uint16_t UniDefaultChar; uint16_t TransDefaultChar; uint16_t TransUniDefaultChar; uint8_t LeadByte[MAXIMUM_LEADBYTES];};static_assert(sizeof(NLS_FILE_HEADER) == 26, "Wrong size for NLS_FILE_HEADER");#pragma pack(pop)
static std::istream& get_clean_line(std::istream& stream, std::string& str){ do { std::istream& ret = std::getline(stream, str); if (!ret) return ret;
/* Ignore comments */ std::size_t comment_pos = str.find_first_of(';'); if (comment_pos != std::string::npos) { str.erase(comment_pos); }
/* Remove trailing spaces */ std::size_t end_of_line = str.find_last_not_of(whitespaces); if (end_of_line != std::string::npos) str.erase(end_of_line + 1); else str.clear();
line_number++; } while (str.empty());
return stream;}
static void tokenize(std::string& str, std::string& token){ std::size_t token_start = str.find_first_not_of(whitespaces); if (token_start == std::string::npos) { token = ""; str.clear(); return; }
std::size_t token_end = str.find_first_of(whitespaces, token_start); if (token_end == std::string::npos) { token = str.substr(token_start); str.clear(); return; }
token = str.substr(token_start, token_end); str.erase(0, str.find_first_not_of(whitespaces, token_end));}
template<typename T>static void tokenize(std::string& str, T& int_token, int base = 0){ std::string token; tokenize(str, token);
long val; val = std::stol(token, nullptr, base); if ((val > std::numeric_limits<T>::max()) || (val < std::numeric_limits<T>::min())) throw std::invalid_argument(token + " does not fit range [" + std::to_string(std::numeric_limits<T>::min()) + ":" + std::to_string(std::numeric_limits<T>::max()) + "]");
int_token = val;}
void error(const std::string& err){ std::cerr << "Error parsing line " << line_number <<": " << err << std::endl; std::exit(1);}
int main(int argc, char* argv[]){ if (argc != 3) { std::cerr << "Usage: " << argv[0] << " <txt_in> <nls_out>" << std::endl; return 1; }
std::ifstream input(argv[1]); if (!input.is_open()) { std::cerr << "Unable to open " << argv[1] << std::endl; return 1; }
NLS_FILE_HEADER FileHeader; memset(&FileHeader, 0, sizeof(FileHeader));
std::string curr_line; // Get code page if (!get_clean_line(input, curr_line)) { std::cerr << "ERROR: File is empty" << std::endl; return 1; }
std::string token; tokenize(curr_line, token); if (token != "CODEPAGE") error("expected CODEPAGE, got \"" + token + "\" instead"); try { tokenize(curr_line, FileHeader.CodePage, 10); } catch(const std::invalid_argument& ia) { error(ia.what()); }
if (!curr_line.empty()) error("Garbage after CODEPAGE statement: \"" + curr_line + "\"");
/* Get CPINFO */ if (!get_clean_line(input, curr_line)) error("Nothing after CODEPAGE statement");
tokenize(curr_line, token); if (token != "CPINFO") error("Expected CPINFO, got \"" + token + "\" instead"); try { tokenize(curr_line, FileHeader.MaximumCharacterSize); tokenize(curr_line, FileHeader.DefaultChar); tokenize(curr_line, FileHeader.UniDefaultChar); } catch(const std::invalid_argument& ia) { error(ia.what()); return 1; } if (!curr_line.empty()) error("Garbage after CPINFO statement: \"" + curr_line + "\""); if ((FileHeader.MaximumCharacterSize != 1) && (FileHeader.MaximumCharacterSize != 2)) error("Expected 1 or 2 as max char size in CPINFO, got \"" + std::to_string(FileHeader.MaximumCharacterSize) + "\" instead"); if ((FileHeader.MaximumCharacterSize == 1) && (FileHeader.DefaultChar > std::numeric_limits<uint8_t>::max())) error("Default MB character " + std::to_string(FileHeader.DefaultChar) + " doesn't fit in a 8-bit value");
/* Setup tables & default values */ bool has_mbtable = false; uint16_t mb_table[256] = {0};
bool has_wctable = false; uint8_t* wc_table = new uint8_t[65536 * FileHeader.MaximumCharacterSize]; if (FileHeader.MaximumCharacterSize == 1) { for (int i = 0; i < 65536; i++) wc_table[i] = FileHeader.DefaultChar; } else { uint16_t* wc_table_dbcs = reinterpret_cast<uint16_t*>(wc_table); for (int i = 0; i < 65536; i++) wc_table_dbcs[i] = FileHeader.DefaultChar; }
std::vector<uint16_t> dbcs_table; uint16_t lb_offsets[256] = {0}; uint16_t dbcs_range_count = 0;
uint16_t glyph_table[256] = {0}; bool has_glyphs = false;
/* Now parse */ while (get_clean_line(input, curr_line)) { tokenize(curr_line, token);
if (token == "ENDCODEPAGE") { if (!curr_line.empty()) error("Garbage after ENDCODEPAGE statement: \"" + curr_line + "\""); break; } else if (token == "MBTABLE") { uint16_t table_size; try { tokenize(curr_line, table_size); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (has_mbtable) error("MBTABLE can only be declared once"); if (table_size > 256) error("MBTABLE size can't be larger than 256"); if (!curr_line.empty()) error("Garbage after MBTABLE statement: \"" + curr_line + "\"");
has_mbtable = true; while (table_size--) { if (!get_clean_line(input, curr_line)) error("Expected " + std::to_string(table_size + 1) + " more lines after MBTABLE token");
uint8_t mb; uint16_t wc;
try { tokenize(curr_line, mb); tokenize(curr_line, wc); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after MBTABLE entry: \"" + curr_line + "\""); mb_table[mb] = wc; } } else if (token == "WCTABLE") { uint32_t table_size; try { tokenize(curr_line, table_size); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (has_wctable) error("WCTABLE can only be declared once"); if (!curr_line.empty()) error("Garbage after WCTABLE statement: \"" + curr_line + "\""); if (table_size > 65536) error("WCTABLE size can't be larger than 65536");
has_wctable = true;
if (FileHeader.MaximumCharacterSize == 1) { while (table_size--) { if (!get_clean_line(input, curr_line)) error("Expected " + std::to_string(table_size + 1) + " more lines after WCTABLE token");
uint8_t mb; uint16_t wc;
try { tokenize(curr_line, wc); tokenize(curr_line, mb); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after WCTABLE entry: \"" + curr_line + "\""); wc_table[wc] = mb; } } else { uint16_t* wc_table_dbcs = reinterpret_cast<uint16_t*>(wc_table); while (table_size--) { if (!get_clean_line(input, curr_line)) error("Expected " + std::to_string(table_size + 1) + " more lines after WCTABLE token"); uint16_t mb; uint16_t wc;
try { tokenize(curr_line, wc); tokenize(curr_line, mb); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after MBTABLE entry: \"" + curr_line + "\""); wc_table_dbcs[wc] = mb; } } } else if (token == "DBCSRANGE") { if (dbcs_range_count != 0) error("DBCSRANGE can only be declared once");
try { tokenize(curr_line, dbcs_range_count); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (dbcs_range_count > (MAXIMUM_LEADBYTES / 2)) error("DBCSRANGE count can't exceed " + std::to_string(MAXIMUM_LEADBYTES / 2)); if (!curr_line.empty()) error("Garbage after DBCSRANGE token");
std::size_t current_offset = 0;
uint16_t range_count = dbcs_range_count; uint16_t current_range = 0; while (range_count--) { if (!get_clean_line(input, curr_line)) error("Expected new range after DBCSRANGE");
uint8_t RangeStart, RangeEnd; try { tokenize(curr_line, RangeStart); tokenize(curr_line, RangeEnd); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after DBCS range declaration");
if (RangeStart > RangeEnd) error("Invalid range specified for DBCSRANGE");
FileHeader.LeadByte[current_range*2] = RangeStart; FileHeader.LeadByte[current_range*2+1] = RangeEnd; current_range++;
dbcs_table.resize(dbcs_table.size() + 256 * (RangeEnd - RangeStart + 1), FileHeader.UniDefaultChar);
for (uint8_t LeadByte = RangeStart; LeadByte <= RangeEnd; LeadByte++) { if (!get_clean_line(input, curr_line)) error("Expected new DBCSTABLE after DBCS range declaration");
tokenize(curr_line, token); if (token != "DBCSTABLE") error("Expected new DBCSTABLE after DBCS range declaration");
uint16_t table_size; try { tokenize(curr_line, table_size); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (table_size > 256) error("DBCSTABLE can't have more than 256 entries"); while (table_size--) { if (!get_clean_line(input, curr_line)) error("Expected " + std::to_string(table_size + 1) + " more lines after DBCSTABLE token");
uint8_t mb; uint16_t wc;
try { tokenize(curr_line, mb); tokenize(curr_line, wc); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after DBCSTABLE entry: \"" + curr_line + "\"");
dbcs_table[current_offset + mb] = wc; } current_offset += 256; /* Offsets start at 256 for the offset table. */ lb_offsets[LeadByte] = current_offset; } } } else if (token == "GLYPHTABLE") { uint16_t table_size; try { tokenize(curr_line, table_size); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (has_glyphs) error("GLYPHTABLE can only be declared once"); if (table_size > 256) error("GLYPHTABLE size can't be larger than 256"); if (!curr_line.empty()) error("Garbage after GLYPHTABLE statement: \"" + curr_line + "\""); has_glyphs = true;
while (table_size--) { if (!get_clean_line(input, curr_line)) error("Expected " + std::to_string(table_size + 1) + " more lines after GLYPHTABLE token");
uint8_t mb; uint16_t wc;
try { tokenize(curr_line, mb); tokenize(curr_line, wc); } catch(const std::invalid_argument& ia) { error(ia.what()); } if (!curr_line.empty()) error("Garbage after GLYPHTABLE entry: \"" + curr_line + "\""); glyph_table[mb] = wc; } } else { error("Unexpected token \"" + token + "\""); } }
if (token != "ENDCODEPAGE") error("Expected last token to be \"ENDCODEPAGE\"");
input.close();
/* Ensure this is minimally workable */ if (!has_mbtable) error("File has no MBTABLE statement"); if (!has_wctable) error("File has no WCTABLE statement");
/* Glyph table fixup */ if (has_glyphs) { for(int i = 0; i < 256; i++) { if (glyph_table[i] == 0) glyph_table[i] = mb_table[i]; } }
/* Translated default char fixup */ if (FileHeader.MaximumCharacterSize == 1) { FileHeader.TransDefaultChar = mb_table[FileHeader.DefaultChar]; FileHeader.TransUniDefaultChar = wc_table[FileHeader.UniDefaultChar]; } else { if (FileHeader.DefaultChar > 0xFF) { uint16_t offset = lb_offsets[FileHeader.DefaultChar >> 8]; if (!offset) error("Default MB char is not translatable!"); FileHeader.TransDefaultChar = dbcs_table[(FileHeader.DefaultChar & 0xFF) + (offset - 256)]; } else { FileHeader.TransDefaultChar = mb_table[FileHeader.DefaultChar]; } uint16_t* wc_table_dbcs = reinterpret_cast<uint16_t*>(wc_table); FileHeader.TransUniDefaultChar = wc_table_dbcs[FileHeader.UniDefaultChar]; } FileHeader.HeaderSize = sizeof(NLS_FILE_HEADER) / sizeof(uint16_t);
std::ofstream output(argv[2], std::ios_base::binary);
output.write(reinterpret_cast<char*>(&FileHeader), sizeof(FileHeader));
uint16_t wc_table_offset = sizeof(mb_table) / sizeof(uint16_t) + 1 /* size of glyph table */ + (has_glyphs ? 256 : 0) /* Glyph table */ + 1 /* Number of DBCS LeadByte ranges */ + (dbcs_range_count ? 256 : 0) /* offsets of lead byte sub tables */ + dbcs_table.size() /* LeadByte sub tables */ + 1; /* Unknown flag */
output.write(reinterpret_cast<char*>(&wc_table_offset), sizeof(wc_table_offset));
output.write(reinterpret_cast<char*>(mb_table), sizeof(mb_table));
uint16_t glyph_table_size = has_glyphs ? 256 : 0; output.write(reinterpret_cast<char*>(&glyph_table_size), sizeof(glyph_table_size)); if (has_glyphs) output.write(reinterpret_cast<char*>(glyph_table), sizeof(glyph_table));
output.write(reinterpret_cast<char*>(&dbcs_range_count), sizeof(dbcs_range_count)); if (dbcs_range_count) { output.write(reinterpret_cast<char*>(lb_offsets), sizeof(lb_offsets)); } if (dbcs_table.size()) { output.write(reinterpret_cast<char*>(dbcs_table.data()), dbcs_table.size() * sizeof(uint16_t)); }
uint16_t unknown_flag = FileHeader.MaximumCharacterSize == 1 ? 0 : 4; output.write(reinterpret_cast<char*>(&unknown_flag), sizeof(unknown_flag));
output.write(reinterpret_cast<char*>(wc_table), 65536 * FileHeader.MaximumCharacterSize);
output.close(); delete[] wc_table;
return 0;}