Something went wrong. Try again.
A fork of https://github.com/crosspoint-reader/crosspoint-reader
Something went wrong. Try again.
14 kB · 390 lines
C++
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390#include <Utf8.h>
#include <algorithm>#include <cctype>#include <cmath>#include <fstream>#include <functional>#include <iostream>#include <sstream>#include <string>#include <vector>
#include "lib/Epub/Epub/hyphenation/HyphenationCommon.h"#include "lib/Epub/Epub/hyphenation/LanguageHyphenator.h"#include "lib/Epub/Epub/hyphenation/LanguageRegistry.h"
struct TestCase { std::string word; std::string hyphenated; std::vector<size_t> expectedPositions; int frequency;};
struct EvaluationResult { int truePositives = 0; int falsePositives = 0; int falseNegatives = 0; double precision = 0.0; double recall = 0.0; double f1Score = 0.0; double weightedScore = 0.0;};
struct LanguageConfig { std::string cliName; std::string testDataFile; const char* primaryTag;};
const std::vector<LanguageConfig> kSupportedLanguages = { {"english", "test/hyphenation_eval/resources/english_hyphenation_tests.txt", "en"}, {"french", "test/hyphenation_eval/resources/french_hyphenation_tests.txt", "fr"}, {"german", "test/hyphenation_eval/resources/german_hyphenation_tests.txt", "de"}, {"russian", "test/hyphenation_eval/resources/russian_hyphenation_tests.txt", "ru"}, {"spanish", "test/hyphenation_eval/resources/spanish_hyphenation_tests.txt", "es"}, {"italian", "test/hyphenation_eval/resources/italian_hyphenation_tests.txt", "it"},};
std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) { std::vector<size_t> positions; const unsigned char* ptr = reinterpret_cast<const unsigned char*>(annotated.c_str()); size_t codepointIndex = 0;
while (*ptr != 0) { if (*ptr == '=') { positions.push_back(codepointIndex); ++ptr; continue; }
utf8NextCodepoint(&ptr); ++codepointIndex; }
return positions;}
std::vector<TestCase> loadTestData(const std::string& filename) { std::vector<TestCase> testCases; std::ifstream file(filename);
if (!file.is_open()) { std::cerr << "Error: Could not open file " << filename << std::endl; return testCases; }
std::string line; while (std::getline(file, line)) { if (line.empty() || line[0] == '#') { continue; }
std::istringstream iss(line); std::string word, hyphenated, freqStr;
if (std::getline(iss, word, '|') && std::getline(iss, hyphenated, '|') && std::getline(iss, freqStr, '|')) { TestCase testCase; testCase.word = word; testCase.hyphenated = hyphenated; testCase.frequency = std::stoi(freqStr);
testCase.expectedPositions = expectedPositionsFromAnnotatedWord(hyphenated);
testCases.push_back(testCase); } }
file.close(); return testCases;}
std::string positionsToHyphenated(const std::string& word, const std::vector<size_t>& positions) { std::string result; std::vector<size_t> sortedPositions = positions; std::sort(sortedPositions.begin(), sortedPositions.end());
const unsigned char* ptr = reinterpret_cast<const unsigned char*>(word.c_str()); size_t codepointIndex = 0; size_t posIdx = 0;
while (*ptr != 0) { while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) { result.push_back('='); ++posIdx; }
const unsigned char* current = ptr; utf8NextCodepoint(&ptr); result.append(reinterpret_cast<const char*>(current), reinterpret_cast<const char*>(ptr)); ++codepointIndex; }
while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) { result.push_back('='); ++posIdx; }
return result;}
std::vector<size_t> hyphenateWordWithHyphenator(const std::string& word, const LanguageHyphenator& hyphenator) { auto cps = collectCodepoints(word); trimSurroundingPunctuationAndFootnote(cps);
return hyphenator.breakIndexes(cps);}
std::vector<LanguageConfig> resolveLanguages(const std::string& selection) { if (selection == "all") { return kSupportedLanguages; }
for (const auto& config : kSupportedLanguages) { if (config.cliName == selection) { return {config}; } }
return {};}
EvaluationResult evaluateWord(const TestCase& testCase, std::function<std::vector<size_t>(const std::string&)> hyphenateFunc) { EvaluationResult result;
std::vector<size_t> actualPositions = hyphenateFunc(testCase.word);
std::vector<size_t> expected = testCase.expectedPositions; std::vector<size_t> actual = actualPositions;
std::sort(expected.begin(), expected.end()); std::sort(actual.begin(), actual.end());
for (size_t pos : actual) { if (std::find(expected.begin(), expected.end(), pos) != expected.end()) { result.truePositives++; } else { result.falsePositives++; } }
for (size_t pos : expected) { if (std::find(actual.begin(), actual.end(), pos) == actual.end()) { result.falseNegatives++; } }
if (result.truePositives + result.falsePositives > 0) { result.precision = static_cast<double>(result.truePositives) / (result.truePositives + result.falsePositives); }
if (result.truePositives + result.falseNegatives > 0) { result.recall = static_cast<double>(result.truePositives) / (result.truePositives + result.falseNegatives); }
if (result.precision + result.recall > 0) { result.f1Score = 2 * result.precision * result.recall / (result.precision + result.recall); }
// Treat words that contain no hyphenation marks in both the expected data and the // algorithmic output as perfect matches so they don't drag down the per-word averages. if (expected.empty() && actual.empty()) { result.precision = 1.0; result.recall = 1.0; result.f1Score = 1.0; }
double fpPenalty = 2.0; double fnPenalty = 1.0;
int totalErrors = result.falsePositives * fpPenalty + result.falseNegatives * fnPenalty; int totalPossible = expected.size() * fpPenalty;
if (totalPossible > 0) { result.weightedScore = 1.0 - (static_cast<double>(totalErrors) / totalPossible); result.weightedScore = std::max(0.0, result.weightedScore); } else if (result.falsePositives == 0) { result.weightedScore = 1.0; }
return result;}
void printResults(const std::string& language, const std::vector<TestCase>& testCases, const std::vector<std::pair<TestCase, EvaluationResult>>& worstCases, int perfectMatches, int partialMatches, int completeMisses, double totalPrecision, double totalRecall, double totalF1, double totalWeighted, int totalTP, int totalFP, int totalFN, std::function<std::vector<size_t>(const std::string&)> hyphenateFunc) { std::string lang_upper = language; if (!lang_upper.empty()) { lang_upper[0] = std::toupper(lang_upper[0]); }
std::cout << "================================================================================" << std::endl; std::cout << lang_upper << " HYPHENATION EVALUATION RESULTS" << std::endl; std::cout << "================================================================================" << std::endl; std::cout << std::endl;
std::cout << "Total test cases: " << testCases.size() << std::endl; std::cout << "Perfect matches: " << perfectMatches << " (" << (perfectMatches * 100.0 / testCases.size()) << "%)" << std::endl; std::cout << "Partial matches: " << partialMatches << std::endl; std::cout << "Complete misses: " << completeMisses << std::endl; std::cout << std::endl;
std::cout << "--- Overall Metrics (averaged per word) ---" << std::endl; std::cout << "Average Precision: " << (totalPrecision / testCases.size() * 100.0) << "%" << std::endl; std::cout << "Average Recall: " << (totalRecall / testCases.size() * 100.0) << "%" << std::endl; std::cout << "Average F1 Score: " << (totalF1 / testCases.size() * 100.0) << "%" << std::endl; std::cout << "Average Weighted Score: " << (totalWeighted / testCases.size() * 100.0) << "% (FP penalty: 2x)" << std::endl; std::cout << std::endl;
std::cout << "--- Overall Metrics (total counts) ---" << std::endl; std::cout << "True Positives: " << totalTP << std::endl; std::cout << "False Positives: " << totalFP << " (incorrect hyphenation points)" << std::endl; std::cout << "False Negatives: " << totalFN << " (missed hyphenation points)" << std::endl;
double overallPrecision = totalTP + totalFP > 0 ? static_cast<double>(totalTP) / (totalTP + totalFP) : 0.0; double overallRecall = totalTP + totalFN > 0 ? static_cast<double>(totalTP) / (totalTP + totalFN) : 0.0; double overallF1 = overallPrecision + overallRecall > 0 ? 2 * overallPrecision * overallRecall / (overallPrecision + overallRecall) : 0.0;
std::cout << "Overall Precision: " << (overallPrecision * 100.0) << "%" << std::endl; std::cout << "Overall Recall: " << (overallRecall * 100.0) << "%" << std::endl; std::cout << "Overall F1 Score: " << (overallF1 * 100.0) << "%" << std::endl; std::cout << std::endl;
// Filter out perfect matches from the “worst cases” section so that only actionable failures appear. auto hasImperfection = [](const EvaluationResult& r) { return r.weightedScore < 0.999999; }; std::vector<std::pair<TestCase, EvaluationResult>> imperfectCases; imperfectCases.reserve(worstCases.size()); for (const auto& entry : worstCases) { if (hasImperfection(entry.second)) { imperfectCases.push_back(entry); } }
std::cout << "--- Worst Cases (lowest weighted scores) ---" << std::endl; int showCount = std::min(10, static_cast<int>(imperfectCases.size())); for (int i = 0; i < showCount; i++) { const auto& testCase = imperfectCases[i].first; const auto& result = imperfectCases[i].second;
std::vector<size_t> actualPositions = hyphenateFunc(testCase.word); std::string actualHyphenated = positionsToHyphenated(testCase.word, actualPositions);
std::cout << "Word: " << testCase.word << " (freq: " << testCase.frequency << ")" << std::endl; std::cout << " Expected: " << testCase.hyphenated << std::endl; std::cout << " Got: " << actualHyphenated << std::endl; std::cout << " Precision: " << (result.precision * 100.0) << "%" << " Recall: " << (result.recall * 100.0) << "%" << " F1: " << (result.f1Score * 100.0) << "%" << " Weighted: " << (result.weightedScore * 100.0) << "%" << std::endl; std::cout << " TP: " << result.truePositives << " FP: " << result.falsePositives << " FN: " << result.falseNegatives << std::endl; std::cout << std::endl; }
// Additional compact list of the worst ~100 words to aid iteration int compactCount = std::min(100, static_cast<int>(imperfectCases.size())); if (compactCount > 0) { std::cout << "--- Compact Worst Cases (" << compactCount << ") ---" << std::endl; for (int i = 0; i < compactCount; i++) { const auto& testCase = imperfectCases[i].first; std::vector<size_t> actualPositions = hyphenateFunc(testCase.word); std::string actualHyphenated = positionsToHyphenated(testCase.word, actualPositions); std::cout << testCase.word << " | exp:" << testCase.hyphenated << " | got:" << actualHyphenated << std::endl; } std::cout << std::endl; }}
int main(int argc, char* argv[]) { const bool summaryMode = argc <= 1; const std::string languageSelection = summaryMode ? "all" : argv[1];
std::vector<LanguageConfig> languages = resolveLanguages(languageSelection); if (languages.empty()) { std::cerr << "Unknown language: " << languageSelection << std::endl; return 1; }
for (const auto& lang : languages) { const auto* hyphenator = getLanguageHyphenatorForPrimaryTag(lang.primaryTag); if (!hyphenator) { std::cerr << "No hyphenator registered for tag: " << lang.primaryTag << std::endl; continue; } const auto hyphenateFunc = [hyphenator](const std::string& word) { return hyphenateWordWithHyphenator(word, *hyphenator); };
if (!summaryMode) { std::cout << "Loading test data from: " << lang.testDataFile << std::endl; } std::vector<TestCase> testCases = loadTestData(lang.testDataFile);
if (testCases.empty()) { std::cerr << "No test cases loaded for " << lang.cliName << ". Skipping." << std::endl; continue; }
if (!summaryMode) { std::cout << "Loaded " << testCases.size() << " test cases for " << lang.cliName << std::endl; std::cout << std::endl; }
int perfectMatches = 0; int partialMatches = 0; int completeMisses = 0;
double totalPrecision = 0.0; double totalRecall = 0.0; double totalF1 = 0.0; double totalWeighted = 0.0;
int totalTP = 0, totalFP = 0, totalFN = 0;
std::vector<std::pair<TestCase, EvaluationResult>> worstCases;
for (const auto& testCase : testCases) { EvaluationResult result = evaluateWord(testCase, hyphenateFunc);
totalTP += result.truePositives; totalFP += result.falsePositives; totalFN += result.falseNegatives;
totalPrecision += result.precision; totalRecall += result.recall; totalF1 += result.f1Score; totalWeighted += result.weightedScore;
if (result.f1Score == 1.0) { perfectMatches++; } else if (result.f1Score > 0.0) { partialMatches++; } else { completeMisses++; }
worstCases.push_back({testCase, result}); }
if (summaryMode) { const double averageF1Percent = testCases.empty() ? 0.0 : (totalF1 / testCases.size() * 100.0); std::cout << lang.cliName << ": " << averageF1Percent << "%" << std::endl; continue; }
std::sort(worstCases.begin(), worstCases.end(), [](const auto& a, const auto& b) { return a.second.weightedScore < b.second.weightedScore; });
printResults(lang.cliName, testCases, worstCases, perfectMatches, partialMatches, completeMisses, totalPrecision, totalRecall, totalF1, totalWeighted, totalTP, totalFP, totalFN, hyphenateFunc); }
return 0;}