Something went wrong. Try again.
A fork of https://github.com/crosspoint-reader/crosspoint-reader
Something went wrong. Try again.
8.3 kB · 235 lines
C++
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236#include <Utf8.h>#include <gtest/gtest.h>
#include <algorithm>#include <fstream>#include <iostream>#include <sstream>#include <string>#include <vector>
#include "lib/Epub/Epub/hyphenation/HyphenationCommon.h"#include "lib/Epub/Epub/hyphenation/LanguageHyphenator.h"#include "lib/Epub/Epub/hyphenation/LanguageRegistry.h"
#ifndef HYPHENATION_RESOURCES_DIR#error "HYPHENATION_RESOURCES_DIR must be defined by the build system"#endif
namespace {
struct TestCase { std::string word; std::string hyphenated; std::vector<size_t> expectedPositions; int frequency;};
struct EvaluationResult { int truePositives = 0; int falsePositives = 0; int falseNegatives = 0; double precision = 0.0; double recall = 0.0; double f1Score = 0.0; double weightedScore = 0.0;};
std::vector<size_t> expectedPositionsFromAnnotatedWord(const std::string& annotated) { std::vector<size_t> positions; const unsigned char* ptr = reinterpret_cast<const unsigned char*>(annotated.c_str()); size_t codepointIndex = 0;
while (*ptr != 0) { if (*ptr == '=') { positions.push_back(codepointIndex); ++ptr; continue; }
utf8NextCodepoint(&ptr); ++codepointIndex; }
return positions;}
std::vector<TestCase> loadTestData(const std::string& filename) { std::vector<TestCase> testCases; std::ifstream file(filename);
if (!file.is_open()) { return testCases; }
std::string line; while (std::getline(file, line)) { if (line.empty() || line[0] == '#') { continue; }
std::istringstream iss(line); std::string word, hyphenated, freqStr;
if (std::getline(iss, word, '|') && std::getline(iss, hyphenated, '|') && std::getline(iss, freqStr, '|')) { TestCase testCase; testCase.word = word; testCase.hyphenated = hyphenated; testCase.frequency = std::stoi(freqStr); testCase.expectedPositions = expectedPositionsFromAnnotatedWord(hyphenated); testCases.push_back(testCase); } }
return testCases;}
std::string positionsToHyphenated(const std::string& word, const std::vector<size_t>& positions) { std::string result; std::vector<size_t> sortedPositions = positions; std::sort(sortedPositions.begin(), sortedPositions.end());
const unsigned char* ptr = reinterpret_cast<const unsigned char*>(word.c_str()); size_t codepointIndex = 0; size_t posIdx = 0;
while (*ptr != 0) { while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) { result.push_back('='); ++posIdx; }
const unsigned char* current = ptr; utf8NextCodepoint(&ptr); result.append(reinterpret_cast<const char*>(current), reinterpret_cast<const char*>(ptr)); ++codepointIndex; }
while (posIdx < sortedPositions.size() && sortedPositions[posIdx] == codepointIndex) { result.push_back('='); ++posIdx; }
return result;}
std::vector<size_t> hyphenateWordWithHyphenator(const std::string& word, const LanguageHyphenator& hyphenator) { auto cps = collectCodepoints(word); trimSurroundingPunctuationAndFootnote(cps); return hyphenator.breakIndexes(cps);}
EvaluationResult evaluateWord(const TestCase& testCase, const std::vector<size_t>& actualPositions) { EvaluationResult result;
std::vector<size_t> expected = testCase.expectedPositions; std::vector<size_t> actual = actualPositions;
std::sort(expected.begin(), expected.end()); std::sort(actual.begin(), actual.end());
for (size_t pos : actual) { if (std::find(expected.begin(), expected.end(), pos) != expected.end()) { result.truePositives++; } else { result.falsePositives++; } }
for (size_t pos : expected) { if (std::find(actual.begin(), actual.end(), pos) == actual.end()) { result.falseNegatives++; } }
if (result.truePositives + result.falsePositives > 0) { result.precision = static_cast<double>(result.truePositives) / (result.truePositives + result.falsePositives); }
if (result.truePositives + result.falseNegatives > 0) { result.recall = static_cast<double>(result.truePositives) / (result.truePositives + result.falseNegatives); }
if (result.precision + result.recall > 0) { result.f1Score = 2 * result.precision * result.recall / (result.precision + result.recall); }
// Treat words with no expected and no actual hyphenation marks as perfect. if (expected.empty() && actual.empty()) { result.precision = 1.0; result.recall = 1.0; result.f1Score = 1.0; }
double fpPenalty = 2.0; double fnPenalty = 1.0; int totalErrors = result.falsePositives * fpPenalty + result.falseNegatives * fnPenalty; int totalPossible = static_cast<int>(expected.size() * fpPenalty);
if (totalPossible > 0) { result.weightedScore = 1.0 - (static_cast<double>(totalErrors) / totalPossible); result.weightedScore = std::max(0.0, result.weightedScore); } else if (result.falsePositives == 0) { result.weightedScore = 1.0; }
return result;}
// Runs the evaluation for a single language and asserts the per-word average F1// is at or above `minF1Percent`. Thresholds are set ~1pp below measured// baselines so unrelated tweaks don't fail CI but real regressions still trip.void runLanguageEval(const char* langName, const char* primaryTag, const char* resourceFile, double minF1Percent) { const auto* hyphenator = getLanguageHyphenatorForPrimaryTag(primaryTag); ASSERT_NE(hyphenator, nullptr) << "No hyphenator registered for tag: " << primaryTag;
std::string path = std::string(HYPHENATION_RESOURCES_DIR) + "/" + resourceFile; std::vector<TestCase> testCases = loadTestData(path); ASSERT_FALSE(testCases.empty()) << "No test cases loaded from " << path;
double totalF1 = 0.0; std::vector<std::pair<TestCase, EvaluationResult>> imperfect;
for (const auto& tc : testCases) { std::vector<size_t> actual = hyphenateWordWithHyphenator(tc.word, *hyphenator); EvaluationResult res = evaluateWord(tc, actual); totalF1 += res.f1Score; if (res.weightedScore < 0.999999) { imperfect.emplace_back(tc, res); } }
double averageF1Percent = totalF1 / testCases.size() * 100.0; ::testing::Test::RecordProperty("avg_f1_percent", std::to_string(averageF1Percent)); ::testing::Test::RecordProperty("test_cases", std::to_string(testCases.size()));
std::cout << langName << ": F1=" << averageF1Percent << "% (threshold " << minF1Percent << "%, " << testCases.size() << " cases)\n";
if (averageF1Percent < minF1Percent) { std::sort(imperfect.begin(), imperfect.end(), [](const auto& a, const auto& b) { return a.second.weightedScore < b.second.weightedScore; }); std::cout << "Worst cases for " << langName << ":\n"; int show = std::min<int>(10, static_cast<int>(imperfect.size())); for (int i = 0; i < show; ++i) { const TestCase& tc = imperfect[i].first; std::vector<size_t> actual = hyphenateWordWithHyphenator(tc.word, *hyphenator); std::cout << " " << tc.word << " | expected=" << tc.hyphenated << " | got=" << positionsToHyphenated(tc.word, actual) << "\n"; } }
EXPECT_GE(averageF1Percent, minF1Percent) << "Hyphenation quality regressed for " << langName;}
} // namespace
TEST(HyphenationEval, English) { runLanguageEval("english", "en", "english_hyphenation_tests.txt", 98.10); }TEST(HyphenationEval, French) { runLanguageEval("french", "fr", "french_hyphenation_tests.txt", 99.00); }TEST(HyphenationEval, German) { runLanguageEval("german", "de", "german_hyphenation_tests.txt", 96.73); }TEST(HyphenationEval, Russian) { runLanguageEval("russian", "ru", "russian_hyphenation_tests.txt", 96.22); }TEST(HyphenationEval, Spanish) { runLanguageEval("spanish", "es", "spanish_hyphenation_tests.txt", 98.02); }TEST(HyphenationEval, Italian) { runLanguageEval("italian", "it", "italian_hyphenation_tests.txt", 98.99); }TEST(HyphenationEval, Polish) { runLanguageEval("polish", "pl", "polish_hyphenation_tests.txt", 98.92); }TEST(HyphenationEval, Swedish) { runLanguageEval("swedish", "sv", "swedish_hyphenation_tests.txt", 94.01); }TEST(HyphenationEval, Portuguese) { runLanguageEval("portuguese", "pt", "portuguese_hyphenation_tests.txt", 98.21); }