This repository has no description
Something went wrong. Try again.
1234567891011121314151617181920212223242526272829303132333435363738394041424344454647484950515253545556575859diff --git a/tools/server/server-task.cpp b/tools/server/server-task.cppindex ad6565c7..e67050a9 100644--- a/tools/server/server-task.cpp+++ b/tools/server/server-task.cpp@@ -1749,7 +1749,8 @@ constexpr size_t META_OFFSET = 16; constexpr size_t META_SIZE = 512; constexpr size_t META_FIXED = 74; -constexpr size_t FAMILY_TOKENS = 256;+constexpr size_t FAMILY_TOKENS = 256; // cap on the system block that is hashed+constexpr size_t FAMILY_USER_TOKENS = 32; // without a system block: the start of the first user turn constexpr size_t FAMILY_MIN_N = 8; constexpr size_t FAMILIES_MAX = 500; constexpr size_t CLASSIFY_HEAD = 1500;@@ -2503,6 +2504,19 @@ void server_prompt_cache::policy_init(const cache_policy & policy, std::function this->post_maint = std::move(post_maint); } +// a family is the service that sent the prompt: its system block, which stays the same across calls while the+// user turn carries the per-call query or page; without one, the opening of the first user turn+uint64_t server_prompt_cache::family_of(const llama_tokens & tokens) const {+ auto it = user_marker.empty() ? tokens.end()+ : std::search(tokens.begin(), tokens.end(), user_marker.begin(), user_marker.end());+ size_t n = it == tokens.end() ? tokens.size() : (size_t) (it - tokens.begin());+ if (n == 0) {+ n = user_marker.size() + FAMILY_USER_TOKENS;+ }+ n = std::min({ n, tokens.size(), FAMILY_TOKENS });+ return fnv1a(tokens.data(), n * sizeof(llama_token));+}+ void server_prompt_cache::policy_start() { policy_thread = std::thread([this] { policy_loop(); }); }@@ -2588,8 +2602,7 @@ void server_prompt_cache::note_saved(server_prompt_cache_state & state) { } if (m.family == 0) {- const size_t n = std::min(tokens.size(), FAMILY_TOKENS);- m.family = fnv1a(tokens.data(), n * sizeof(llama_token));+ m.family = family_of(tokens); } auto & fam = families[m.family]; if (!m.family_saved) {diff --git a/tools/server/server-task.h b/tools/server/server-task.hindex fcf7ae34..c4c37df9 100644--- a/tools/server/server-task.h+++ b/tools/server/server-task.h@@ -815,6 +815,9 @@ private: llama_tokens user_marker; std::function<void()> post_maint; + // hash of the prompt's system block (or, without one, the start of the first user turn)+ uint64_t family_of(const llama_tokens & tokens) const;+ // server loop only std::map<uint64_t, family_stat> families; bool families_dirty = false;