diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 7fdeab1..f02fc31 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -19,7 +19,7 @@ class KoborsUploadBase(InterfaceActionBase): description = "Send selected books to a kobors server" supported_platforms = ["windows", "osx", "linux"] author = "kobors" - version = (1, 0, 0) + version = (1, 1, 0) # Qt6-based (uses `qt.core` and scoped enums); calibre 6.0 is the first # release on Qt6. minimum_calibre_version = (6, 0, 0) -- 2.51.2 From 94070805331b00693f2110291f2f390d89cc1e8c Mon Sep 17 00:00:00 2001 From: servius Date: Tue, 28 Jul 2026 00:43:51 +0530 Subject: [PATCH 2/4] fix: dedup on the plugin's declared UUID so the pre-check actually matches MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Calibre's set_metadata leaves TWO identifiers in the uploaded EPUB's OPF: the book's original epub uuid (first) and Calibre's library uuid (appended). The server parsed the first, but the plugin's pre-check keys on db.field_for("uuid") = the library uuid, so they never matched — nothing was skipped and every book re-uploaded (the "too slow" report). Have the plugin send its library uuid as a `book_uuid` multipart field on /api/upload; the server stores that as source_uuid when present (else falls back to the OPF identifier, e.g. web-UI uploads). Now the stored value, the /api/upload/check value, and re-upload all use the same uuid the plugin knows, so already-synced books are skipped without transferring the file. Verified live: upload with book_uuid != OPF uuid -> stored/checked on the declared uuid; re-upload same uuid with different bytes -> 409, one row. --- calibre-plugin/__init__.py | 2 +- calibre-plugin/main.py | 22 +++++++++++---- src/upload/handler.rs | 58 +++++++++++++++++++++++++------------- 3 files changed, 57 insertions(+), 25 deletions(-) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index f02fc31..31446a0 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -19,7 +19,7 @@ class KoborsUploadBase(InterfaceActionBase): description = "Send selected books to a kobors server" supported_platforms = ["windows", "osx", "linux"] author = "kobors" - version = (1, 1, 0) + version = (1, 1, 1) # Qt6-based (uses `qt.core` and scoped enums); calibre 6.0 is the first # release on Qt6. minimum_calibre_version = (6, 0, 0) diff --git a/calibre-plugin/main.py b/calibre-plugin/main.py index 2736596..6690360 100644 --- a/calibre-plugin/main.py +++ b/calibre-plugin/main.py @@ -79,7 +79,9 @@ def run_upload(gui): continue filename, data = prepared - code, message = _post_book(server, token, filename, data) + code, message = _post_book( + server, token, filename, data, book_uuids[book_id] + ) if code == 200: uploaded += 1 elif code == 409: @@ -161,19 +163,29 @@ def _check_existing(server, token, uuids, timeout=30): return set() -def _post_book(server, token, filename, data, timeout=180): +def _post_book(server, token, filename, data, book_uuid="", timeout=180): """POST one book to ``/api/upload`` as multipart/form-data. - Returns ``(status_code, message)``; ``status_code`` is ``None`` on a - transport-level failure (with the error text in ``message``).""" + Sends the book's Calibre uuid as a ``book_uuid`` field so the server dedups + on the same identifier the pre-check uses (the EPUB's own OPF identifier + differs from Calibre's library uuid). Returns ``(status_code, message)``; + ``status_code`` is ``None`` on a transport-level failure (error in + ``message``).""" boundary = "----kobors" + uuid.uuid4().hex + uuid_part = b"" + if book_uuid: + uuid_part = ( + '--%s\r\n' + 'Content-Disposition: form-data; name="book_uuid"\r\n\r\n' + "%s\r\n" % (boundary, book_uuid) + ).encode("utf-8") preamble = ( '--%s\r\n' 'Content-Disposition: form-data; name="file"; filename="%s"\r\n' "Content-Type: application/epub+zip\r\n\r\n" % (boundary, filename) ).encode("utf-8") epilogue = ("\r\n--%s--\r\n" % boundary).encode("utf-8") - body = preamble + data + epilogue + body = uuid_part + preamble + data + epilogue req = urllib.request.Request(server + "/api/upload", data=body, method="POST") req.add_header("Content-Type", "multipart/form-data; boundary=" + boundary) diff --git a/src/upload/handler.rs b/src/upload/handler.rs index db36b40..2a36eb0 100644 --- a/src/upload/handler.rs +++ b/src/upload/handler.rs @@ -72,28 +72,46 @@ pub async fn api_upload( State(state): State, mut multipart: Multipart, ) -> Result, (StatusCode, String)> { - // Read the first file field from the multipart body. + // Read the `file` part and an optional `book_uuid` text part. The Calibre + // plugin sends `book_uuid` (its library uuid) so dedup keys off a value the + // plugin also knows — the EPUB's own OPF identifier differs from it, so + // parsing the file alone would give the plugin's pre-check nothing to match. let mut bytes: Option> = None; let mut filename = String::from("book.epub"); + let mut declared_uuid: Option = None; while let Some(field) = multipart .next_field() .await .map_err(|e| (StatusCode::BAD_REQUEST, format!("Malformed upload: {e}")))? { - let is_file = field.name() == Some("file") || field.file_name().is_some(); - if bytes.is_some() || !is_file { - continue; - } - if let Some(name) = field.file_name() { - filename = name.to_string(); + let name = field.name().map(str::to_string); + let is_file = name.as_deref() == Some("file") || field.file_name().is_some(); + if is_file { + if bytes.is_some() { + continue; + } + if let Some(fname) = field.file_name() { + filename = fname.to_string(); + } + let data = field.bytes().await.map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Failed to read upload: {e}"), + ) + })?; + bytes = Some(data.to_vec()); + } else if name.as_deref() == Some("book_uuid") { + let text = field.text().await.map_err(|e| { + ( + StatusCode::BAD_REQUEST, + format!("Failed to read field: {e}"), + ) + })?; + let normalized = text.trim().to_ascii_lowercase(); + if !normalized.is_empty() { + declared_uuid = Some(normalized); + } } - let data = field.bytes().await.map_err(|e| { - ( - StatusCode::BAD_REQUEST, - format!("Failed to read upload: {e}"), - ) - })?; - bytes = Some(data.to_vec()); } let bytes = bytes.ok_or((StatusCode::BAD_REQUEST, "No file uploaded".to_string()))?; @@ -136,10 +154,12 @@ pub async fn api_upload( (StatusCode::BAD_REQUEST, format!("Invalid EPUB: {e}")) })?; - // Dedupe by the EPUB's stable identifier. Calibre re-exports the same book - // with different bytes each send (re-zip, refreshed metadata), so the - // content hash above misses re-uploads; the `urn:uuid:` identifier does not. - if let Some(src_uuid) = parsed.meta.book_uuid.as_deref() { + // Stable dedup identifier: the plugin's declared `book_uuid` if present, + // else the EPUB's own OPF identifier. Calibre re-exports the same book with + // different bytes each send (re-zip, refreshed metadata), so the content + // hash above misses re-uploads; a stable UUID does not. + let source_uuid = declared_uuid.or_else(|| parsed.meta.book_uuid.clone()); + if let Some(src_uuid) = source_uuid.as_deref() { if let Some(existing) = state .books .find_by_source_uuid(user.id, src_uuid) @@ -213,7 +233,7 @@ pub async fn api_upload( locator: uuid.clone(), file_basename: "book".to_string(), content_hash: Some(hash), - source_uuid: meta.book_uuid.clone(), + source_uuid, has_cover: false, owner_user_id: Some(user.id), default_sync, -- 2.51.2 From 6b5b960394ddc07aa5f9c00f38fb7237a95b85ab Mon Sep 17 00:00:00 2001 From: servius Date: Tue, 28 Jul 2026 00:48:08 +0530 Subject: [PATCH 3/4] feat: make the plugin's upload report self-verifying MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit "Is the pre-check working?" was invisible: a run that skipped nothing looked identical whether the server recognized zero books or the /api/upload/check endpoint was missing entirely. The report now distinguishes: - Skipped (already on server, not transferred) — the pre-check working - Skipped (rejected as duplicate after upload) — the 409 backstop - Uploaded / total / elapsed time - An explicit warning when the pre-check endpoint did not answer (server not on the current build), which explains a slow, no-skip run instead of leaving it a mystery. _check_existing now returns (known, available) so the report can tell "server knows none of these" apart from "endpoint unavailable". --- calibre-plugin/__init__.py | 2 +- calibre-plugin/main.py | 79 ++++++++++++++++++++++++++++++-------- 2 files changed, 63 insertions(+), 18 deletions(-) diff --git a/calibre-plugin/__init__.py b/calibre-plugin/__init__.py index 31446a0..2ab7e41 100644 --- a/calibre-plugin/__init__.py +++ b/calibre-plugin/__init__.py @@ -19,7 +19,7 @@ class KoborsUploadBase(InterfaceActionBase): description = "Send selected books to a kobors server" supported_platforms = ["windows", "osx", "linux"] author = "kobors" - version = (1, 1, 1) + version = (1, 1, 2) # Qt6-based (uses `qt.core` and scoped enums); calibre 6.0 is the first # release on Qt6. minimum_calibre_version = (6, 0, 0) diff --git a/calibre-plugin/main.py b/calibre-plugin/main.py index 6690360..83c01c9 100644 --- a/calibre-plugin/main.py +++ b/calibre-plugin/main.py @@ -7,6 +7,7 @@ use the Python standard library only (no extra plugin dependencies). import io import json +import time import urllib.error import urllib.request import uuid @@ -47,18 +48,22 @@ def run_upload(gui): progress.setWindowModality(Qt.WindowModality.WindowModal) progress.setMinimumDuration(0) + started = time.monotonic() uploaded = 0 - skipped = 0 + presynced = 0 # skipped by the pre-check, without transferring the file + skipped = 0 # server rejected as a duplicate (409) after transfer no_format = [] failures = [] # list of (title, reason) # Ask the server which of these books it already has (matched by Calibre's - # per-book uuid, which is embedded in every EPUB's OPF). Skipping them here - # avoids uploading the whole file only to be 409'd: /api/upload can only - # reject a duplicate after receiving it. Best-effort — an older server that - # lacks the endpoint yields an empty set, so every book is still uploaded. + # per-book uuid). Skipping them here avoids uploading the whole file only to + # be 409'd: /api/upload can only reject a duplicate after receiving it. + # `precheck_ok` is False when the endpoint is missing (older server) or the + # request failed — the run then uploads every book and the report says so. book_uuids = {book_id: (db.field_for("uuid", book_id) or "") for book_id in ids} - known = _check_existing(server, token, [u for u in book_uuids.values() if u]) + known, precheck_ok = _check_existing( + server, token, [u for u in book_uuids.values() if u] + ) for i, book_id in enumerate(ids): if progress.wasCanceled(): @@ -70,7 +75,7 @@ def run_upload(gui): if book_uuids[book_id].strip().lower() in known: # Already on the server — do not read or transfer the file. - skipped += 1 + presynced += 1 continue prepared = _prepare_book(db, book_id) @@ -104,7 +109,18 @@ def run_upload(gui): progress.setValue(len(ids)) - _report(gui, uploaded, skipped, no_format, failures) + elapsed = time.monotonic() - started + _report( + gui, + uploaded, + presynced, + skipped, + no_format, + failures, + precheck_ok, + len(ids), + elapsed, + ) def _prepare_book(db, book_id): @@ -142,13 +158,15 @@ def _embed_metadata(db, book_id, raw, fmt): def _check_existing(server, token, uuids, timeout=30): - """Return the lowercased set of book UUIDs the server already has. + """Return ``(known, available)``: the lowercased set of book UUIDs the + server already has, and whether the pre-check endpoint actually answered. - Best-effort: any failure (older server without the endpoint, network error) - returns an empty set, so the caller falls back to uploading everything and - relying on the server's per-book 409 dedup.""" + ``available`` is False when the endpoint is missing (older server) or the + request failed; the caller then uploads everything and relies on the + server's per-book 409 dedup, and the report flags that the pre-check was + unavailable so a slow, no-skip run is explainable rather than mysterious.""" if not uuids: - return set() + return set(), True body = json.dumps({"uuids": uuids}).encode("utf-8") req = urllib.request.Request( server + "/api/upload/check", data=body, method="POST" @@ -158,9 +176,9 @@ def _check_existing(server, token, uuids, timeout=30): try: with urllib.request.urlopen(req, timeout=timeout) as resp: payload = json.loads(resp.read().decode("utf-8")) - return {str(u).strip().lower() for u in payload.get("known", [])} + return {str(u).strip().lower() for u in payload.get("known", [])}, True except Exception: - return set() + return set(), False def _post_book(server, token, filename, data, book_uuid="", timeout=180): @@ -212,15 +230,42 @@ def _safe_filename(title): return (cleaned or "book")[:120] -def _report(gui, uploaded, skipped, no_format, failures): +def _report( + gui, + uploaded, + presynced, + skipped, + no_format, + failures, + precheck_ok, + total, + elapsed, +): lines = [ + "Selected: %d" % total, "Uploaded: %d" % uploaded, - "Already on server (skipped): %d" % skipped, + "Skipped (already on server, not transferred): %d" % presynced, ] + if skipped: + lines.append("Skipped (rejected as duplicate after upload): %d" % skipped) if no_format: lines.append("No EPUB/KEPUB format: %d" % len(no_format)) if failures: lines.append("Failed: %d" % len(failures)) + lines.append("Time: %.1fs" % elapsed) + if not precheck_ok: + lines.append("") + lines.append( + "⚠ Pre-check unavailable — the server did not answer " + "/api/upload/check (is it running the current build?). Every book " + "was uploaded; duplicates were still rejected server-side." + ) + elif presynced == 0 and uploaded > 0: + lines.append("") + lines.append( + "Note: the server recognized none of these books, so all were " + "uploaded. On the next run these should skip." + ) summary = "\n".join(lines) details = [] -- 2.51.2 From ce50943e513318b2d377fba67b021943e658b508 Mon Sep 17 00:00:00 2001 From: servius Date: Tue, 28 Jul 2026 00:54:36 +0530 Subject: [PATCH 4/4] fix: self-heal source_uuid on duplicate so stuck books skip next run MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Books first uploaded under a different identifier (old plugin with no book_uuid, so the server stored the EPUB's OPF uuid) were invisible to the pre-check, which keys on Calibre's library uuid: every run re-uploaded them and the 409 backstop fired only after the full transfer. Nothing ever taught the server the uuid the pre-check uses, so it never converged. On any duplicate detection, stamp the plugin's declared book_uuid onto the matched row (BookStore::set_source_uuid, identity-only — no last_modified bump). Also match a re-upload against the declared uuid OR the OPF uuid, so a book stored under either is recognized even when the bytes differ (no new duplicate row). After one more reconciling run, the pre-check recognizes the book and skips it without transferring. Verified live: old-style upload -> pre-check misses -> re-upload with book_uuid 409s and reconciles -> pre-check now hits; identical-bytes and differing-bytes paths both heal with no duplicate created. --- src/db/book_store.rs | 17 +++++++++++++++++ src/upload/handler.rs | 39 +++++++++++++++++++++++++++++++-------- 2 files changed, 48 insertions(+), 8 deletions(-) diff --git a/src/db/book_store.rs b/src/db/book_store.rs index 662ef3e..3e4ff16 100644 --- a/src/db/book_store.rs +++ b/src/db/book_store.rs @@ -631,6 +631,23 @@ impl BookStore { Ok(row.map(|r| r.get::("id"))) } + /// Point an existing book's `source_uuid` at `uuid`. Used to reconcile a row + /// that was first stored under a different identifier (an old-plugin upload + /// keyed on the EPUB's OPF uuid, say) to the Calibre library uuid the plugin + /// pre-check queries, so the book skips on the next run instead of always + /// re-uploading. Identity-only: it deliberately does not bump `last_modified` + /// (no device re-sync). + pub async fn set_source_uuid(&self, book_id: i64, uuid: &str) -> Result<()> { + sqlx::query("UPDATE books SET source_uuid = ? WHERE id = ?") + .bind(uuid) + .bind(book_id) + .execute(&self.pool) + .await + .change_context(Error::Database) + .attach("Failed to update source_uuid")?; + Ok(()) + } + /// Every source UUID this owner has already uploaded. Lets the Calibre /// plugin pre-check a batch of books and skip re-sending the ones already /// present, rather than POSTing each full file only to receive a 409. The diff --git a/src/upload/handler.rs b/src/upload/handler.rs index 2a36eb0..8301d3e 100644 --- a/src/upload/handler.rs +++ b/src/upload/handler.rs @@ -124,7 +124,9 @@ pub async fn api_upload( )); } - // Dedupe by content hash for this owner. + // Dedupe by content hash for this owner (fast path for byte-identical + // re-sends). Reconcile the matched row's source_uuid to the plugin's + // declared uuid so the pre-check can skip it next time. let hash = sha256_hex(&bytes); if let Some(existing) = state .books @@ -132,6 +134,13 @@ pub async fn api_upload( .await .map_err(internal)? { + if let Some(decl) = declared_uuid.as_deref() { + state + .books + .set_source_uuid(existing, decl) + .await + .map_err(internal)?; + } return Err(( StatusCode::CONFLICT, format!("This book was already uploaded (id {existing})"), @@ -154,24 +163,38 @@ pub async fn api_upload( (StatusCode::BAD_REQUEST, format!("Invalid EPUB: {e}")) })?; - // Stable dedup identifier: the plugin's declared `book_uuid` if present, - // else the EPUB's own OPF identifier. Calibre re-exports the same book with - // different bytes each send (re-zip, refreshed metadata), so the content - // hash above misses re-uploads; a stable UUID does not. - let source_uuid = declared_uuid.or_else(|| parsed.meta.book_uuid.clone()); - if let Some(src_uuid) = source_uuid.as_deref() { + // Stable-UUID dedup. Calibre re-exports the same book with different bytes + // each send (re-zip, refreshed metadata), so the content hash above misses + // re-uploads; a stable UUID does not. Match on the plugin's declared uuid or + // the EPUB's own OPF uuid, so a book first stored under either identifier is + // still recognized, and reconcile the matched row to the declared uuid (the + // value the pre-check queries) so it skips next time. + let opf_uuid = parsed.meta.book_uuid.clone(); + for candidate in [declared_uuid.as_deref(), opf_uuid.as_deref()] + .into_iter() + .flatten() + { if let Some(existing) = state .books - .find_by_source_uuid(user.id, src_uuid) + .find_by_source_uuid(user.id, candidate) .await .map_err(internal)? { + if let Some(decl) = declared_uuid.as_deref() { + state + .books + .set_source_uuid(existing, decl) + .await + .map_err(internal)?; + } return Err(( StatusCode::CONFLICT, format!("This book was already uploaded (id {existing})"), )); } } + // Store the declared (canonical) uuid when present, else the OPF uuid. + let source_uuid = declared_uuid.or(opf_uuid); // Persist the file under upload_dir/{uuid}/ before responding. The heavy, // multi-second work (online enrichment, cover transcode, kepubify) is