diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index ca6d7eda..5e4de2dd 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -2,7 +2,11 @@ package management import ( "bufio" + "crypto/sha256" + "encoding/base64" + "encoding/json" "fmt" + "io" "math" "net/http" "os" @@ -20,6 +24,8 @@ const ( defaultLogFileName = "main.log" logScannerInitialBuffer = 64 * 1024 logScannerMaxBuffer = 8 * 1024 * 1024 + logCursorVersion = 1 + logCursorFingerprintMax = 4 * 1024 ) // GetLogs returns log lines with optional incremental loading. @@ -475,6 +481,299 @@ func (acc *logAccumulator) result() ([]string, int, int64) { return acc.lines, acc.total, acc.latest } +type logCursor struct { + Version int `json:"v"` + File string `json:"file"` + Offset int64 `json:"offset"` + Size int64 `json:"size"` + ModTime int64 `json:"modTime"` + LatestTimestamp int64 `json:"latestTimestamp"` + Fingerprint string `json:"fingerprint"` +} + +type completeLogRead struct { + lines []string + endOffset int64 + latest int64 + hitLimit bool +} + +func encodeLogCursor(cursor logCursor) (string, error) { + raw, err := json.Marshal(cursor) + if err != nil { + return "", err + } + return base64.RawURLEncoding.EncodeToString(raw), nil +} + +func decodeLogCursor(raw string) (logCursor, error) { + value := strings.TrimSpace(raw) + if value == "" { + return logCursor{}, fmt.Errorf("empty cursor") + } + data, err := base64.RawURLEncoding.DecodeString(value) + if err != nil { + data, err = base64.URLEncoding.DecodeString(value) + } + if err != nil { + return logCursor{}, fmt.Errorf("invalid cursor encoding") + } + var cursor logCursor + if errUnmarshal := json.Unmarshal(data, &cursor); errUnmarshal != nil { + return logCursor{}, fmt.Errorf("invalid cursor payload") + } + if errValidate := validateLogCursor(cursor); errValidate != nil { + return logCursor{}, errValidate + } + return cursor, nil +} + +func validateLogCursor(cursor logCursor) error { + if cursor.Version != logCursorVersion { + return fmt.Errorf("unsupported cursor version") + } + if !isAllowedLogCursorFile(cursor.File) { + return fmt.Errorf("invalid cursor file") + } + if cursor.Offset < 0 || cursor.Size < 0 || cursor.ModTime < 0 || cursor.LatestTimestamp < 0 { + return fmt.Errorf("invalid cursor position") + } + if strings.TrimSpace(cursor.Fingerprint) == "" { + return fmt.Errorf("invalid cursor fingerprint") + } + return nil +} + +func isAllowedLogCursorFile(name string) bool { + if name == "" || name == "." || name == ".." { + return false + } + if strings.ContainsAny(name, `/\`) { + return false + } + if filepath.Base(name) != name { + return false + } + return name == defaultLogFileName || isRotatedLogFile(name) +} + +func safeLogFilePath(logDir, name string) (string, error) { + if !isAllowedLogCursorFile(name) { + return "", fmt.Errorf("invalid log file") + } + dirAbs, errAbs := filepath.Abs(logDir) + if errAbs != nil { + return "", fmt.Errorf("resolve log directory: %w", errAbs) + } + dirAbs = filepath.Clean(dirAbs) + fullPath := filepath.Clean(filepath.Join(dirAbs, name)) + rel, errRel := filepath.Rel(dirAbs, fullPath) + if errRel != nil { + return "", fmt.Errorf("resolve log file: %w", errRel) + } + if rel == "." || strings.HasPrefix(rel, ".."+string(os.PathSeparator)) || rel == ".." || filepath.IsAbs(rel) { + return "", fmt.Errorf("invalid log file") + } + return fullPath, nil +} + +func newLogCursor(path string, offset, latest int64) (string, error) { + info, errStat := os.Stat(path) + if errStat != nil { + return "", errStat + } + if info.IsDir() { + return "", fmt.Errorf("invalid log file") + } + if offset < 0 || offset > info.Size() { + return "", fmt.Errorf("invalid cursor offset") + } + fingerprint, errFingerprint := logFileFingerprint(path, offset) + if errFingerprint != nil { + return "", errFingerprint + } + return encodeLogCursor(logCursor{ + Version: logCursorVersion, + File: filepath.Base(path), + Offset: offset, + Size: info.Size(), + ModTime: info.ModTime().Unix(), + LatestTimestamp: latest, + Fingerprint: fingerprint, + }) +} + +func logFileFingerprint(path string, boundary int64) (string, error) { + if boundary < 0 { + return "", fmt.Errorf("invalid fingerprint boundary") + } + file, errOpen := os.Open(path) + if errOpen != nil { + return "", errOpen + } + defer func() { + _ = file.Close() + }() + info, errStat := file.Stat() + if errStat != nil { + return "", errStat + } + if info.IsDir() { + return "", fmt.Errorf("invalid log file") + } + if boundary > info.Size() { + return "", fmt.Errorf("invalid fingerprint boundary") + } + + hash := sha256.New() + _, _ = fmt.Fprintf(hash, "log-cursor-v1:%d:", boundary) + firstLen := minInt64(boundary, logCursorFingerprintMax) + if errRead := writeFileRange(hash, file, 0, firstLen); errRead != nil { + return "", errRead + } + tailLen := minInt64(boundary, logCursorFingerprintMax) + tailStart := boundary - tailLen + _, _ = fmt.Fprintf(hash, ":%d:", tailStart) + if errRead := writeFileRange(hash, file, tailStart, tailLen); errRead != nil { + return "", errRead + } + sum := hash.Sum(nil) + return base64.RawURLEncoding.EncodeToString(sum[:12]), nil +} + +func writeFileRange(dst io.Writer, file *os.File, start, length int64) error { + if length <= 0 { + return nil + } + buf := make([]byte, 32*1024) + pos := start + remaining := length + for remaining > 0 { + chunk := minInt64(int64(len(buf)), remaining) + n, errRead := file.ReadAt(buf[:chunk], pos) + if n > 0 { + if _, errWrite := dst.Write(buf[:n]); errWrite != nil { + return errWrite + } + pos += int64(n) + remaining -= int64(n) + } + if errRead != nil { + if errRead == io.EOF && remaining == 0 { + return nil + } + return errRead + } + } + return nil +} + +func readCompleteLogLines(path string, offset, maxOffset int64, limit int) (completeLogRead, error) { + if offset < 0 { + return completeLogRead{}, fmt.Errorf("invalid log offset") + } + file, errOpen := os.Open(path) + if errOpen != nil { + return completeLogRead{}, errOpen + } + defer func() { + _ = file.Close() + }() + info, errStat := file.Stat() + if errStat != nil { + return completeLogRead{}, errStat + } + if info.IsDir() { + return completeLogRead{}, fmt.Errorf("invalid log file") + } + size := info.Size() + if maxOffset < 0 || maxOffset > size { + maxOffset = size + } + if offset > maxOffset { + return completeLogRead{}, fmt.Errorf("invalid log offset") + } + + reader := bufio.NewReader(io.NewSectionReader(file, offset, maxOffset-offset)) + result := completeLogRead{ + lines: []string{}, + endOffset: offset, + } + currentOffset := offset + for { + raw, errRead := reader.ReadString('\n') + if strings.HasSuffix(raw, "\n") { + currentOffset += int64(len(raw)) + line := strings.TrimSuffix(raw, "\n") + line = strings.TrimRight(line, "\r") + result.lines = append(result.lines, line) + result.endOffset = currentOffset + if ts := parseTimestamp(line); ts > result.latest { + result.latest = ts + } + if limit > 0 && len(result.lines) >= limit { + result.hitLimit = true + break + } + if errRead == nil { + continue + } + } + if errRead == io.EOF { + break + } + if errRead != nil { + return completeLogRead{}, errRead + } + } + return result, nil +} + +func completeLogBoundary(path string) (int64, error) { + file, errOpen := os.Open(path) + if errOpen != nil { + return 0, errOpen + } + defer func() { + _ = file.Close() + }() + info, errStat := file.Stat() + if errStat != nil { + return 0, errStat + } + if info.IsDir() { + return 0, fmt.Errorf("invalid log file") + } + size := info.Size() + if size == 0 { + return 0, nil + } + buf := make([]byte, 32*1024) + pos := size + for pos > 0 { + chunk := minInt64(int64(len(buf)), pos) + pos -= chunk + n, errRead := file.ReadAt(buf[:chunk], pos) + if errRead != nil && errRead != io.EOF { + return 0, errRead + } + if n <= 0 { + continue + } + if idx := strings.LastIndexByte(string(buf[:n]), '\n'); idx >= 0 { + return pos + int64(idx) + 1, nil + } + } + return 0, nil +} + +func minInt64(a, b int64) int64 { + if a < b { + return a + } + return b +} + func parseCutoff(raw string) int64 { value := strings.TrimSpace(raw) if value == "" { diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go new file mode 100644 index 00000000..4f15f706 --- /dev/null +++ b/internal/api/handlers/management/logs_test.go @@ -0,0 +1,136 @@ +package management + +import ( + "encoding/base64" + "encoding/json" + "os" + "path/filepath" + "reflect" + "strings" + "testing" +) + +func TestDecodeLogCursorRejectsUnsafeFiles(t *testing.T) { + unsafeNames := []string{ + "", + ".", + "..", + "../secret", + "nested/main.log", + `nested\main.log`, + "error.log", + } + + for _, name := range unsafeNames { + t.Run(name, func(t *testing.T) { + raw := mustEncodeRawCursor(t, logCursor{ + Version: logCursorVersion, + File: name, + Fingerprint: "fingerprint", + }) + if _, err := decodeLogCursor(raw); err == nil { + t.Fatalf("decodeLogCursor(%q) succeeded, want error", name) + } + }) + } + + for _, name := range []string{defaultLogFileName, defaultLogFileName + ".1", "main-2026-06-15T10-00-00.log"} { + t.Run("allowed_"+name, func(t *testing.T) { + raw := mustEncodeRawCursor(t, logCursor{ + Version: logCursorVersion, + File: name, + Fingerprint: "fingerprint", + }) + if _, err := decodeLogCursor(raw); err != nil { + t.Fatalf("decodeLogCursor(%q) error = %v", name, err) + } + }) + } +} + +func TestLogCursorRoundTripOmitsAbsolutePath(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, defaultLogFileName) + if err := os.WriteFile(path, []byte("line one\nline two\n"), 0o644); err != nil { + t.Fatalf("write log file: %v", err) + } + + boundary, errBoundary := completeLogBoundary(path) + if errBoundary != nil { + t.Fatalf("completeLogBoundary() error = %v", errBoundary) + } + raw, errCursor := newLogCursor(path, boundary, 123) + if errCursor != nil { + t.Fatalf("newLogCursor() error = %v", errCursor) + } + decoded, errDecode := decodeLogCursor(raw) + if errDecode != nil { + t.Fatalf("decodeLogCursor() error = %v", errDecode) + } + if decoded.File != defaultLogFileName { + t.Fatalf("cursor file = %q, want %q", decoded.File, defaultLogFileName) + } + if decoded.Offset != boundary { + t.Fatalf("cursor offset = %d, want %d", decoded.Offset, boundary) + } + if decoded.LatestTimestamp != 123 { + t.Fatalf("cursor latest timestamp = %d, want 123", decoded.LatestTimestamp) + } + if strings.Contains(raw, dir) { + t.Fatalf("encoded cursor contains log directory %q: %q", dir, raw) + } +} + +func TestReadCompleteLogLinesSkipsTrailingPartial(t *testing.T) { + dir := t.TempDir() + path := filepath.Join(dir, defaultLogFileName) + initial := "first\nsecond\r\npartial" + if err := os.WriteFile(path, []byte(initial), 0o644); err != nil { + t.Fatalf("write log file: %v", err) + } + + read, errRead := readCompleteLogLines(path, 0, -1, 0) + if errRead != nil { + t.Fatalf("readCompleteLogLines() error = %v", errRead) + } + wantLines := []string{"first", "second"} + if !reflect.DeepEqual(read.lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", read.lines, wantLines) + } + wantOffset := int64(len("first\nsecond\r\n")) + if read.endOffset != wantOffset { + t.Fatalf("endOffset = %d, want %d", read.endOffset, wantOffset) + } + + file, errOpen := os.OpenFile(path, os.O_APPEND|os.O_WRONLY, 0) + if errOpen != nil { + t.Fatalf("open log file: %v", errOpen) + } + if _, errWrite := file.WriteString("\n"); errWrite != nil { + _ = file.Close() + t.Fatalf("append newline: %v", errWrite) + } + if errClose := file.Close(); errClose != nil { + t.Fatalf("close log file: %v", errClose) + } + + next, errNext := readCompleteLogLines(path, read.endOffset, -1, 0) + if errNext != nil { + t.Fatalf("readCompleteLogLines() after append error = %v", errNext) + } + if !reflect.DeepEqual(next.lines, []string{"partial"}) { + t.Fatalf("next lines = %#v, want partial", next.lines) + } + if next.endOffset != int64(len(initial)+1) { + t.Fatalf("next endOffset = %d, want %d", next.endOffset, len(initial)+1) + } +} + +func mustEncodeRawCursor(t *testing.T, cursor logCursor) string { + t.Helper() + raw, err := json.Marshal(cursor) + if err != nil { + t.Fatalf("json.Marshal cursor: %v", err) + } + return base64.RawURLEncoding.EncodeToString(raw) +} -- 2.51.2 From 95a72a47c8e7f73b58d5b3451ced7b4be9cd977b Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 02:08:06 +0800 Subject: [PATCH 02/10] Tail management logs with cursors --- internal/api/handlers/management/logs.go | 167 ++++++++++++++++-- internal/api/handlers/management/logs_test.go | 122 +++++++++++++ 2 files changed, 277 insertions(+), 12 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index 5e4de2dd..fa3b6cb0 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -2,9 +2,11 @@ package management import ( "bufio" + "bytes" "crypto/sha256" "encoding/base64" "encoding/json" + "errors" "fmt" "io" "math" @@ -53,11 +55,7 @@ func (h *Handler) GetLogs(c *gin.Context) { if err != nil { if os.IsNotExist(err) { cutoff := parseCutoff(c.Query("after")) - c.JSON(http.StatusOK, gin.H{ - "lines": []string{}, - "line-count": 0, - "latest-timestamp": cutoff, - }) + writeLogsResponse(c, []string{}, 0, cutoff, "", false) return } c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to list log files: %v", err)}) @@ -71,10 +69,20 @@ func (h *Handler) GetLogs(c *gin.Context) { } cutoff := parseCutoff(c.Query("after")) + if strings.TrimSpace(c.Query("cursor")) == "" && cutoff == 0 && limit > 0 { + result, errTail := tailLogFiles(files, limit, 0) + if errTail != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errTail)}) + return + } + writeLogsResponse(c, result.lines, len(result.lines), result.latest, result.nextCursor, false) + return + } + acc := newLogAccumulator(cutoff, limit) for i := range files { if errProcess := acc.consumeFile(files[i]); errProcess != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log file %s: %v", files[i], errProcess)}) + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log file: %v", errProcess)}) return } } @@ -83,11 +91,12 @@ func (h *Handler) GetLogs(c *gin.Context) { if latest == 0 || latest < cutoff { latest = cutoff } - c.JSON(http.StatusOK, gin.H{ - "lines": lines, - "line-count": total, - "latest-timestamp": latest, - }) + nextCursor, errCursor := cursorForLatestLogFile(files, latest) + if errCursor != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to prepare log cursor: %v", errCursor)}) + return + } + writeLogsResponse(c, lines, total, latest, nextCursor, false) } // DeleteLogs removes all rotated log files and truncates the active log. @@ -498,6 +507,140 @@ type completeLogRead struct { hitLimit bool } +type logReadResult struct { + lines []string + latest int64 + nextCursor string +} + +func writeLogsResponse(c *gin.Context, lines []string, lineCount int, latest int64, nextCursor string, cursorReset bool) { + if lines == nil { + lines = []string{} + } + payload := gin.H{ + "lines": lines, + "line-count": lineCount, + "latest-timestamp": latest, + "next-cursor": nextCursor, + } + if cursorReset { + payload["cursor-reset"] = true + } + c.JSON(http.StatusOK, payload) +} + +func tailLogFiles(files []string, limit int, fallbackLatest int64) (logReadResult, error) { + result := logReadResult{ + lines: []string{}, + latest: fallbackLatest, + } + for i := len(files) - 1; i >= 0; i-- { + remaining := 0 + if limit > 0 { + remaining = limit - len(result.lines) + if remaining <= 0 { + break + } + } + read, errRead := readTailLogLines(files[i], remaining) + if errRead != nil { + if errors.Is(errRead, os.ErrNotExist) { + continue + } + return logReadResult{}, errRead + } + if len(read.lines) == 0 { + continue + } + result.lines = append(append([]string{}, read.lines...), result.lines...) + if read.latest > result.latest { + result.latest = read.latest + } + } + nextCursor, errCursor := cursorForLatestLogFile(files, result.latest) + if errCursor != nil { + return logReadResult{}, errCursor + } + result.nextCursor = nextCursor + return result, nil +} + +func readTailLogLines(path string, limit int) (completeLogRead, error) { + boundary, errBoundary := completeLogBoundary(path) + if errBoundary != nil { + return completeLogRead{}, errBoundary + } + if boundary == 0 { + return completeLogRead{lines: []string{}}, nil + } + start, errStart := tailStartOffset(path, boundary, limit) + if errStart != nil { + return completeLogRead{}, errStart + } + return readCompleteLogLines(path, start, boundary, limit) +} + +func tailStartOffset(path string, boundary int64, limit int) (int64, error) { + if limit <= 0 { + return 0, nil + } + file, errOpen := os.Open(path) + if errOpen != nil { + return 0, errOpen + } + defer func() { + _ = file.Close() + }() + buf := make([]byte, 32*1024) + pos := boundary + lineBreaks := 0 + for pos > 0 { + chunk := minInt64(int64(len(buf)), pos) + pos -= chunk + n, errRead := file.ReadAt(buf[:chunk], pos) + if errRead != nil && errRead != io.EOF { + return 0, errRead + } + if n <= 0 { + continue + } + data := buf[:n] + for len(data) > 0 { + idx := bytes.LastIndexByte(data, '\n') + if idx < 0 { + break + } + lineBreaks++ + if lineBreaks > limit { + return pos + int64(idx) + 1, nil + } + data = data[:idx] + } + } + return 0, nil +} + +func cursorForLatestLogFile(files []string, latest int64) (string, error) { + for i := len(files) - 1; i >= 0; i-- { + boundary, errBoundary := completeLogBoundary(files[i]) + if errBoundary != nil { + if errors.Is(errBoundary, os.ErrNotExist) { + continue + } + return "", errBoundary + } + cursor, errCursor := newLogCursor(files[i], boundary, latest) + if errCursor != nil { + if errors.Is(errCursor, os.ErrNotExist) { + continue + } + return "", errCursor + } + return cursor, nil + } + return "", nil +} + func encodeLogCursor(cursor logCursor) (string, error) { raw, err := json.Marshal(cursor) if err != nil { @@ -760,7 +903,7 @@ func completeLogBoundary(path string) (int64, error) { if n <= 0 { continue } - if idx := strings.LastIndexByte(string(buf[:n]), '\n'); idx >= 0 { + if idx := bytes.LastIndexByte(buf[:n], '\n'); idx >= 0 { return pos + int64(idx) + 1, nil } } diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index 4f15f706..acc021a1 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -3,11 +3,18 @@ package management import ( "encoding/base64" "encoding/json" + "net/http" + "net/http/httptest" "os" "path/filepath" "reflect" + "strconv" "strings" "testing" + "time" + + "github.com/gin-gonic/gin" + "github.com/router-for-me/CLIProxyAPI/v7/internal/config" ) func TestDecodeLogCursorRejectsUnsafeFiles(t *testing.T) { @@ -126,6 +133,80 @@ func TestReadCompleteLogLinesSkipsTrailingPartial(t *testing.T) { } } +func TestGetLogsTailLimitReturnsRecentLinesWithCursor(t *testing.T) { + dir := t.TempDir() + lines := []string{ + "[2026-06-15 10:00:00] first", + "[2026-06-15 10:00:01] second", + "[2026-06-15 10:00:02] third", + "[2026-06-15 10:00:03] fourth", + } + writeMainLog(t, dir, strings.Join(lines, "\n")+"\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=2") + wantLines := []string{lines[2], lines[3]} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.LineCount != 2 { + t.Fatalf("line-count = %d, want 2", resp.LineCount) + } + if resp.NextCursor == "" { + t.Fatal("next-cursor is empty") + } + wantLatest := time.Date(2026, 6, 15, 10, 0, 3, 0, time.Local).Unix() + if resp.LatestTimestamp != wantLatest { + t.Fatalf("latest-timestamp = %d, want %d", resp.LatestTimestamp, wantLatest) + } +} + +func TestGetLogsNoLimitKeepsFullScanBehavior(t *testing.T) { + dir := t.TempDir() + writeMainLog(t, dir, "complete\npartial") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs") + wantLines := []string{"complete", "partial"} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.LineCount != 2 { + t.Fatalf("line-count = %d, want full scan count 2", resp.LineCount) + } + if resp.NextCursor == "" { + t.Fatal("next-cursor is empty") + } + cursor, errCursor := decodeLogCursor(resp.NextCursor) + if errCursor != nil { + t.Fatalf("decode next-cursor: %v", errCursor) + } + if cursor.Offset != int64(len("complete\n")) { + t.Fatalf("cursor offset = %d, want complete-line boundary", cursor.Offset) + } +} + +func TestGetLogsAfterKeepsTimestampScanAndReturnsCursor(t *testing.T) { + dir := t.TempDir() + lines := []string{ + "[2026-06-15 10:00:00] first", + "[2026-06-15 10:00:01] second", + "[2026-06-15 10:00:02] third", + } + writeMainLog(t, dir, strings.Join(lines, "\n")+"\n") + + cutoff := time.Date(2026, 6, 15, 10, 0, 0, 0, time.Local).Unix() + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?after="+strconv.FormatInt(cutoff, 10)) + wantLines := []string{lines[1], lines[2]} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.LineCount != 3 { + t.Fatalf("line-count = %d, want full scan count 3", resp.LineCount) + } + if resp.NextCursor == "" { + t.Fatal("next-cursor is empty") + } +} + func mustEncodeRawCursor(t *testing.T, cursor logCursor) string { t.Helper() raw, err := json.Marshal(cursor) @@ -134,3 +215,44 @@ func mustEncodeRawCursor(t *testing.T, cursor logCursor) string { } return base64.RawURLEncoding.EncodeToString(raw) } + +type logsAPIResponse struct { + Lines []string `json:"lines"` + LineCount int `json:"line-count"` + LatestTimestamp int64 `json:"latest-timestamp"` + NextCursor string `json:"next-cursor"` + CursorReset bool `json:"cursor-reset"` +} + +func newLogsTestHandler(dir string, loggingToFile bool) *Handler { + h := NewHandlerWithoutConfigFilePath(&config.Config{LoggingToFile: loggingToFile}, nil) + h.SetLogDirectory(dir) + return h +} + +func performGetLogs(t *testing.T, h *Handler, target string) logsAPIResponse { + t.Helper() + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodGet, target, nil) + h.GetLogs(c) + if rec.Code != http.StatusOK { + t.Fatalf("GetLogs status = %d, body = %s", rec.Code, rec.Body.String()) + } + var resp logsAPIResponse + if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + t.Fatalf("decode response: %v", err) + } + if resp.Lines == nil { + resp.Lines = []string{} + } + return resp +} + +func writeMainLog(t *testing.T, dir, content string) { + t.Helper() + if err := os.WriteFile(filepath.Join(dir, defaultLogFileName), []byte(content), 0o644); err != nil { + t.Fatalf("write main log: %v", err) + } +} -- 2.51.2 From 331daa24ad9061b20b28d87e21e61c2089f7f707 Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 02:10:25 +0800 Subject: [PATCH 03/10] Read management logs from cursors --- internal/api/handlers/management/logs.go | 164 +++++++++++- internal/api/handlers/management/logs_test.go | 242 +++++++++++++++++- 2 files changed, 396 insertions(+), 10 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index fa3b6cb0..c41a0ed1 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -51,11 +51,18 @@ func (h *Handler) GetLogs(c *gin.Context) { return } + rawCursor := strings.TrimSpace(c.Query("cursor")) files, err := h.collectLogFiles(logDir) if err != nil { if os.IsNotExist(err) { cutoff := parseCutoff(c.Query("after")) - writeLogsResponse(c, []string{}, 0, cutoff, "", false) + latest := cutoff + if rawCursor != "" { + if cursor, errCursor := decodeLogCursor(rawCursor); errCursor == nil && cursor.LatestTimestamp > latest { + latest = cursor.LatestTimestamp + } + } + writeLogsResponse(c, []string{}, 0, latest, "", rawCursor != "") return } c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to list log files: %v", err)}) @@ -69,7 +76,26 @@ func (h *Handler) GetLogs(c *gin.Context) { } cutoff := parseCutoff(c.Query("after")) - if strings.TrimSpace(c.Query("cursor")) == "" && cutoff == 0 && limit > 0 { + if rawCursor != "" { + result, reset, errCursor := readLogFilesFromCursor(logDir, files, rawCursor, limit) + if errCursor != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errCursor)}) + return + } + if reset { + result, errCursor = tailLogFiles(files, limit, result.latest) + if errCursor != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errCursor)}) + return + } + writeLogsResponse(c, result.lines, len(result.lines), result.latest, result.nextCursor, true) + return + } + writeLogsResponse(c, result.lines, len(result.lines), result.latest, result.nextCursor, false) + return + } + + if cutoff == 0 && limit > 0 { result, errTail := tailLogFiles(files, limit, 0) if errTail != nil { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errTail)}) @@ -641,6 +667,140 @@ func cursorForLatestLogFile(files []string, latest int64) (string, error) { return "", nil } +func readLogFilesFromCursor(logDir string, files []string, raw string, limit int) (logReadResult, bool, error) { + cursor, errDecode := decodeLogCursor(raw) + if errDecode != nil { + return logReadResult{lines: []string{}}, true, nil + } + result := logReadResult{ + lines: []string{}, + latest: cursor.LatestTimestamp, + nextCursor: raw, + } + if _, errPath := safeLogFilePath(logDir, cursor.File); errPath != nil { + return result, true, nil + } + startIndex, found, errLocate := locateLogCursorFile(files, cursor) + if errLocate != nil { + return result, false, errLocate + } + if !found { + return result, true, nil + } + + currentCursorPath := files[startIndex] + currentCursorOffset := cursor.Offset + advanced := false + for i := startIndex; i < len(files); i++ { + remaining := 0 + if limit > 0 { + remaining = limit - len(result.lines) + if remaining <= 0 { + break + } + } + offset := int64(0) + if i == startIndex { + offset = cursor.Offset + } + read, errRead := readCompleteLogLines(files[i], offset, -1, remaining) + if errRead != nil { + if errors.Is(errRead, os.ErrNotExist) { + return result, true, nil + } + return result, false, errRead + } + if len(read.lines) > 0 { + result.lines = append(result.lines, read.lines...) + if read.latest > result.latest { + result.latest = read.latest + } + currentCursorPath = files[i] + currentCursorOffset = read.endOffset + advanced = true + } + if read.hitLimit { + break + } + } + if !advanced { + return result, false, nil + } + + nextCursor, errCursor := newLogCursor(currentCursorPath, currentCursorOffset, result.latest) + if errCursor != nil { + if errors.Is(errCursor, os.ErrNotExist) { + return result, true, nil + } + return result, false, errCursor + } + result.nextCursor = nextCursor + return result, false, nil +} + +func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { + nameToIndex := make(map[string]int, len(files)) + for i := range files { + nameToIndex[filepath.Base(files[i])] = i + } + if index, ok := nameToIndex[cursor.File]; ok { + matches, truncated, errMatch := logFileMatchesCursor(files[index], cursor) + if errMatch != nil { + if errors.Is(errMatch, os.ErrNotExist) { + return 0, false, nil + } + return 0, false, errMatch + } + if truncated { + return 0, false, nil + } + if matches { + return index, true, nil + } + } + + if cursor.File != defaultLogFileName || cursor.Offset == 0 { + return 0, false, nil + } + for i := range files { + if filepath.Base(files[i]) == defaultLogFileName { + continue + } + matches, truncated, errMatch := logFileMatchesCursor(files[i], cursor) + if errMatch != nil { + if errors.Is(errMatch, os.ErrNotExist) { + continue + } + return 0, false, errMatch + } + if truncated { + continue + } + if matches { + return i, true, nil + } + } + return 0, false, nil +} + +func logFileMatchesCursor(path string, cursor logCursor) (bool, bool, error) { + info, errStat := os.Stat(path) + if errStat != nil { + return false, false, errStat + } + if info.IsDir() { + return false, false, fmt.Errorf("invalid log file") + } + if info.Size() < cursor.Offset { + return false, true, nil + } + fingerprint, errFingerprint := logFileFingerprint(path, cursor.Offset) + if errFingerprint != nil { + return false, false, errFingerprint + } + return fingerprint == cursor.Fingerprint, false, nil +} + func encodeLogCursor(cursor logCursor) (string, error) { raw, err := json.Marshal(cursor) if err != nil { diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index acc021a1..b7b73b4d 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -5,6 +5,7 @@ import ( "encoding/json" "net/http" "net/http/httptest" + "net/url" "os" "path/filepath" "reflect" @@ -207,6 +208,210 @@ func TestGetLogsAfterKeepsTimestampScanAndReturnsCursor(t *testing.T) { } } +func TestGetLogsCursorReturnsOnlyNewCompleteLines(t *testing.T) { + dir := t.TempDir() + lines := []string{ + "[2026-06-15 10:00:00] first", + "[2026-06-15 10:00:01] second", + "[2026-06-15 10:00:02] third", + } + writeMainLog(t, dir, strings.Join(lines, "\n")+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=2") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + + appendMainLog(t, dir, "[2026-06-15 10:00:03] fourth\n") + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + wantLines := []string{"[2026-06-15 10:00:03] fourth"} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.LineCount != 1 { + t.Fatalf("line-count = %d, want 1", resp.LineCount) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } + wantLatest := time.Date(2026, 6, 15, 10, 0, 3, 0, time.Local).Unix() + if resp.LatestTimestamp != wantLatest { + t.Fatalf("latest-timestamp = %d, want %d", resp.LatestTimestamp, wantLatest) + } +} + +func TestGetLogsCursorNoNewLinesKeepsCursorStable(t *testing.T) { + dir := t.TempDir() + line := "[2026-06-15 10:00:00] first" + writeMainLog(t, dir, line+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + if len(resp.Lines) != 0 { + t.Fatalf("lines = %#v, want empty", resp.Lines) + } + if resp.LineCount != 0 { + t.Fatalf("line-count = %d, want 0", resp.LineCount) + } + if resp.NextCursor != initial.NextCursor { + t.Fatalf("next-cursor changed with no complete lines") + } + if resp.LatestTimestamp != initial.LatestTimestamp { + t.Fatalf("latest-timestamp = %d, want %d", resp.LatestTimestamp, initial.LatestTimestamp) + } +} + +func TestGetLogsCursorDoesNotAdvancePastTrailingPartial(t *testing.T) { + dir := t.TempDir() + line := "[2026-06-15 10:00:00] first" + writeMainLog(t, dir, line+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + + appendMainLog(t, dir, "partial") + partial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + if len(partial.Lines) != 0 { + t.Fatalf("partial lines = %#v, want empty", partial.Lines) + } + if partial.NextCursor != initial.NextCursor { + t.Fatalf("cursor advanced past partial line") + } + + appendMainLog(t, dir, "\n") + complete := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + if !reflect.DeepEqual(complete.Lines, []string{"partial"}) { + t.Fatalf("complete lines = %#v, want partial", complete.Lines) + } + if complete.LatestTimestamp != initial.LatestTimestamp { + t.Fatalf("latest-timestamp = %d, want %d", complete.LatestTimestamp, initial.LatestTimestamp) + } +} + +func TestGetLogsCursorResetAfterTruncateTailsLimit(t *testing.T) { + dir := t.TempDir() + lines := []string{ + "[2026-06-15 10:00:00] first", + "[2026-06-15 10:00:01] second", + "[2026-06-15 10:00:02] third", + } + writeMainLog(t, dir, strings.Join(lines, "\n")+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=3") + + resetLine := "[2026-06-15 10:00:03] reset" + writeMainLog(t, dir, resetLine+"\n") + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=1") + if !resp.CursorReset { + t.Fatal("cursor-reset = false, want true") + } + if !reflect.DeepEqual(resp.Lines, []string{resetLine}) { + t.Fatalf("lines = %#v, want reset tail", resp.Lines) + } + if resp.LineCount != 1 { + t.Fatalf("line-count = %d, want 1", resp.LineCount) + } +} + +func TestGetLogsCursorReadsAcrossRotation(t *testing.T) { + dir := t.TempDir() + line1 := "[2026-06-15 10:00:00] first" + line2 := "[2026-06-15 10:00:01] second" + line3 := "[2026-06-15 10:00:02] third" + writeMainLog(t, dir, line1+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + + appendMainLog(t, dir, line2+"\n") + if err := os.Rename(filepath.Join(dir, defaultLogFileName), filepath.Join(dir, defaultLogFileName+".1")); err != nil { + t.Fatalf("rotate main log: %v", err) + } + writeMainLog(t, dir, line3+"\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + wantLines := []string{line2, line3} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } +} + +func TestGetLogsInvalidCursorResetsToTail(t *testing.T) { + dir := t.TempDir() + lines := []string{ + "[2026-06-15 10:00:00] first", + "[2026-06-15 10:00:01] second", + } + writeMainLog(t, dir, strings.Join(lines, "\n")+"\n") + + cases := []string{ + "not-base64", + mustEncodeRawCursor(t, logCursor{ + Version: logCursorVersion, + File: "../secret", + Fingerprint: "fingerprint", + }), + } + for _, raw := range cases { + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(raw)+"&limit=1") + if !resp.CursorReset { + t.Fatalf("cursor-reset = false for cursor %q", raw) + } + if !reflect.DeepEqual(resp.Lines, []string{lines[1]}) { + t.Fatalf("lines = %#v, want latest line", resp.Lines) + } + if resp.LineCount != 1 { + t.Fatalf("line-count = %d, want 1", resp.LineCount) + } + } +} + +func TestGetLogsMissingRotatedCursorFileResetsToTail(t *testing.T) { + dir := t.TempDir() + current := "[2026-06-15 10:00:01] current" + writeMainLog(t, dir, current+"\n") + rotatedPath := filepath.Join(dir, defaultLogFileName+".1") + if err := os.WriteFile(rotatedPath, []byte("[2026-06-15 10:00:00] old\n"), 0o644); err != nil { + t.Fatalf("write rotated log: %v", err) + } + cursor, errCursor := newLogCursor(rotatedPath, int64(len("[2026-06-15 10:00:00] old\n")), 0) + if errCursor != nil { + t.Fatalf("newLogCursor() error = %v", errCursor) + } + if errRemove := os.Remove(rotatedPath); errRemove != nil { + t.Fatalf("remove rotated log: %v", errRemove) + } + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(cursor)+"&limit=1") + if !resp.CursorReset { + t.Fatal("cursor-reset = false, want true") + } + if !reflect.DeepEqual(resp.Lines, []string{current}) { + t.Fatalf("lines = %#v, want current tail", resp.Lines) + } +} + +func TestGetLogsMissingLogDirKeepsOKEmptyResponse(t *testing.T) { + dir := filepath.Join(t.TempDir(), "missing") + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape("not-base64")+"&limit=1") + if len(resp.Lines) != 0 { + t.Fatalf("lines = %#v, want empty", resp.Lines) + } + if resp.LineCount != 0 { + t.Fatalf("line-count = %d, want 0", resp.LineCount) + } + if !resp.CursorReset { + t.Fatal("cursor-reset = false, want true for cursor against missing log dir") + } +} + +func TestGetLogsLoggingDisabledKeepsBadRequest(t *testing.T) { + status, body := performGetLogsRaw(t, newLogsTestHandler(t.TempDir(), false), "/v0/management/logs?cursor=not-base64&limit=1") + if status != http.StatusBadRequest { + t.Fatalf("status = %d, want %d", status, http.StatusBadRequest) + } + if !strings.Contains(body, "logging to file disabled") { + t.Fatalf("body = %s, want logging disabled error", body) + } +} + func mustEncodeRawCursor(t *testing.T, cursor logCursor) string { t.Helper() raw, err := json.Marshal(cursor) @@ -232,16 +437,12 @@ func newLogsTestHandler(dir string, loggingToFile bool) *Handler { func performGetLogs(t *testing.T, h *Handler, target string) logsAPIResponse { t.Helper() - gin.SetMode(gin.TestMode) - rec := httptest.NewRecorder() - c, _ := gin.CreateTestContext(rec) - c.Request = httptest.NewRequest(http.MethodGet, target, nil) - h.GetLogs(c) - if rec.Code != http.StatusOK { - t.Fatalf("GetLogs status = %d, body = %s", rec.Code, rec.Body.String()) + status, body := performGetLogsRaw(t, h, target) + if status != http.StatusOK { + t.Fatalf("GetLogs status = %d, body = %s", status, body) } var resp logsAPIResponse - if err := json.Unmarshal(rec.Body.Bytes(), &resp); err != nil { + if err := json.Unmarshal([]byte(body), &resp); err != nil { t.Fatalf("decode response: %v", err) } if resp.Lines == nil { @@ -250,9 +451,34 @@ func performGetLogs(t *testing.T, h *Handler, target string) logsAPIResponse { return resp } +func performGetLogsRaw(t *testing.T, h *Handler, target string) (int, string) { + t.Helper() + gin.SetMode(gin.TestMode) + rec := httptest.NewRecorder() + c, _ := gin.CreateTestContext(rec) + c.Request = httptest.NewRequest(http.MethodGet, target, nil) + h.GetLogs(c) + return rec.Code, rec.Body.String() +} + func writeMainLog(t *testing.T, dir, content string) { t.Helper() if err := os.WriteFile(filepath.Join(dir, defaultLogFileName), []byte(content), 0o644); err != nil { t.Fatalf("write main log: %v", err) } } + +func appendMainLog(t *testing.T, dir, content string) { + t.Helper() + file, errOpen := os.OpenFile(filepath.Join(dir, defaultLogFileName), os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0o644) + if errOpen != nil { + t.Fatalf("open main log: %v", errOpen) + } + if _, errWrite := file.WriteString(content); errWrite != nil { + _ = file.Close() + t.Fatalf("append main log: %v", errWrite) + } + if errClose := file.Close(); errClose != nil { + t.Fatalf("close main log: %v", errClose) + } +} -- 2.51.2 From 0d82daca6bf2348872cc0c309060dcb2e0db3895 Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 02:17:57 +0800 Subject: [PATCH 04/10] Preserve management log line counts --- internal/api/handlers/management/logs.go | 67 ++++++++++++++++++- internal/api/handlers/management/logs_test.go | 4 +- 2 files changed, 68 insertions(+), 3 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index c41a0ed1..f42433e2 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -101,7 +101,12 @@ func (h *Handler) GetLogs(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errTail)}) return } - writeLogsResponse(c, result.lines, len(result.lines), result.latest, result.nextCursor, false) + total, errCount := countLogFileLines(files) + if errCount != nil { + c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errCount)}) + return + } + writeLogsResponse(c, result.lines, total, result.latest, result.nextCursor, false) return } @@ -606,6 +611,66 @@ func readTailLogLines(path string, limit int) (completeLogRead, error) { return readCompleteLogLines(path, start, boundary, limit) } +func countLogFileLines(files []string) (int, error) { + total := 0 + for i := range files { + count, errCount := countLogLines(files[i]) + if errCount != nil { + if errors.Is(errCount, os.ErrNotExist) { + continue + } + return 0, errCount + } + total += count + } + return total, nil +} + +func countLogLines(path string) (int, error) { + file, errOpen := os.Open(path) + if errOpen != nil { + return 0, errOpen + } + defer func() { + _ = file.Close() + }() + info, errStat := file.Stat() + if errStat != nil { + return 0, errStat + } + if info.IsDir() { + return 0, fmt.Errorf("invalid log file") + } + + buf := make([]byte, 32*1024) + count := 0 + lineLen := 0 + for { + n, errRead := file.Read(buf) + for _, b := range buf[:n] { + if b == '\n' { + count++ + lineLen = 0 + continue + } + lineLen++ + if lineLen > logScannerMaxBuffer { + return 0, fmt.Errorf("log line exceeds %d bytes", logScannerMaxBuffer) + } + } + if errRead == io.EOF { + break + } + if errRead != nil { + return 0, errRead + } + } + if lineLen > 0 { + count++ + } + return count, nil +} + func tailStartOffset(path string, boundary int64, limit int) (int64, error) { if limit <= 0 { return 0, nil diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index b7b73b4d..55b5ba3f 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -149,8 +149,8 @@ func TestGetLogsTailLimitReturnsRecentLinesWithCursor(t *testing.T) { if !reflect.DeepEqual(resp.Lines, wantLines) { t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) } - if resp.LineCount != 2 { - t.Fatalf("line-count = %d, want 2", resp.LineCount) + if resp.LineCount != 4 { + t.Fatalf("line-count = %d, want full scan count 4", resp.LineCount) } if resp.NextCursor == "" { t.Fatal("next-cursor is empty") -- 2.51.2 From d417fa534fa77ed34899133a9bbe1d5503dd812a Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 02:18:58 +0800 Subject: [PATCH 05/10] Bound management log cursor reads --- internal/api/handlers/management/logs.go | 52 +++++++++++++------ internal/api/handlers/management/logs_test.go | 18 +++++++ 2 files changed, 53 insertions(+), 17 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index f42433e2..75cdcde6 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -1062,29 +1062,47 @@ func readCompleteLogLines(path string, offset, maxOffset int64, limit int) (comp return completeLogRead{}, fmt.Errorf("invalid log offset") } - reader := bufio.NewReader(io.NewSectionReader(file, offset, maxOffset-offset)) + reader := io.NewSectionReader(file, offset, maxOffset-offset) result := completeLogRead{ lines: []string{}, endOffset: offset, } currentOffset := offset + buf := make([]byte, 32*1024) + line := make([]byte, 0, logScannerInitialBuffer) for { - raw, errRead := reader.ReadString('\n') - if strings.HasSuffix(raw, "\n") { - currentOffset += int64(len(raw)) - line := strings.TrimSuffix(raw, "\n") - line = strings.TrimRight(line, "\r") - result.lines = append(result.lines, line) - result.endOffset = currentOffset - if ts := parseTimestamp(line); ts > result.latest { - result.latest = ts - } - if limit > 0 && len(result.lines) >= limit { - result.hitLimit = true - break - } - if errRead == nil { - continue + n, errRead := reader.Read(buf) + if n > 0 { + data := buf[:n] + for len(data) > 0 { + idx := bytes.IndexByte(data, '\n') + if idx < 0 { + if len(line)+len(data) > logScannerMaxBuffer { + return completeLogRead{}, fmt.Errorf("log line exceeds %d bytes", logScannerMaxBuffer) + } + line = append(line, data...) + currentOffset += int64(len(data)) + break + } + + segment := data[:idx] + if len(line)+len(segment) > logScannerMaxBuffer { + return completeLogRead{}, fmt.Errorf("log line exceeds %d bytes", logScannerMaxBuffer) + } + line = append(line, segment...) + currentOffset += int64(idx) + 1 + text := strings.TrimRight(string(line), "\r") + result.lines = append(result.lines, text) + result.endOffset = currentOffset + if ts := parseTimestamp(text); ts > result.latest { + result.latest = ts + } + line = line[:0] + if limit > 0 && len(result.lines) >= limit { + result.hitLimit = true + return result, nil + } + data = data[idx+1:] } } if errRead == io.EOF { diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index 55b5ba3f..c0275e6a 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -239,6 +239,24 @@ func TestGetLogsCursorReturnsOnlyNewCompleteLines(t *testing.T) { } } +func TestGetLogsCursorRejectsOversizedLine(t *testing.T) { + dir := t.TempDir() + writeMainLog(t, dir, "[2026-06-15 10:00:00] first\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + + appendMainLog(t, dir, strings.Repeat("x", logScannerMaxBuffer+1)+"\n") + status, body := performGetLogsRaw(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=1") + if status != http.StatusInternalServerError { + t.Fatalf("status = %d, want %d", status, http.StatusInternalServerError) + } + if !strings.Contains(body, "log line exceeds") { + t.Fatalf("body = %s, want oversized line error", body) + } +} + func TestGetLogsCursorNoNewLinesKeepsCursorStable(t *testing.T) { dir := t.TempDir() line := "[2026-06-15 10:00:00] first" -- 2.51.2 From 917cec3bf622121af6e2a8252f6db61217914fbf Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 15:46:58 +0800 Subject: [PATCH 06/10] Continue log cursors across rotation --- internal/api/handlers/management/logs.go | 5 +-- internal/api/handlers/management/logs_test.go | 31 +++++++++++++++++++ 2 files changed, 32 insertions(+), 4 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index 75cdcde6..72e95c6f 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -816,10 +816,7 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { } return 0, false, errMatch } - if truncated { - return 0, false, nil - } - if matches { + if matches && !truncated { return index, true, nil } } diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index c0275e6a..1f1056e5 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -351,6 +351,37 @@ func TestGetLogsCursorReadsAcrossRotation(t *testing.T) { } } +func TestGetLogsCursorReadsRotatedFileWhenNewMainIsSmaller(t *testing.T) { + dir := t.TempDir() + line1 := "[2026-06-15 10:00:00] first line with enough bytes" + line2 := "[2026-06-15 10:00:01] second" + line3 := "new" + writeMainLog(t, dir, line1+"\n") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + + appendMainLog(t, dir, line2+"\n") + if err := os.Rename(filepath.Join(dir, defaultLogFileName), filepath.Join(dir, defaultLogFileName+".1")); err != nil { + t.Fatalf("rotate main log: %v", err) + } + writeMainLog(t, dir, line3+"\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=1") + if !reflect.DeepEqual(resp.Lines, []string{line2}) { + t.Fatalf("lines = %#v, want rotated unread line", resp.Lines) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } + + next := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(resp.NextCursor)+"&limit=1") + if !reflect.DeepEqual(next.Lines, []string{line3}) { + t.Fatalf("next lines = %#v, want new main line", next.Lines) + } + if next.CursorReset { + t.Fatal("next cursor-reset = true, want false") + } +} + func TestGetLogsInvalidCursorResetsToTail(t *testing.T) { dir := t.TempDir() lines := []string{ -- 2.51.2 From db3fdea4a13b8f2e68ae392833905835f54b645b Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 15:50:03 +0800 Subject: [PATCH 07/10] Disambiguate zero-offset log cursors --- internal/api/handlers/management/logs.go | 65 ++++++++++++++-- internal/api/handlers/management/logs_test.go | 74 +++++++++++++++++++ 2 files changed, 134 insertions(+), 5 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index 72e95c6f..3dfc635a 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -527,6 +527,7 @@ type logCursor struct { Offset int64 `json:"offset"` Size int64 `json:"size"` ModTime int64 `json:"modTime"` + ModTimeUnixNano int64 `json:"modTimeUnixNano,omitempty"` LatestTimestamp int64 `json:"latestTimestamp"` Fingerprint string `json:"fingerprint"` } @@ -808,6 +809,7 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { for i := range files { nameToIndex[filepath.Base(files[i])] = i } + deferEmptyMainMatch := false if index, ok := nameToIndex[cursor.File]; ok { matches, truncated, errMatch := logFileMatchesCursor(files[index], cursor) if errMatch != nil { @@ -817,17 +819,24 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { return 0, false, errMatch } if matches && !truncated { - return index, true, nil + if shouldDeferEmptyMainCursorToRotated(files, cursor) { + deferEmptyMainMatch = true + } else { + return index, true, nil + } } } - if cursor.File != defaultLogFileName || cursor.Offset == 0 { + if cursor.File != defaultLogFileName || (cursor.Offset == 0 && cursor.Size == 0 && !deferEmptyMainMatch) { return 0, false, nil } - for i := range files { + for i := len(files) - 1; i >= 0; i-- { if filepath.Base(files[i]) == defaultLogFileName { continue } + if cursor.Offset == 0 && cursor.Size == 0 && !logFileChangedAfterCursor(files[i], cursor) { + continue + } matches, truncated, errMatch := logFileMatchesCursor(files[i], cursor) if errMatch != nil { if errors.Is(errMatch, os.ErrNotExist) { @@ -845,6 +854,29 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { return 0, false, nil } +func shouldDeferEmptyMainCursorToRotated(files []string, cursor logCursor) bool { + if cursor.File != defaultLogFileName || cursor.Offset != 0 || cursor.Size != 0 { + return false + } + for i := range files { + if filepath.Base(files[i]) == defaultLogFileName { + continue + } + if logFileChangedAfterCursor(files[i], cursor) { + return true + } + } + return false +} + +func logFileChangedAfterCursor(path string, cursor logCursor) bool { + info, errStat := os.Stat(path) + if errStat != nil || info.IsDir() || info.Size() == 0 { + return false + } + return info.ModTime().UnixNano() > cursorModTimeUnixNano(cursor) +} + func logFileMatchesCursor(path string, cursor logCursor) (bool, bool, error) { info, errStat := os.Stat(path) if errStat != nil { @@ -856,7 +888,11 @@ func logFileMatchesCursor(path string, cursor logCursor) (bool, bool, error) { if info.Size() < cursor.Offset { return false, true, nil } - fingerprint, errFingerprint := logFileFingerprint(path, cursor.Offset) + boundary := cursorFingerprintBoundary(cursor) + if info.Size() < boundary { + return false, true, nil + } + fingerprint, errFingerprint := logFileFingerprint(path, boundary) if errFingerprint != nil { return false, false, errFingerprint } @@ -953,7 +989,11 @@ func newLogCursor(path string, offset, latest int64) (string, error) { if offset < 0 || offset > info.Size() { return "", fmt.Errorf("invalid cursor offset") } - fingerprint, errFingerprint := logFileFingerprint(path, offset) + fingerprintCursor := logCursor{ + Offset: offset, + Size: info.Size(), + } + fingerprint, errFingerprint := logFileFingerprint(path, cursorFingerprintBoundary(fingerprintCursor)) if errFingerprint != nil { return "", errFingerprint } @@ -963,11 +1003,26 @@ func newLogCursor(path string, offset, latest int64) (string, error) { Offset: offset, Size: info.Size(), ModTime: info.ModTime().Unix(), + ModTimeUnixNano: info.ModTime().UnixNano(), LatestTimestamp: latest, Fingerprint: fingerprint, }) } +func cursorFingerprintBoundary(cursor logCursor) int64 { + if cursor.Offset == 0 && cursor.Size > 0 { + return cursor.Size + } + return cursor.Offset +} + +func cursorModTimeUnixNano(cursor logCursor) int64 { + if cursor.ModTimeUnixNano > 0 { + return cursor.ModTimeUnixNano + } + return cursor.ModTime * int64(time.Second) +} + func logFileFingerprint(path string, boundary int64) (string, error) { if boundary < 0 { return "", fmt.Errorf("invalid fingerprint boundary") diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index 1f1056e5..eb38176c 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -382,6 +382,80 @@ func TestGetLogsCursorReadsRotatedFileWhenNewMainIsSmaller(t *testing.T) { } } +func TestGetLogsZeroOffsetCursorWithPartialLineReadsAcrossRotation(t *testing.T) { + dir := t.TempDir() + writeMainLog(t, dir, "partial") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + cursor, errCursor := decodeLogCursor(initial.NextCursor) + if errCursor != nil { + t.Fatalf("decode initial cursor: %v", errCursor) + } + if cursor.Offset != 0 || cursor.Size == 0 { + t.Fatalf("cursor offset/size = %d/%d, want zero offset with partial size", cursor.Offset, cursor.Size) + } + + appendMainLog(t, dir, " complete\n") + if err := os.Rename(filepath.Join(dir, defaultLogFileName), filepath.Join(dir, defaultLogFileName+".1")); err != nil { + t.Fatalf("rotate main log: %v", err) + } + writeMainLog(t, dir, "new\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=10") + wantLines := []string{"partial complete", "new"} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } +} + +func TestGetLogsZeroOffsetCursorWithEmptyFileReadsAcrossRotation(t *testing.T) { + dir := t.TempDir() + writeMainLog(t, dir, "") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + cursor, errCursor := decodeLogCursor(initial.NextCursor) + if errCursor != nil { + t.Fatalf("decode initial cursor: %v", errCursor) + } + if cursor.Offset != 0 || cursor.Size != 0 { + t.Fatalf("cursor offset/size = %d/%d, want empty zero offset", cursor.Offset, cursor.Size) + } + + appendMainLog(t, dir, "first\n") + mainPath := filepath.Join(dir, defaultLogFileName) + nextModTime := time.Unix(0, cursorModTimeUnixNano(cursor)+int64(time.Second)) + if err := os.Chtimes(mainPath, nextModTime, nextModTime); err != nil { + t.Fatalf("update main log mtime: %v", err) + } + if err := os.Rename(mainPath, filepath.Join(dir, defaultLogFileName+".1")); err != nil { + t.Fatalf("rotate main log: %v", err) + } + writeMainLog(t, dir, "second\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=1") + if !reflect.DeepEqual(resp.Lines, []string{"first"}) { + t.Fatalf("lines = %#v, want first rotated line", resp.Lines) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } + + next := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(resp.NextCursor)+"&limit=1") + if !reflect.DeepEqual(next.Lines, []string{"second"}) { + t.Fatalf("next lines = %#v, want second main line", next.Lines) + } + if next.CursorReset { + t.Fatal("next cursor-reset = true, want false") + } +} + func TestGetLogsInvalidCursorResetsToTail(t *testing.T) { dir := t.TempDir() lines := []string{ -- 2.51.2 From a47c38631979d4ab4614af3c98ca2808255c0f3a Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 15:51:15 +0800 Subject: [PATCH 08/10] Avoid counting all logs for tail reads --- internal/api/handlers/management/logs.go | 67 +------------------ internal/api/handlers/management/logs_test.go | 22 +++++- 2 files changed, 21 insertions(+), 68 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index 3dfc635a..6d8477ba 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -101,12 +101,7 @@ func (h *Handler) GetLogs(c *gin.Context) { c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errTail)}) return } - total, errCount := countLogFileLines(files) - if errCount != nil { - c.JSON(http.StatusInternalServerError, gin.H{"error": fmt.Sprintf("failed to read log files: %v", errCount)}) - return - } - writeLogsResponse(c, result.lines, total, result.latest, result.nextCursor, false) + writeLogsResponse(c, result.lines, len(result.lines), result.latest, result.nextCursor, false) return } @@ -612,66 +607,6 @@ func readTailLogLines(path string, limit int) (completeLogRead, error) { return readCompleteLogLines(path, start, boundary, limit) } -func countLogFileLines(files []string) (int, error) { - total := 0 - for i := range files { - count, errCount := countLogLines(files[i]) - if errCount != nil { - if errors.Is(errCount, os.ErrNotExist) { - continue - } - return 0, errCount - } - total += count - } - return total, nil -} - -func countLogLines(path string) (int, error) { - file, errOpen := os.Open(path) - if errOpen != nil { - return 0, errOpen - } - defer func() { - _ = file.Close() - }() - info, errStat := file.Stat() - if errStat != nil { - return 0, errStat - } - if info.IsDir() { - return 0, fmt.Errorf("invalid log file") - } - - buf := make([]byte, 32*1024) - count := 0 - lineLen := 0 - for { - n, errRead := file.Read(buf) - for _, b := range buf[:n] { - if b == '\n' { - count++ - lineLen = 0 - continue - } - lineLen++ - if lineLen > logScannerMaxBuffer { - return 0, fmt.Errorf("log line exceeds %d bytes", logScannerMaxBuffer) - } - } - if errRead == io.EOF { - break - } - if errRead != nil { - return 0, errRead - } - } - if lineLen > 0 { - count++ - } - return count, nil -} - func tailStartOffset(path string, boundary int64, limit int) (int64, error) { if limit <= 0 { return 0, nil diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index eb38176c..34a07c7e 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -149,8 +149,8 @@ func TestGetLogsTailLimitReturnsRecentLinesWithCursor(t *testing.T) { if !reflect.DeepEqual(resp.Lines, wantLines) { t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) } - if resp.LineCount != 4 { - t.Fatalf("line-count = %d, want full scan count 4", resp.LineCount) + if resp.LineCount != len(wantLines) { + t.Fatalf("line-count = %d, want returned line count %d", resp.LineCount, len(wantLines)) } if resp.NextCursor == "" { t.Fatal("next-cursor is empty") @@ -161,6 +161,24 @@ func TestGetLogsTailLimitReturnsRecentLinesWithCursor(t *testing.T) { } } +func TestGetLogsTailLimitDoesNotScanOlderFilesForLineCount(t *testing.T) { + dir := t.TempDir() + rotatedPath := filepath.Join(dir, defaultLogFileName+".1") + if err := os.WriteFile(rotatedPath, []byte(strings.Repeat("x", logScannerMaxBuffer+1)+"\n"), 0o644); err != nil { + t.Fatalf("write rotated log: %v", err) + } + writeMainLog(t, dir, "[2026-06-15 10:00:00] current\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + wantLines := []string{"[2026-06-15 10:00:00] current"} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if resp.LineCount != len(wantLines) { + t.Fatalf("line-count = %d, want returned line count %d", resp.LineCount, len(wantLines)) + } +} + func TestGetLogsNoLimitKeepsFullScanBehavior(t *testing.T) { dir := t.TempDir() writeMainLog(t, dir, "complete\npartial") -- 2.51.2 From 5036513bf9f6b1f80a5510705db60927b7fbc77b Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 16:53:50 +0800 Subject: [PATCH 09/10] Fix ambiguous empty log cursor handling --- internal/api/handlers/management/logs.go | 34 ++++++++++++++ internal/api/handlers/management/logs_test.go | 47 +++++++++++++++++++ 2 files changed, 81 insertions(+) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index 6d8477ba..ad15a741 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -31,6 +31,12 @@ const ( ) // GetLogs returns log lines with optional incremental loading. +// +// The legacy timestamp path keeps line-count as the total scanned line count for +// compatibility. Cursor and tail reads avoid scanning older files, so line-count +// is the number of returned lines there. A cursor emitted by the legacy path +// points at the latest complete log boundary; combining after with limit is +// therefore tail semantics and does not replay lines trimmed by limit. func (h *Handler) GetLogs(c *gin.Context) { if h == nil { c.JSON(http.StatusInternalServerError, gin.H{"error": "handler unavailable"}) @@ -756,6 +762,8 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { if matches && !truncated { if shouldDeferEmptyMainCursorToRotated(files, cursor) { deferEmptyMainMatch = true + } else if shouldResetAmbiguousEmptyMainCursor(files, index, cursor) { + return 0, false, nil } else { return index, true, nil } @@ -804,6 +812,32 @@ func shouldDeferEmptyMainCursorToRotated(files []string, cursor logCursor) bool return false } +func shouldResetAmbiguousEmptyMainCursor(files []string, mainIndex int, cursor logCursor) bool { + if cursor.File != defaultLogFileName || cursor.Offset != 0 || cursor.Size != 0 { + return false + } + info, errStat := os.Stat(files[mainIndex]) + if errStat != nil || info.IsDir() { + return false + } + if info.Size() == cursor.Size && info.ModTime().UnixNano() == cursorModTimeUnixNano(cursor) { + return false + } + for i := range files { + if i == mainIndex || filepath.Base(files[i]) == defaultLogFileName { + continue + } + rotatedInfo, errRotated := os.Stat(files[i]) + if errRotated != nil || rotatedInfo.IsDir() || rotatedInfo.Size() == 0 { + continue + } + if !logFileChangedAfterCursor(files[i], cursor) { + return true + } + } + return false +} + func logFileChangedAfterCursor(path string, cursor logCursor) bool { info, errStat := os.Stat(path) if errStat != nil || info.IsDir() || info.Size() == 0 { diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index 34a07c7e..f803b2cf 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -474,6 +474,53 @@ func TestGetLogsZeroOffsetCursorWithEmptyFileReadsAcrossRotation(t *testing.T) { } } +func TestGetLogsZeroOffsetCursorWithEmptyFileResetsWhenRotationModTimeAmbiguous(t *testing.T) { + dir := t.TempDir() + mainPath := filepath.Join(dir, defaultLogFileName) + fixedModTime := time.Date(2026, 6, 15, 10, 0, 0, 0, time.Local) + writeMainLog(t, dir, "") + if err := os.Chtimes(mainPath, fixedModTime, fixedModTime); err != nil { + t.Fatalf("set initial main mtime: %v", err) + } + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + cursor, errCursor := decodeLogCursor(initial.NextCursor) + if errCursor != nil { + t.Fatalf("decode initial cursor: %v", errCursor) + } + if cursor.Offset != 0 || cursor.Size != 0 { + t.Fatalf("cursor offset/size = %d/%d, want empty zero offset", cursor.Offset, cursor.Size) + } + + first := "[2026-06-15 10:00:01] first" + second := "[2026-06-15 10:00:02] second" + appendMainLog(t, dir, first+"\n") + if err := os.Chtimes(mainPath, fixedModTime, fixedModTime); err != nil { + t.Fatalf("set rotated mtime: %v", err) + } + if err := os.Rename(mainPath, filepath.Join(dir, defaultLogFileName+".1")); err != nil { + t.Fatalf("rotate main log: %v", err) + } + writeMainLog(t, dir, second+"\n") + if err := os.Chtimes(mainPath, fixedModTime, fixedModTime); err != nil { + t.Fatalf("set new main mtime: %v", err) + } + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=2") + wantLines := []string{first, second} + if !reflect.DeepEqual(resp.Lines, wantLines) { + t.Fatalf("lines = %#v, want %#v", resp.Lines, wantLines) + } + if !resp.CursorReset { + t.Fatal("cursor-reset = false, want true for ambiguous empty cursor rotation") + } + if resp.LineCount != len(wantLines) { + t.Fatalf("line-count = %d, want returned line count %d", resp.LineCount, len(wantLines)) + } +} + func TestGetLogsInvalidCursorResetsToTail(t *testing.T) { dir := t.TempDir() lines := []string{ -- 2.51.2 From 0b21b0711523448be1432af5a2e0cc148f01996e Mon Sep 17 00:00:00 2001 From: LTbinglingfeng Date: Mon, 15 Jun 2026 18:37:21 +0800 Subject: [PATCH 10/10] fix log cursor rotation gap --- internal/api/handlers/management/logs.go | 27 +++++++- internal/api/handlers/management/logs_test.go | 65 +++++++++++++++++++ 2 files changed, 89 insertions(+), 3 deletions(-) diff --git a/internal/api/handlers/management/logs.go b/internal/api/handlers/management/logs.go index ad15a741..b6de20e6 100644 --- a/internal/api/handlers/management/logs.go +++ b/internal/api/handlers/management/logs.go @@ -773,13 +773,34 @@ func locateLogCursorFile(files []string, cursor logCursor) (int, bool, error) { if cursor.File != defaultLogFileName || (cursor.Offset == 0 && cursor.Size == 0 && !deferEmptyMainMatch) { return 0, false, nil } + if cursor.Offset == 0 && cursor.Size == 0 { + for i := range files { + if filepath.Base(files[i]) == defaultLogFileName { + continue + } + if !logFileChangedAfterCursor(files[i], cursor) { + continue + } + matches, truncated, errMatch := logFileMatchesCursor(files[i], cursor) + if errMatch != nil { + if errors.Is(errMatch, os.ErrNotExist) { + continue + } + return 0, false, errMatch + } + if truncated { + continue + } + if matches { + return i, true, nil + } + } + return 0, false, nil + } for i := len(files) - 1; i >= 0; i-- { if filepath.Base(files[i]) == defaultLogFileName { continue } - if cursor.Offset == 0 && cursor.Size == 0 && !logFileChangedAfterCursor(files[i], cursor) { - continue - } matches, truncated, errMatch := logFileMatchesCursor(files[i], cursor) if errMatch != nil { if errors.Is(errMatch, os.ErrNotExist) { diff --git a/internal/api/handlers/management/logs_test.go b/internal/api/handlers/management/logs_test.go index f803b2cf..8c3e0ead 100644 --- a/internal/api/handlers/management/logs_test.go +++ b/internal/api/handlers/management/logs_test.go @@ -474,6 +474,71 @@ func TestGetLogsZeroOffsetCursorWithEmptyFileReadsAcrossRotation(t *testing.T) { } } +func TestGetLogsZeroOffsetCursorWithEmptyFileReadsAcrossTwoRotations(t *testing.T) { + dir := t.TempDir() + writeMainLog(t, dir, "") + initial := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?limit=1") + if initial.NextCursor == "" { + t.Fatal("initial next-cursor is empty") + } + cursor, errCursor := decodeLogCursor(initial.NextCursor) + if errCursor != nil { + t.Fatalf("decode initial cursor: %v", errCursor) + } + if cursor.Offset != 0 || cursor.Size != 0 { + t.Fatalf("cursor offset/size = %d/%d, want empty zero offset", cursor.Offset, cursor.Size) + } + + mainPath := filepath.Join(dir, defaultLogFileName) + firstRotatedPath := filepath.Join(dir, defaultLogFileName+".1") + secondRotatedPath := filepath.Join(dir, defaultLogFileName+".2") + firstModTime := time.Unix(0, cursorModTimeUnixNano(cursor)+int64(time.Second)) + secondModTime := time.Unix(0, cursorModTimeUnixNano(cursor)+2*int64(time.Second)) + + appendMainLog(t, dir, "first\n") + if err := os.Chtimes(mainPath, firstModTime, firstModTime); err != nil { + t.Fatalf("update first main log mtime: %v", err) + } + if err := os.Rename(mainPath, firstRotatedPath); err != nil { + t.Fatalf("rotate first main log: %v", err) + } + writeMainLog(t, dir, "second\n") + if err := os.Chtimes(mainPath, secondModTime, secondModTime); err != nil { + t.Fatalf("update second main log mtime: %v", err) + } + if err := os.Rename(firstRotatedPath, secondRotatedPath); err != nil { + t.Fatalf("advance first rotated log: %v", err) + } + if err := os.Rename(mainPath, firstRotatedPath); err != nil { + t.Fatalf("rotate second main log: %v", err) + } + writeMainLog(t, dir, "third\n") + + resp := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(initial.NextCursor)+"&limit=1") + if !reflect.DeepEqual(resp.Lines, []string{"first"}) { + t.Fatalf("lines = %#v, want oldest rotated line", resp.Lines) + } + if resp.CursorReset { + t.Fatal("cursor-reset = true, want false") + } + + next := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(resp.NextCursor)+"&limit=1") + if !reflect.DeepEqual(next.Lines, []string{"second"}) { + t.Fatalf("next lines = %#v, want newer rotated line", next.Lines) + } + if next.CursorReset { + t.Fatal("next cursor-reset = true, want false") + } + + latest := performGetLogs(t, newLogsTestHandler(dir, true), "/v0/management/logs?cursor="+url.QueryEscape(next.NextCursor)+"&limit=1") + if !reflect.DeepEqual(latest.Lines, []string{"third"}) { + t.Fatalf("latest lines = %#v, want main line", latest.Lines) + } + if latest.CursorReset { + t.Fatal("latest cursor-reset = true, want false") + } +} + func TestGetLogsZeroOffsetCursorWithEmptyFileResetsWhenRotationModTimeAmbiguous(t *testing.T) { dir := t.TempDir() mainPath := filepath.Join(dir, defaultLogFileName)