diff --git a/flake.nix b/flake.nix index e550bb5..f9dc9e3 100644 --- a/flake.nix +++ b/flake.nix @@ -19,7 +19,7 @@ name = "lazuli"; version = "0.1.7"; src = pkgs.nix-gitignore.gitignoreSource [ "*.csv" "*.zip" "*.json" ] ./.; - vendorHash = "sha256-O6R8jC8Ms5gsY2FUmuL8lTGTODfMW1CsSWuWbN27zeY="; + vendorHash = "sha256-KnWoZ5UK8eigYw5uMSsLu4DIhzkSXmVHaE51Mr6hFmA="; ldflags = [ "-X" "main.Version=${version}" diff --git a/go.mod b/go.mod index ef02765..5da1891 100644 --- a/go.mod +++ b/go.mod @@ -7,7 +7,6 @@ require ( github.com/failsafe-go/failsafe-go v0.9.5 github.com/urfave/cli/v3 v3.6.2 go.etcd.io/bbolt v1.4.3 - golang.org/x/text v0.33.0 ) require ( @@ -26,6 +25,7 @@ require ( gitlab.com/yawning/tuplehash v0.0.0-20230713102510-df83abbf9a02 // indirect go.yaml.in/yaml/v2 v2.4.3 // indirect golang.org/x/crypto v0.47.0 // indirect + golang.org/x/sync v0.19.0 // indirect golang.org/x/sys v0.40.0 // indirect golang.org/x/time v0.14.0 // indirect google.golang.org/protobuf v1.36.11 // indirect diff --git a/sync/record.go b/sync/record.go index 5fca73f..f7bfc66 100644 --- a/sync/record.go +++ b/sync/record.go @@ -7,7 +7,6 @@ import ( "unicode" "github.com/bluesky-social/indigo/atproto/syntax" - "golang.org/x/text/unicode/norm" ) type ExistingRecord struct { @@ -24,29 +23,12 @@ func normalizeString(s string) string { result.Grow(len(s)) for _, r := range s { - if unicode.Is(unicode.Mn, r) { - continue - } - decomposed := norm.NFKD.String(string(r)) - if len(decomposed) > 0 { - result.WriteByte(decomposed[0]) - } else { + if r >= 128 || unicode.IsLetter(r) || unicode.IsNumber(r) { result.WriteRune(r) } } - s = result.String() - s = strings.Map(func(r rune) rune { - if r >= 128 { - return r - } - if unicode.IsLetter(r) || unicode.IsNumber(r) { - return r - } - return -1 - }, s) - - return s + return result.String() } type PlayRecord struct { @@ -61,6 +43,9 @@ type PlayRecord struct { RecordingMbId string `json:"recordingMbId,omitempty"` OriginUrl string `json:"originUrl"` MsPlayed int `json:"msPlayed,omitempty"` + + normalizedTrack string `json:"-"` + normalizedArtist string `json:"-"` } func (r PlayRecord) ArtistName() string { @@ -71,11 +56,23 @@ func (r PlayRecord) ArtistName() string { } func (r PlayRecord) normalizeArtist() string { - return normalizeString(r.ArtistName()) + if r.normalizedArtist != "" { + return r.normalizedArtist + } + + r.normalizedArtist = normalizeString(r.ArtistName()) + + return r.normalizedArtist } func (r PlayRecord) normalizeTrack() string { - return normalizeString(r.TrackName) + if r.normalizedTrack != "" { + return r.normalizedTrack + } + + r.normalizedTrack = normalizeString(r.TrackName) + + return r.normalizedTrack } func (r PlayRecord) hasMBID() bool { @@ -84,6 +81,7 @@ func (r PlayRecord) hasMBID() bool { return true } } + return r.RecordingMbId != "" } @@ -92,17 +90,7 @@ func (r PlayRecord) isLastFM() bool { } func (r PlayRecord) betterThan(other PlayRecord) bool { - if r.isLastFM() && !other.isLastFM() { - return true - } - if !r.isLastFM() && other.isLastFM() { - return false - } - // Both same source, prefer the one with MBID - if r.hasMBID() && !other.hasMBID() { - return true - } - return false + return (r.hasMBID() && !other.hasMBID()) || (r.isLastFM() && !other.isLastFM()) } func (r PlayRecord) IsDuplicate(other PlayRecord, tolerance time.Duration) (bool, bool) { @@ -118,10 +106,7 @@ func (r PlayRecord) sameAs(other PlayRecord, tolerance time.Duration) bool { } diff := r.PlayedTime.Sub(other.PlayedTime.Time) - if diff < 0 { - diff = -diff - } - return diff <= tolerance + return max(diff, -diff) <= tolerance } func (r PlayRecord) Time() time.Time { diff --git a/sync/record_test.go b/sync/record_test.go index 832ddc2..8995b6a 100644 --- a/sync/record_test.go +++ b/sync/record_test.go @@ -1,6 +1,7 @@ package sync import ( + "fmt" "testing" "time" @@ -438,3 +439,35 @@ func TestMergeRecordsComprehensive(t *testing.T) { }) } } + +func BenchmarkMergeRecords(b *testing.B) { + // Generate test data with multiple sources and items + numSources := 10 + itemsPerSource := 1000 + tolerance := 10 * time.Minute + baseTime := time.Date(2024, 1, 15, 10, 0, 0, 0, time.UTC) + + sources := make([][]PlayRecord, numSources) + for i := range numSources { + sources[i] = make([]PlayRecord, itemsPerSource) + for j := range itemsPerSource { + sources[i][j] = PlayRecord{ + Type: "app.bsky.feed.post", + TrackName: fmt.Sprintf("Song %d", (i+j)%100), + Artists: []PlayRecordArtist{{ArtistName: fmt.Sprintf("Artist %d", i%20)}}, + PlayedTime: Timestamp{Time: baseTime.Add(time.Duration(i*itemsPerSource+j) * time.Minute)}, + SubmissionClientAgent: DefaultClientAgent, + MusicServiceBaseDomain: []string{MusicServiceLastFM, MusicServiceSpotify}[i%2], + OriginUrl: "https://example.com", + MsPlayed: 180000, + } + if (i+j)%3 == 0 { + sources[i][j].RecordingMbId = "mbid-123" + } + } + } + + for b.Loop() { + kway.Merge(sources, tolerance) + } +}