diff --git a/pkg/cmd/streamplace.go b/pkg/cmd/streamplace.go index 6e1cf4ba..88c6e04b 100644 --- a/pkg/cmd/streamplace.go +++ b/pkg/cmd/streamplace.go @@ -399,9 +399,11 @@ func runMain(ctx context.Context, build *config.BuildFlags, platformJobs []jobFu return storage.StartSegmentCleaner(ctx, ldb, cli) }) - group.Go(func() error { - return ldb.StartSegmentCleaner(ctx) - }) + if cli.LegacySegmentCleaner { + group.Go(func() error { + return ldb.StartSegmentCleaner(ctx) + }) + } group.Go(func() error { return replicator.Start(ctx, cli) diff --git a/pkg/config/config.go b/pkg/config/config.go index 6a37d957..3df0d7ce 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -137,6 +137,8 @@ type CLI struct { DisableIrohRelay bool DevAccountCreds map[string]string StreamSessionTimeout time.Duration + LegacySegmentCleaner bool + SegmentArchiveRetention time.Duration Replicators []string WebsocketURL string BehindHTTPSProxy bool @@ -735,6 +737,20 @@ func (cli *CLI) NewCommand(name string) *urfavecli.Command { Destination: &cli.StreamSessionTimeout, Sources: urfavecli.EnvVars("SP_STREAM_SESSION_TIMEOUT"), }, + &urfavecli.BoolFlag{ + Name: "legacy-segment-cleaner", + Usage: "re-enable the legacy segment cleaner. shouldn't be needed but can be useful in cases where localdb is too big.", + Value: false, + Destination: &cli.LegacySegmentCleaner, + Sources: urfavecli.EnvVars("SP_LEGACY_SEGMENT_CLEANER"), + }, + &urfavecli.DurationFlag{ + Name: "segment-archive-retention", + Usage: "for users who don't specify a distribution policy, how long to keep segments around?", + Value: 24 * time.Hour, + Destination: &cli.SegmentArchiveRetention, + Sources: urfavecli.EnvVars("SP_SEGMENT_ARCHIVE_RETENTION"), + }, &urfavecli.StringFlag{ Name: "replicators", Usage: "comma-separated list of replication protocols to use (websocket, iroh)", @@ -850,6 +866,7 @@ func (cli *CLI) NewCommand(name string) *urfavecli.Command { Destination: &cli.MistHTTPPort, Sources: urfavecli.EnvVars("SP_MIST_HTTP_PORT"), }) + } LivepeerFlagSet = flag.NewFlagSet("livepeer", flag.ContinueOnError) diff --git a/pkg/media/validate.go b/pkg/media/validate.go index 871ab4ed..909606f7 100644 --- a/pkg/media/validate.go +++ b/pkg/media/validate.go @@ -114,8 +114,18 @@ func (mm *MediaManager) ValidateMP4(ctx context.Context, input io.Reader, local } var deleteAfter *time.Time if meta.DistributionPolicy != nil && meta.DistributionPolicy.DeleteAfterSeconds != nil { - expiryTime := meta.StartTime.Time().Add(time.Duration(*meta.DistributionPolicy.DeleteAfterSeconds) * time.Second) - deleteAfter = &expiryTime + secs := *meta.DistributionPolicy.DeleteAfterSeconds + if secs == -1 { + deleteAfter = nil + } else { + expiryTime := meta.StartTime.Time().Add(time.Duration(secs) * time.Second) + deleteAfter = &expiryTime + } + } else { + if mm.cli.SegmentArchiveRetention.Seconds() != 0 { + tomorrow := time.Now().Add(mm.cli.SegmentArchiveRetention).UTC() + deleteAfter = &tomorrow + } } seg := &localdb.Segment{ ID: *label, diff --git a/pkg/storage/storage.go b/pkg/storage/storage.go index fe28ec8b..15535071 100644 --- a/pkg/storage/storage.go +++ b/pkg/storage/storage.go @@ -49,7 +49,7 @@ func StartSegmentCleaner(ctx context.Context, localDB localdb.LocalDB, cli *conf func deleteSegment(ctx context.Context, localDB localdb.LocalDB, cli *config.CLI, seg localdb.Segment) error { if time.Since(seg.StartTime) < moderationRetention { - log.Debug(ctx, "Skipping deletion of segment", "id", seg.ID, "time since start", time.Since(seg.StartTime)) + log.Debug(ctx, "Skipping deletion of segment for moderation retention", "id", seg.ID, "time since start", time.Since(seg.StartTime)) return nil } aqt := aqtime.FromTime(seg.StartTime)