diff --git a/appview/db/strings.go b/appview/db/strings.go index e7d8d8a9..4317476d 100644 --- a/appview/db/strings.go +++ b/appview/db/strings.go @@ -11,7 +11,6 @@ import ( "github.com/bluesky-social/indigo/atproto/syntax" lexutil "github.com/bluesky-social/indigo/lex/util" "github.com/ipfs/go-cid" - "tangled.org/core/api/tangled" "tangled.org/core/appview/models" "tangled.org/core/orm" ) @@ -73,24 +72,13 @@ func AddString(d *DB, s models.String) error { vals := make([]string, len(s.Files)) args := make([]any, 0, len(s.Files)*8) for i, file := range s.Files { - vals[i] = "(?, ?, ?, ?, ?, ?, ?, ?)" - var gzipRealSize *int64 - var gzipRealMime *string - var gzipRealContent *string - if file.Gzip != nil { - gzipRealSize = &file.Gzip.RealSize - gzipRealMime = &file.Gzip.RealMime - gzipRealContent = &file.Gzip.Content - } + vals[i] = "(?, ?, ?, ?, ?)" args = append(args, s.AtUri(), file.Name, file.Content.Ref.String(), file.Content.Size, file.Content.MimeType, - gzipRealSize, - gzipRealMime, - gzipRealContent, ) } _, err = tx.Exec( @@ -100,10 +88,7 @@ func AddString(d *DB, s models.String) error { name, content_ref, content_size, - content_mimetype, - gzip_realsize, - gzip_realmime, - gzip_realcontent + content_mimetype ) values %s`, strings.Join(vals, ","), @@ -255,10 +240,7 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er name, content_ref, content_size, - content_mimetype, - gzip_realsize, - gzip_realmime, - gzip_realcontent + content_mimetype from string_files where at_uri in (%s) order by at_uri, id`, inClause, @@ -275,32 +257,18 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er var file models.String_File var contentRef string - var gzipRealSize sql.Null[int64] - var gzipRealMime, gzipRealContent sql.Null[string] if err := rows.Scan( &stringAt, &file.Name, &contentRef, &file.Content.Size, &file.Content.MimeType, - &gzipRealSize, - &gzipRealMime, - &gzipRealContent, ); err != nil { return nil, fmt.Errorf("failed to execute string_files query: %w", err) } file.Content.Ref = lexutil.LexLink(cid.MustParse(contentRef)) - if gzipRealMime.Valid && gzipRealSize.Valid && gzipRealContent.Valid { - file.Gzip = &models.String_GzipInfo{ - String_File_Gzip: tangled.String_File_Gzip{ - RealMime: gzipRealMime.V, - RealSize: gzipRealSize.V, - }, - Content: gzipRealContent.V, - } - } if s, ok := stringMap[stringAt]; ok { s.Files = append(s.Files, file) } diff --git a/appview/ingester.go b/appview/ingester.go index 685147f4..7ecdad9a 100644 --- a/appview/ingester.go +++ b/appview/ingester.go @@ -1,7 +1,6 @@ package appview import ( - "compress/gzip" "context" "database/sql" "encoding/json" @@ -934,36 +933,6 @@ func (i *Ingester) ingestString(ctx context.Context, e *jmodels.Event) error { return err } - g, gctx := errgroup.WithContext(ctx) - for idx, file := range str.Files { - if file.Gzip == nil { - continue - } - g.Go(func() error { - blob, err := i.BlobStore.GetBlob(gctx, str.Did, cid.Cid(file.Content.Ref)) - if err != nil { - return fmt.Errorf("files[%d]: failed to fetch blob: %w", idx, err) - } - defer blob.Close() - gzr, err := gzip.NewReader(blob) - if err != nil { - return fmt.Errorf("files[%d]: invalid gzip stream: %w", idx, err) - } - gzr.Close() - - content, err := io.ReadAll(gzr) - if err != nil { - return fmt.Errorf("files[%d]: failed to read blob: %w", idx, err) - } - file.Gzip.Content = string(content) - str.Files[idx] = file - return nil - }) - } - if err := g.Wait(); err != nil { - return err - } - if err = db.AddString(i.Db, str); err != nil { l.Error("failed to add string", "err", err) return err diff --git a/appview/models/string.go b/appview/models/string.go index 9711944a..c1b174bf 100644 --- a/appview/models/string.go +++ b/appview/models/string.go @@ -41,13 +41,6 @@ type String struct { type String_File struct { Name string Content lexutil.LexBlob - Gzip *String_GzipInfo -} -type String_GzipInfo struct { - tangled.String_File_Gzip - // Optional uncompressed content. - // Populated when the content is first requested. - Content string } func (s *String) AtUri() syntax.ATURI { @@ -57,17 +50,9 @@ func (s *String) AtUri() syntax.ATURI { func (s *String) AsRecord() *tangled.String { var files []*tangled.String_File for _, f := range s.Files { - var gzip *tangled.String_File_Gzip - if f.Gzip != nil { - gzip = &tangled.String_File_Gzip{ - RealSize: f.Gzip.RealSize, - RealMime: f.Gzip.RealMime, - } - } files = append(files, &tangled.String_File{ Name: f.Name, Content: &f.Content, - Gzip: gzip, }) } return &tangled.String{ @@ -138,14 +123,9 @@ func StringFromRecord(did syntax.DID, rkey syntax.RecordKey, cid syntax.CID, rec } var files []String_File for _, f := range record.Files { - var gzip *String_GzipInfo - if f.Gzip != nil { - gzip = &String_GzipInfo{String_File_Gzip: *f.Gzip} - } files = append(files, String_File{ Name: f.Name, Content: *f.Content, - Gzip: gzip, }) } return String{ diff --git a/appview/strings/strings.go b/appview/strings/strings.go index 7ae30f5c..643e4b4e 100644 --- a/appview/strings/strings.go +++ b/appview/strings/strings.go @@ -39,7 +39,7 @@ import ( lexutil "github.com/bluesky-social/indigo/lex/util" ) -const ApplicationGzip = "application/gzip" +const textPlain = "text/plain" type Strings struct { Db *db.DB @@ -237,26 +237,20 @@ func (s *Strings) SingleString(w http.ResponseWriter, r *http.Request) { } else { files = make([]pages.StringFileFragmentParams, len(str.Files)) for i, file := range str.Files { - var content string - if file.Gzip != nil { - content = file.Gzip.Content - } else { - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) - if err != nil { - l.Warn("failed to fetch blob", "err", err) - http.NotFound(w, r) - return - } - defer blob.Close() + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) + if err != nil { + l.Warn("failed to fetch blob", "err", err) + http.NotFound(w, r) + return + } + defer blob.Close() - contentBytes, err := io.ReadAll(blob) - if err != nil { - l.Error("failed to read blob", "err", err) - } - content = string(contentBytes) + contentBytes, err := io.ReadAll(blob) + if err != nil { + l.Error("failed to read blob", "err", err) } - files[i] = s.makeFileFragmentParams(&str, file.Name, content, false) + files[i] = s.makeFileFragmentParams(&str, file.Name, string(contentBytes), false) } } @@ -312,27 +306,21 @@ func (s *Strings) edit(w http.ResponseWriter, r *http.Request) { } else { files = make([]pages.StringFileEditFragmentParams, len(oldString.Files)) for i, file := range oldString.Files { - var content string - if file.Gzip != nil { - content = file.Gzip.Content - } else { - blob, err := s.BlobStore.GetBlob(r.Context(), oldString.Did, cid.Cid(file.Content.Ref)) - if err != nil { - l.Warn("failed to fetch blob", "err", err) - http.NotFound(w, r) - return - } - defer blob.Close() - - contentBytes, err := io.ReadAll(blob) - if err != nil { - l.Error("failed to read blob", "err", err) - } - content = string(contentBytes) + blob, err := s.BlobStore.GetBlob(r.Context(), oldString.Did, cid.Cid(file.Content.Ref)) + if err != nil { + l.Warn("failed to fetch blob", "err", err) + http.NotFound(w, r) + return + } + defer blob.Close() + + contentBytes, err := io.ReadAll(blob) + if err != nil { + l.Error("failed to read blob", "err", err) } files[i] = pages.StringFileEditFragmentParams{ Name: file.Name, - Content: content, + Content: string(contentBytes), Size: uint64(file.Content.Size), } } @@ -379,7 +367,7 @@ func (s *Strings) edit(w http.ResponseWriter, r *http.Request) { return } - blob, err := xrpc.RepoUploadBlob(ctx, client, gz(content), ApplicationGzip) + blob, err := xrpc.RepoUploadBlob(ctx, client, strings.NewReader(content), textPlain) if err != nil { fail("Failed to create record.", err) return @@ -392,13 +380,6 @@ func (s *Strings) edit(w http.ResponseWriter, r *http.Request) { { Name: filename, Content: *blob.Blob, - Gzip: &models.String_GzipInfo{ - String_File_Gzip: tangled.String_File_Gzip{ - RealMime: "text/plain", - RealSize: int64(len(content)), - }, - Content: content, - }, }, } @@ -492,7 +473,7 @@ func (s *Strings) create(w http.ResponseWriter, r *http.Request) { return } - blob, err := xrpc.RepoUploadBlob(ctx, client, gz(content), ApplicationGzip) + blob, err := xrpc.RepoUploadBlob(ctx, client, strings.NewReader(content), textPlain) if err != nil { fail("Failed to create record.", err) return @@ -507,13 +488,6 @@ func (s *Strings) create(w http.ResponseWriter, r *http.Request) { { Name: filename, Content: *blob.Blob, - Gzip: &models.String_GzipInfo{ - String_File_Gzip: tangled.String_File_Gzip{ - RealMime: "text/plain", - RealSize: int64(len(content)), - }, - Content: content, - }, }, }, Created: time.Now(), @@ -638,24 +612,18 @@ func (s *Strings) FileRaw(w http.ResponseWriter, r *http.Request) { mimeType := file.Content.MimeType size := file.Content.Size - var reader io.Reader - if file.Gzip != nil { - reader = strings.NewReader(file.Gzip.Content) - } else { - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) - if err != nil { - l.Warn("failed to fetch blob", "err", err) - http.NotFound(w, r) - return - } - defer blob.Close() - reader = blob + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) + if err != nil { + l.Warn("failed to fetch blob", "err", err) + http.NotFound(w, r) + return } + defer blob.Close() w.Header().Set("Content-Type", mimeType) w.Header().Set("Content-Disposition", fmt.Sprintf("inline; filename=%q", filename)) w.Header().Set("Content-Length", strconv.FormatInt(size, 10)) - if _, err := io.Copy(w, reader); err != nil { + if _, err := io.Copy(w, blob); err != nil { l.Error("failed to write raw response", "err", err) } } @@ -724,26 +692,20 @@ func (s *Strings) FileFragment(w http.ResponseWriter, r *http.Request) { return } - var content string - if file.Gzip != nil && file.Gzip.Content != "" { - content = file.Gzip.Content - } else { - blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) - if err != nil { - l.Warn("failed to fetch blob", "err", err) - http.NotFound(w, r) - return - } - defer blob.Close() + blob, err := s.BlobStore.GetBlob(r.Context(), str.Did, cid.Cid(file.Content.Ref)) + if err != nil { + l.Warn("failed to fetch blob", "err", err) + http.NotFound(w, r) + return + } + defer blob.Close() - contentBytes, err := io.ReadAll(blob) - if err != nil { - l.Error("failed to read blob", "err", err) - } - content = string(contentBytes) + contentBytes, err := io.ReadAll(blob) + if err != nil { + l.Error("failed to read blob", "err", err) } - params = s.makeFileFragmentParams(&str, file.Name, content, forceCode) + params = s.makeFileFragmentParams(&str, file.Name, string(contentBytes), forceCode) } s.Pages.StringFileFragment(w, params) }