diff --git a/appview/db/db.go b/appview/db/db.go index 1efa5706..052c421f 100644 --- a/appview/db/db.go +++ b/appview/db/db.go @@ -2094,6 +2094,64 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { return err }) + orm.RunMigration(conn, logger, "multiple-files-for-a-string", func(tx *sql.Tx) error { + _, err := tx.Exec(` + create table strings_new ( + did text not null, + rkey text not null, + at_uri text generated always as ('at://' || did || '/' || 'sh.tangled.string' || '/' || rkey) stored unique, + cid text, + + title text, + description text, + created text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + file_name text not null default '', + file_content text not null default '', + + -- appview-local information + edited text, + + primary key (did, rkey) + ); + `) + if err != nil { + return fmt.Errorf("failed to create tables: %w", err) + } + + _, err = tx.Exec(` + insert into strings_new ( + did, rkey, cid, + title, + description, + file_name, + file_content, + created, + edited + ) + select + did, rkey, cid, + filename, + description, + filename, + content, + created, + edited + from strings; + `) + if err != nil { + return fmt.Errorf("failed to insert strings: %w", err) + } + + _, err = tx.Exec(` + drop table strings; + alter table strings_new rename to strings; + `) + if err != nil { + return fmt.Errorf("failed to drop legacy table: %w", err) + } + return nil + }) + return &DB{ db, logger, diff --git a/appview/db/strings.go b/appview/db/strings.go index 3cffb710..2a4a4baa 100644 --- a/appview/db/strings.go +++ b/appview/db/strings.go @@ -4,6 +4,7 @@ import ( "database/sql" "errors" "fmt" + "slices" "strings" "time" @@ -12,40 +13,74 @@ import ( "tangled.org/core/orm" ) -func AddString(e Execer, s models.String) error { - _, err := e.Exec( +func AddString(d *DB, s models.String) error { + tx, err := d.Begin() + if err != nil { + return fmt.Errorf("starting transaction: %w", err) + } + defer tx.Rollback() + res, err := tx.Exec( `insert into strings ( did, rkey, cid, - filename, + title, description, - content, - created, - edited + file_name, + file_content, + created ) - values (?, ?, ?, ?, ?, ?, ?, null) + values (?, ?, ?, ?, ?, ?, ?, ?) on conflict(did, rkey) do update set cid = excluded.cid, - filename = excluded.filename, + title = excluded.title, description = excluded.description, - content = excluded.content, + file_name = excluded.file_name, + file_content= excluded.file_content, + created = excluded.created, edited = case when strings.cid is not null then ? else strings.edited end where strings.cid is not excluded.cid`, s.Did, s.Rkey, s.Cid, - s.Filename, + s.Title, s.Description, - s.Contents, + s.FileName, + s.FileContent, s.Created.Format(time.RFC3339), time.Now().Format(time.RFC3339), ) - return err + if err != nil { + return fmt.Errorf("inserting string: %w", err) + } + + num, err := res.RowsAffected() + if err != nil { + return fmt.Errorf("calculating affected rows: %w", err) + } + if num == 0 { + return nil + } + + if err := tx.Commit(); err != nil { + return fmt.Errorf("commiting transaction: %w", err) + } + return nil +} + +func GetString(e Execer, filters ...orm.Filter) (models.String, error) { + strings, err := GetStrings(e, 0, filters...) + if err != nil { + return models.String{}, err + } + if len(strings) != 1 { + return models.String{}, sql.ErrNoRows + } + return strings[0], nil } func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, error) { - var all []models.String + stringMap := make(map[syntax.ATURI]*models.String) var conditions []string var args []any @@ -64,13 +99,15 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er limitClause = fmt.Sprintf(" limit %d ", limit) } - query := fmt.Sprintf(`select + query := fmt.Sprintf( + `select did, rkey, cid, - filename, + title, description, - content, + file_name, + file_content, created, edited from strings @@ -91,15 +128,16 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er for rows.Next() { var s models.String var createdAt string - var cid, editedAt sql.Null[string] + var cid, title, description, editedAt sql.Null[string] if err := rows.Scan( &s.Did, &s.Rkey, &cid, - &s.Filename, - &s.Description, - &s.Contents, + &title, + &description, + &s.FileName, + &s.FileContent, &createdAt, &editedAt, ); err != nil { @@ -111,6 +149,16 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er *s.Cid = syntax.CID(cid.V) } + if title.Valid { + s.Title = new(string) + *s.Title = title.V + } + + if description.Valid { + s.Description = new(string) + *s.Description = description.V + } + s.Created, err = time.Parse(time.RFC3339, createdAt) if err != nil { s.Created = time.Now() @@ -124,13 +172,112 @@ func GetStrings(e Execer, limit int, filters ...orm.Filter) ([]models.String, er s.Edited = &e } - all = append(all, s) + s.Stats = &models.StringStats{} + stringMap[s.AtUri()] = &s } if err := rows.Err(); err != nil { return nil, err } + // if no strings, return early + if len(stringMap) == 0 { + return nil, nil + } + + // build IN clause for related queries + inClause := strings.TrimSuffix(strings.Repeat("?, ", len(stringMap)), ", ") + args = make([]any, len(stringMap)) + i := 0 + for _, s := range stringMap { + args[i] = s.AtUri() + i++ + } + + // // get files + // { + // rows, err := e.Query( + // fmt.Sprintf( + // `select at_uri, name, blob from string_files where at_uri in (%s) order by at_uri, id`, + // inClause, + // ), + // args..., + // ) + // if err != nil { + // return nil, fmt.Errorf("failed to execute string_files query: %w", err) + // } + // defer rows.Close() + // + // for rows.Next() { + // var stringAt syntax.ATURI + // var file models.String_File + // file.Blob = &util.LexBlob{} + // var gzipMimeType sql.Null[string] + // var gzipSize sql.Null[int64] + // if err := rows.Scan( + // &stringAt, + // &file.Name, + // &blob, + // ); err != nil { + // return nil, fmt.Errorf("failed to execute string_files query: %w", err) + // } + // if gzipMimeType.Valid && gzipSize.Valid { + // file.Gzip = &models.GzipInfo{ + // MimeType: gzipMimeType.V, + // Size: gzipSize.V, + // } + // } + // if s, ok := stringMap[stringAt]; ok { + // s.Files = append(s.Files, file) + // } + // } + // if err = rows.Err(); err != nil { + // return nil, fmt.Errorf("failed to execute string_files query: %w", err) + // } + // } + + // get star counts + { + rows, err := e.Query( + fmt.Sprintf( + `select subject, count(1) from stars where subject_type = 'string' and subject in (%s) group by subject`, + inClause, + ), + args..., + ) + if err != nil { + return nil, fmt.Errorf("failed to execute star-count query: %w", err) + } + defer rows.Close() + + for rows.Next() { + var stringAt syntax.ATURI + var count int + if err := rows.Scan(&stringAt, &count); err != nil { + continue + } + if s, ok := stringMap[stringAt]; ok { + s.Stats.StarCount = count + } + } + if err = rows.Err(); err != nil { + return nil, fmt.Errorf("failed to execute star-count query: %w", err) + } + } + + var all []models.String + for _, s := range stringMap { + all = append(all, *s) + } + + // sort by created timestamp (desc) + slices.SortFunc(all, func(a, b models.String) int { + if a.Created.After(b.Created) { + return -1 + } + return 1 + }) + return all, nil } diff --git a/appview/ingester.go b/appview/ingester.go index e397fde3..74fcec90 100644 --- a/appview/ingester.go +++ b/appview/ingester.go @@ -112,7 +112,10 @@ func (i *Ingester) Ingest() processFunc { case tangled.RepoNSID: err = i.ingestRepo(ctx, e) } - l = i.Logger.With("nsid", e.Commit.Collection) + l = l.With( + "uri", fmt.Sprintf("at://%s/%s/%s", e.Did, e.Commit.Collection, e.Commit.RKey), + "cid", e.Commit.CID, + ) } if err != nil { @@ -919,9 +922,11 @@ func (i *Ingester) ingestString(e *jmodels.Event) error { return err } - string := models.StringFromRecord(syntax.DID(did), syntax.RecordKey(rkey), syntax.CID(e.Commit.CID), record) - - if err = i.Validator.ValidateString(&string); err != nil { + string, err := models.StringFromRecord(syntax.DID(did), syntax.RecordKey(rkey), syntax.CID(e.Commit.CID), record) + if err != nil { + return fmt.Errorf("failed to parse string record: %w", err) + } + if err = string.Validate(); err != nil { l.Error("invalid record", "err", err) return err } diff --git a/appview/ingester_string_test.go b/appview/ingester_string_test.go index 928e78ba..ecde8bf1 100644 --- a/appview/ingester_string_test.go +++ b/appview/ingester_string_test.go @@ -104,14 +104,17 @@ func TestIngestString_CreateRoundTrip(t *testing.T) { if !ok { t.Fatal("row not inserted") } - if s.Filename != "hello.txt" { - t.Errorf("filename = %q, want hello.txt", s.Filename) + if s.FileName != "hello.txt" { + t.Errorf("filename = %q, want hello.txt", s.FileName) } - if s.Description != "a greeting" { - t.Errorf("description = %q", s.Description) + if s.Description == nil { + t.Errorf("description = nil") } - if s.Contents != "hello world\n" { - t.Errorf("contents = %q", s.Contents) + if *s.Description != "a greeting" { + t.Errorf("description = %q", *s.Description) + } + if s.FileContent != "hello world\n" { + t.Errorf("contents = %q", s.FileContent) } if !s.Created.Equal(created) { t.Errorf("created = %v, want %v (record CreatedAt must round-trip)", s.Created, created) @@ -145,8 +148,8 @@ func TestIngestString_UpdateBumpsEditedOnContentChange(t *testing.T) { if !ok { t.Fatal("row missing after update") } - if s.Contents != "hello, world!\n" { - t.Errorf("contents = %q, want updated value", s.Contents) + if s.FileContent != "hello, world!\n" { + t.Errorf("contents = %q, want updated value", s.FileContent) } if !s.Created.Equal(created) { t.Errorf("update overwrote created: got %v, want %v", s.Created, created) @@ -216,7 +219,6 @@ func TestIngestString_ValidatorRejects(t *testing.T) { name string rec tangled.String }{ - {"empty contents", tangled.String{Filename: "x", Contents: "", CreatedAt: now}}, {"long filename", tangled.String{Filename: strings.Repeat("a", 141), Contents: "x", CreatedAt: now}}, {"long description", tangled.String{Filename: "x", Description: strings.Repeat("d", 281), Contents: "x", CreatedAt: now}}, } diff --git a/appview/models/string.go b/appview/models/string.go index 2b3075ad..f377fece 100644 --- a/appview/models/string.go +++ b/appview/models/string.go @@ -1,13 +1,13 @@ package models import ( - "bytes" + "errors" "fmt" - "io" - "strings" "time" + "unicode/utf8" "github.com/bluesky-social/indigo/atproto/syntax" + lexutil "github.com/bluesky-social/indigo/lex/util" "tangled.org/core/api/tangled" ) @@ -16,11 +16,36 @@ type String struct { Rkey syntax.RecordKey Cid *syntax.CID - Filename string - Description string - Contents string + // String_File will remain, and we will still use it. + // We just use `FileContent` when fetching the file. + + // after that, change lexicon, start migrating to blobs + // when string is migrated, clear FileName and FileContent. + // when they are cleared, fetch the blob on page load. + + Title *string + Description *string + Files []String_File Created time.Time Edited *time.Time + + // legacy string data + FileName string + FileContent string + + // optionally, populate this when querying for reverse mappings + Stats *StringStats +} + +// TODO: replace this with [tangled.String_File] +type String_File struct { + Name string + Blob *lexutil.LexBlob + Gzip *GzipInfo +} +type GzipInfo struct { + MimeType string + Size int64 } func (s *String) AtUri() syntax.ATURI { @@ -28,70 +53,95 @@ func (s *String) AtUri() syntax.ATURI { } func (s *String) AsRecord() *tangled.String { + var description string + if s.Description != nil { + description = *s.Description + } return &tangled.String{ - Filename: s.Filename, - Description: s.Description, - Contents: s.Contents, + Filename: s.FileName, + Description: description, + Contents: s.FileContent, CreatedAt: s.Created.Format(time.RFC3339), } } -func StringFromRecord(did syntax.DID, rkey syntax.RecordKey, cid syntax.CID, record tangled.String) String { +func (s *String) Validate() error { + var err error + if s.FileName == "" && len(s.Files) == 0 { + err = errors.Join(err, fmt.Errorf("string should have more than one files")) + } + // legacy record check + if utf8.RuneCountInString(s.FileName) > 140 { + err = errors.Join(err, fmt.Errorf("filename too long")) + } + for i, file := range s.Files { + if utf8.RuneCountInString(file.Name) > 140 { + err = errors.Join(err, fmt.Errorf("filename too long at files[%d]", i)) + } + } + if s.Title != nil { + if utf8.RuneCountInString(*s.Title) > 140 { + err = errors.Join(err, fmt.Errorf("title too long")) + } + } + if s.Description != nil { + if utf8.RuneCountInString(*s.Description) > 280 { + err = errors.Join(err, fmt.Errorf("description too long")) + } + } + return err +} + +func (s String) RenderTitle() string { + if s.Title != nil { + return *s.Title + } + if len(s.Files) > 0 { + return s.Files[0].Name + } + return s.FileName +} + +// FileByName returns first item in files with given filename +func (s *String) FileByName(name string) (String_File, bool) { + for _, file := range s.Files { + if file.Name == name { + return file, true + } + } + return String_File{}, false +} + +func (s String) IsLegacySingleFile() bool { + return len(s.Files) == 0 +} + +func StringFromRecord(did syntax.DID, rkey syntax.RecordKey, cid syntax.CID, record tangled.String) (String, error) { created, err := time.Parse(time.RFC3339, record.CreatedAt) if err != nil { - created = time.Now() + return String{}, fmt.Errorf("invalid createdAt: %w", err) + } + var description *string + if record.Description != "" { + description = &record.Description } return String{ Did: did, Rkey: rkey, Cid: &cid, - Filename: record.Filename, - Description: record.Description, - Contents: record.Contents, + Description: description, Created: created, - } + FileName: record.Filename, + FileContent: record.Contents, + }, nil } type StringStats struct { - LineCount uint64 - ByteCount uint64 + StarCount int + // CommentCount int } -func (s String) Stats() StringStats { - lineCount, err := countLines(strings.NewReader(s.Contents)) - if err != nil { - // non-fatal - // TODO: log this? - } - - return StringStats{ - LineCount: uint64(lineCount), - ByteCount: uint64(len(s.Contents)), - } -} - -func countLines(r io.Reader) (int, error) { - buf := make([]byte, 32*1024) - bufLen := 0 - count := 0 - nl := []byte{'\n'} - - for { - c, err := r.Read(buf) - if c > 0 { - bufLen += c - } - count += bytes.Count(buf[:c], nl) - - switch { - case err == io.EOF: - /* handle last line not having a newline at the end */ - if bufLen >= 1 && buf[(bufLen-1)%(32*1024)] != '\n' { - count++ - } - return count, nil - case err != nil: - return 0, err - } - } +type StringFileStats struct { + LineCount int + ByteCount int } diff --git a/appview/pages/pages.go b/appview/pages/pages.go index 8ef90bbe..e8164941 100644 --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -29,7 +29,6 @@ import ( "tangled.org/core/idresolver" "tangled.org/core/types" - "github.com/bluesky-social/indigo/atproto/identity" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/go-git/go-git/v5/plumbing" ) @@ -1556,26 +1555,27 @@ func (p *Pages) Workflow(w io.Writer, params WorkflowParams) error { return p.executeRepo("repo/pipelines/workflow", w, params) } -type PutStringParams struct { +type NewStringParams struct { LoggedInUser *oauth.MultiAccountUser - Action string - - // this is supplied in the case of editing an existing string - String models.String + String models.String + FileParams []StringFileEditFragmentParams } -func (p *Pages) PutString(w io.Writer, params PutStringParams) error { - return p.execute("strings/put", w, params) +func (p *Pages) NewString(w io.Writer, params NewStringParams) error { + // use default string value to render template + params.String = models.String{Files: make([]models.String_File, 1)} + params.FileParams = make([]StringFileEditFragmentParams, 1) + return p.execute("strings/new", w, params) } -type StringsDashboardParams struct { +type EditStringParams struct { LoggedInUser *oauth.MultiAccountUser - Card ProfileCard - Strings []models.String + String models.String + FileParams []StringFileEditFragmentParams } -func (p *Pages) StringsDashboard(w io.Writer, params StringsDashboardParams) error { - return p.execute("strings/dashboard", w, params) +func (p *Pages) EditString(w io.Writer, params EditStringParams) error { + return p.execute("strings/edit", w, params) } type StringTimelineParams struct { @@ -1588,16 +1588,12 @@ func (p *Pages) StringsTimeline(w io.Writer, params StringTimelineParams) error } type SingleStringParams struct { - LoggedInUser *oauth.MultiAccountUser - ShowRendered bool - RenderToggle bool - RenderedContents template.HTML - String *models.String - Stats models.StringStats - IsStarred bool - StarCount int - Owner identity.Identity - CommentList []models.CommentListItem + LoggedInUser *oauth.MultiAccountUser + String *models.String + FileParams []StringFileFragmentParams + IsStarred bool + StarCount int + CommentList []models.CommentListItem Reactions map[syntax.ATURI]map[models.ReactionKind]models.ReactionDisplayData UserReacted map[syntax.ATURI]map[models.ReactionKind]bool @@ -1608,6 +1604,32 @@ func (p *Pages) SingleString(w io.Writer, params SingleStringParams) error { return p.execute("strings/string", w, params) } +type StringFileEditFragmentParams struct { + Name string + Content string + Size uint64 +} + +type StringFileFragmentParams struct { + String *models.String + Name string + Content string + + LineCount int + Size uint64 + HasNoTrailingEOL bool + HasRenderedToggle bool + ShowingRendered bool +} + +func (p *Pages) StringFileFragment(w io.Writer, params StringFileFragmentParams) error { + return p.executePlain("strings/fragments/file", w, params) +} + +func (p *Pages) StringFileEditFragment(w io.Writer) error { + return p.executePlain("strings/fragments/fileEdit", w, StringFileEditFragmentParams{}) +} + type SearchReposParams struct { LoggedInUser *oauth.MultiAccountUser Repos []models.Repo diff --git a/appview/pages/ratchet_test.go b/appview/pages/ratchet_test.go index 635042a0..a5cb1fc4 100644 --- a/appview/pages/ratchet_test.go +++ b/appview/pages/ratchet_test.go @@ -42,9 +42,10 @@ func TestNoRepoRkeyInTemplates(t *testing.T) { } var bareDidAllowlist = map[string]bool{ - "templates/strings/string.html": true, - "templates/strings/fragments/form.html": true, "templates/spindles/dashboard.html": true, + "templates/strings/edit.html": true, + "templates/strings/fragments/file.html": true, + "templates/strings/string.html": true, } var didCloseAsUrlSegment = regexp.MustCompile(`\.(?:Did|OwnerDid)\s*\}\}\s*/`) diff --git a/appview/pages/templates/strings/dashboard.html b/appview/pages/templates/strings/dashboard.html deleted file mode 100644 index 006a6a53..00000000 --- a/appview/pages/templates/strings/dashboard.html +++ /dev/null @@ -1,58 +0,0 @@ -{{ define "title" }}strings by {{ resolve .Card.UserDid }}{{ end }} - -{{ define "extrameta" }} - {{ $handle := resolve .Card.UserDid }} - - - - -{{ end }} - - -{{ define "content" }} -
ALL STRINGS
-This user does not have any strings yet.
- {{ end }} -Edit string
+Create a new string
+Store and share code snippets with ease.
+Create a new string
-Store and share code snippets with ease.
- {{ else }} -Edit string
- {{ end }} -