diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go index fbea2be4..5b24fc47 100644 --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -98,6 +98,7 @@ func knotIngester(d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, not } } +// TODO(boltless): remove this. knotmirror should do all sort of indexing func ingestRefUpdate(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, pc posthog.Client, notifier notify.Notifier, dev bool, c *config.Config, cfClient *cloudflare.Client, source ec.Source, msg ec.Message) error { logger := log.FromContext(ctx) diff --git a/knotmirror/db/db.go b/knotmirror/db/db.go index 24540c24..ae318932 100644 --- a/knotmirror/db/db.go +++ b/knotmirror/db/db.go @@ -69,10 +69,48 @@ func Make(ctx context.Context, dbUrl string, maxConns int) (*sql.DB, error) { constraint hosts_pkey primary key (hostname) ); + -- repo indices + create table if not exists repo_languages ( + repo_did text not null, -- repo identifier + commit text not null, -- commit hash + language text not null, + size integer not null check (size >= 0), + + constraint repo_languages_pkey + primary key (repo_did, commit, language) + ); + create table if not exists commits ( + repo_did text not null, -- repo identifier + commit text not null, -- commit hash + date timestamptz not null, -- commit date + + constraint commits_pkey + primary key (repo_did, commit) + ); + create table if not exists commit_signatures ( + repo_did text not null, -- repo identifier + commit text not null, -- commit hash + name text not null, -- user name + email text not null, -- user email + reason text not null, -- 'author'/'commiter'/'co-author'/'signed-off-by' + + constraint commits_emails_pkey + primary key (repo_did, commit, email), + constraint commits_emails_fk_commit + foreign key (repo_did, commit) + references commits (repo_did, commit) + on delete cascade + ); + create index if not exists idx_repos_aturi on repos (at_uri); create index if not exists idx_repos_db_updated_at on repos (db_updated_at desc); create index if not exists idx_hosts_db_updated_at on hosts (db_updated_at desc); + create index if not exists idx_repo_languages_repo_commit + on repo_languages (repo_did, commit); + create index if not exists idx_commits_emails_email_repo_commit + on commits_emails (email, repo_did, commit); + create or replace function set_updated_at() returns trigger as $$ begin diff --git a/knotmirror/db/repo_index.go b/knotmirror/db/repo_index.go new file mode 100644 index 00000000..7590b7aa --- /dev/null +++ b/knotmirror/db/repo_index.go @@ -0,0 +1,33 @@ +package db + +import ( + "context" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/api/tangled" +) + +func IsLanguageIndexed(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) (bool, error) { + panic("unimplemented") +} + +func InsertLanguages(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash, langs []tangled.GitTempListLanguages_Language) error { + panic("unimplemented") +} + +func ListLanguages(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) ([]*tangled.GitTempListLanguages_Language, error) { + panic("unimplemented") +} + +// TODO: use struct for an email. to represent as (email, reason) pair. +// where reason can be "committer", "author", "co-author", "signed-off" + +func IsCommitIndexed(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash) (bool, error) { + panic("unimplemented") +} + +// TODO: include authored/committed dates alongside with author/committer emails +func InsertCommit(ctx context.Context, repoDid syntax.DID, commit plumbing.Hash, emails []string) error { + panic("unimplemented") +} diff --git a/knotmirror/indexer.go b/knotmirror/indexer.go index 5e5f1a7c..8af96b64 100644 --- a/knotmirror/indexer.go +++ b/knotmirror/indexer.go @@ -1,5 +1,24 @@ package knotmirror +import ( + "context" + "fmt" + "io" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-enry/go-enry/v2" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/knotmirror/db" +) + +const LangIndexCap = 16*1024 // 16KB + +func repoPath(uri syntax.DID) string { + panic("unimplemented") +} // HEAD -> store to db // other -> on-demand calculation, cache @@ -24,3 +43,147 @@ package knotmirror // 1. resolve current HEAD // 2. index language stats in current HEAD // 3. store db (ofc, cache it too) + +func IndexLanguages(ctx context.Context, repo syntax.DID, commit plumbing.Hash) (map[string]int64, error) { + gr, err := git.PlainOpen(repoPath(repo)) + if err != nil { + panic("unimplemented") + } + c, err := gr.CommitObject(commit) + if err != nil { + panic("unimplemented") + } + + // check if (uri,tree) is already indexed + indexed, err := db.IsLanguageIndexed(ctx, repo, c.TreeHash) + if err != nil { + panic("unimplemented") + } + if indexed { + return nil, nil + } + + tree, err := c.Tree() + if err != nil { + panic("unimplemented") + } + + sizes := make(map[string]int64) + + err = tree.Files().ForEach(func(file *object.File) error { + // skip binary file + if isbin, _ := file.IsBinary(); isbin { + return nil + } + + // skip large file + if file.Size > LangIndexCap { + return nil + } + + reader, err := file.Reader() + if err != nil { + return err + } + defer reader.Close() + content, err := io.ReadAll(reader) + if err != nil { + return nil + } + + // skip generated file + // TODO: follow gitattributes + if enry.IsGenerated(file.Name, content) || + strings.HasSuffix(file.Name, "bun.lock") { + return nil + } + + language := analyzeLanguage(file.Name, content) + if group := enry.GetLanguageGroup(language); group != "" { + language = group + } + + langType := enry.GetLanguageType(language) + if langType != enry.Programming && langType != enry.Markup && langType != enry.Unknown { + return nil + } + + sizes[language] += file.Size + + return nil + }) + if err != nil { + panic("unimplemented") + } + + panic("unimplemented") +} + +func analyzeLanguage(path string, content []byte) string { + language, ok := enry.GetLanguageByExtension(path) + if ok { + return language + } + + language, ok = enry.GetLanguageByFilename(path) + if ok { + return language + } + + if len(content) == 0 { + return enry.OtherLanguage + } + + return enry.GetLanguage(path, content) +} + +func IndexCommits(ctx context.Context, repo syntax.DID, commit plumbing.Hash) error { + // TODO: index (uri,commit-hash)->{date,[]emails} + // start from given commit, walk through DAG + // for each commits, check if we've indexed it & perform index + // 1. has (uri,commit) been indexed? + // 2. index it + gr, err := git.PlainOpen(repoPath(repo)) + if err != nil { + panic("unimplemented") + } + + _, err = indexCommits(ctx, gr, repo, commit) + if err != nil { + panic("unimplemented") + } + + panic("unimplemented") +} + +// indexCommits indexes a commit and all of its ancestors +func indexCommits(ctx context.Context, gitrepo *git.Repository, repo syntax.DID, commit plumbing.Hash) (int, error) { + isIndexed, err := db.IsCommitIndexed(ctx, repo, commit) + if err != nil { + panic("unimplemented") + } + if isIndexed { + return 0, nil + } + + c, err := gitrepo.CommitObject(commit) + if err != nil { + return 0, fmt.Errorf("commit '%s' not found", commit) + } + + emails := []string{c.Committer.Email} + + if err := db.InsertCommit(ctx, repo, commit, emails); err != nil { + return 0, fmt.Errorf("db: failed to insert commit '%s'", commit) + } + + var count int + for _, pc := range c.ParentHashes { + count, err = indexCommits(ctx, gitrepo, repo, pc) + if err != nil { + return 1, err + } + } + + return 1 + count, nil +} diff --git a/knotmirror/xrpc/git_list_languages.go b/knotmirror/xrpc/git_list_languages.go index c63455e8..f61c1d4e 100644 --- a/knotmirror/xrpc/git_list_languages.go +++ b/knotmirror/xrpc/git_list_languages.go @@ -11,6 +11,8 @@ import ( "github.com/bluesky-social/indigo/atproto/atclient" "github.com/bluesky-social/indigo/atproto/syntax" "tangled.org/core/api/tangled" + "tangled.org/core/knotmirror/db" + "tangled.org/core/knotmirror/xrpc/gitea" "tangled.org/core/knotserver/git" ) @@ -34,7 +36,29 @@ func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { return } - if val, err := x.rdb.Get(r.Context(), fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil { + ctx := r.Context() + + // TODO: wrap everything below with x.indexer.ListLanguages (include db fetch & cache handling) + + repoPath, err := x.makeRepoPath(ctx, repo) + if err != nil { + panic("unimplemented") + } + commit, err := gitea.GetCommit(ctx, repoPath, ref) + if err != nil { + panic("unimplemented") + } + + // serve from pre-indexed stats + if langs, err := db.ListLanguages(ctx, repo, commit.Hash); err == nil { + writeJson(w, http.StatusOK, &tangled.GitTempListLanguages_Output{ + Ref: "", + Total: 0, // do we even need a total? + Languages: langs, + }) + } + + if val, err := x.rdb.Get(ctx, fmt.Sprintf(RepoLanguagesByDid, repo, ref)).Result(); err == nil { l.Debug("served from cache") var langs []*tangled.GitTempListLanguages_Language err = json.Unmarshal([]byte(val), &langs) @@ -48,7 +72,7 @@ func (x *Xrpc) ListLanguages(w http.ResponseWriter, r *http.Request) { } var out *tangled.GitTempListLanguages_Output - out, err = x.listLanguages(r.Context(), repo, ref) + out, err = x.listLanguages(ctx, repo, ref) if err != nil { l.Warn("local mirror failed, trying proxy", "err", err) if x.proxyToKnot(w, r, repo) {