diff --git a/appview/state/knotstream.go b/appview/state/knotstream.go index 5e38552b..ad94b16b 100644 --- a/appview/state/knotstream.go +++ b/appview/state/knotstream.go @@ -97,6 +97,7 @@ func knotIngester(d *db.DB, enforcer *rbac.Enforcer, posthog posthog.Client, not } } +// TODO(boltless): remove this. knotmirror should do all sort of indexing func ingestRefUpdate(ctx context.Context, d *db.DB, enforcer *rbac.Enforcer, pc posthog.Client, notifier notify.Notifier, dev bool, c *config.Config, cfClient *cloudflare.Client, source ec.Source, msg ec.Message) error { logger := log.FromContext(ctx) diff --git a/knotmirror/db/db.go b/knotmirror/db/db.go index 22eda56e..1f47f410 100644 --- a/knotmirror/db/db.go +++ b/knotmirror/db/db.go @@ -68,10 +68,48 @@ func Make(ctx context.Context, dbUrl string, maxConns int) (*sql.DB, error) { constraint hosts_pkey primary key (hostname) ); + -- repo indices + create table if not exists repo_languages ( + repo_at text not null, -- repo identifier + tree text not null, -- tree hash + language text not null, + bytes integer not null check (bytes >= 0), + + constraint repo_languages_pkey + primary key (repo_at, tree, language) + ); + create table if not exists commits ( + repo_at text not null, -- repo identifier + commit text not null, -- commit hash + date timestamptz not null, -- commit date + + constraint commits_pkey + primary key (repo_at, commit) + ); + create table if not exists commit_signatures ( + repo_at text not null, -- repo identifier + commit text not null, -- commit hash + name text not null, -- user name + email text not null, -- user email + reason text not null, -- 'author'/'commiter'/'co-author'/'signed-off-by' + + constraint commits_emails_pkey + primary key (repo_at, commit, email), + constraint commits_emails_fk_commit + foreign key (repo_at, commit) + references commits (repo_at, commit) + on delete cascade + ); + create index if not exists idx_repos_aturi on repos (at_uri); create index if not exists idx_repos_db_updated_at on repos (db_updated_at desc); create index if not exists idx_hosts_db_updated_at on hosts (db_updated_at desc); + create index if not exists idx_repo_languages_repo_tree + on repo_languages (repo_at, tree); + create index if not exists idx_commits_emails_email_repo_commit + on commits_emails (email, repo_at, commit); + create or replace function set_updated_at() returns trigger as $$ begin diff --git a/knotmirror/db/repo_index.go b/knotmirror/db/repo_index.go new file mode 100644 index 00000000..d961c3c5 --- /dev/null +++ b/knotmirror/db/repo_index.go @@ -0,0 +1,33 @@ +package db + +import ( + "context" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-git/go-git/v5/plumbing" + "tangled.org/core/api/tangled" +) + +func IsLanguageIndexed(ctx context.Context, repoAt syntax.ATURI, tree plumbing.Hash) (bool, error) { + panic("unimplemented") +} + +func InsertLanguages(ctx context.Context, repoAt syntax.ATURI, tree plumbing.Hash, langs []tangled.GitTempListLanguages_Language) error { + panic("unimplemented") +} + +func ListLanguages(ctx context.Context, repoAt syntax.ATURI, tree plumbing.Hash) ([]tangled.GitTempListLanguages_Language, error) { + panic("unimplemented") +} + +// TODO: use struct for an email. to represent as (email, reason) pair. +// where reason can be "committer", "author", "co-author", "signed-off" + +func IsCommitIndexed(ctx context.Context, repoAt syntax.ATURI, commit plumbing.Hash) (bool, error) { + panic("unimplemented") +} + +// TODO: include authored/committed dates alongside with author/committer emails +func InsertCommit(ctx context.Context, repoAt syntax.ATURI, commit plumbing.Hash, emails []string) error { + panic("unimplemented") +} diff --git a/knotmirror/indexer.go b/knotmirror/indexer.go new file mode 100644 index 00000000..71c54b50 --- /dev/null +++ b/knotmirror/indexer.go @@ -0,0 +1,169 @@ +package knotmirror + +import ( + "bytes" + "context" + "fmt" + "io" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-enry/go-enry/v2" + "github.com/go-git/go-git/v5" + "github.com/go-git/go-git/v5/plumbing" + "github.com/go-git/go-git/v5/plumbing/object" + "tangled.org/core/knotmirror/db" +) + +// TODO: run index for common hashes like: +// 1. default branch (HEAD) +// 2. all known branches +// 3. all known tags + +func repoPath(uri syntax.ATURI) string { + panic("unimplemented") +} + +func IndexLanguages(ctx context.Context, repoAt syntax.ATURI, commit plumbing.Hash) error { + // TODO: index (uri,tree-hash[,lang])->bytes + // 1. (uri,commit) -> (uri,tree) + // 2. has (uri,tree) been indexed? + // 3. start indexing (update db state, use mutex to avoid double indexing) + gr, err := git.PlainOpen(repoPath(repoAt)) + if err != nil { + panic("unimplemented") + } + c, err := gr.CommitObject(commit) + if err != nil { + panic("unimplemented") + } + + // check if (uri,tree) is already indexed + indexed, err := db.IsLanguageIndexed(ctx, repoAt, c.TreeHash) + if err != nil { + panic("unimplemented") + } + if indexed { + return nil + } + + tree, err := c.Tree() + if err != nil { + panic("unimplemented") + } + + sizes := make(map[string]int64) + + err = tree.Files().ForEach(func(file *object.File) error { + // skip binary file + if isbin, _ := file.IsBinary(); isbin { + return nil + } + + // TODO: just skip super large files + var cap int64 = 16*1024 // 16KB + reader, err := file.Reader() + if err != nil { + return err + } + buf := new(bytes.Buffer) + if _, err = buf.ReadFrom(io.LimitReader(reader, cap)); err != nil { + return err + } + + // skip generated file + // TODO: follow gitattributes + if enry.IsGenerated(file.Name, buf.Bytes()) || + strings.HasSuffix(file.Name, "bun.lock") { + return nil + } + + language := analyzeLanguage(file.Name, buf.Bytes()) + if group := enry.GetLanguageGroup(language); group != "" { + language = group + } + + langType := enry.GetLanguageType(language) + if langType != enry.Programming && langType != enry.Markup && langType != enry.Unknown { + return nil + } + + sizes[language] += file.Size + + return nil + }) + if err != nil { + panic("unimplemented") + } + + panic("unimplemented") +} + +func analyzeLanguage(path string, content []byte) string { + language, ok := enry.GetLanguageByExtension(path) + if ok { + return language + } + + language, ok = enry.GetLanguageByFilename(path) + if ok { + return language + } + + if len(content) == 0 { + return enry.OtherLanguage + } + + return enry.GetLanguage(path, content) +} + +func IndexCommits(ctx context.Context, repoAt syntax.ATURI, commit plumbing.Hash) error { + // TODO: index (uri,commit-hash)->{date,[]emails} + // start from given commit, walk through DAG + // for each commits, check if we've indexed it & perform index + // 1. has (uri,commit) been indexed? + // 2. index it + gr, err := git.PlainOpen(repoPath(repoAt)) + if err != nil { + panic("unimplemented") + } + + count, err := indexCommits(ctx, gr, repoAt, commit) + if err != nil { + panic("unimplemented") + } + + panic("unimplemented") +} + +// indexCommits indexes a commit and all of its ancestors +func indexCommits(ctx context.Context, gitrepo *git.Repository, repoAt syntax.ATURI, commit plumbing.Hash) (int, error) { + isIndexed, err := db.IsCommitIndexed(ctx, repoAt, commit) + if err != nil { + panic("unimplemented") + } + if isIndexed { + return 0, nil + } + + c, err := gitrepo.CommitObject(commit) + if err != nil { + return 0, fmt.Errorf("commit '%s' not found", commit) + } + + emails := []string{c.Committer.Email} + + if err := db.InsertCommit(ctx, repoAt, commit, emails); err != nil { + return 0, fmt.Errorf("db: failed to insert commit '%s'", commit) + } + + var count int + for _, pc := range c.ParentHashes { + count, err = indexCommits(ctx, gitrepo, repoAt, pc) + if err != nil { + return 1, err + } + } + + return 1 + count, nil +} diff --git a/lexicons/git/temp/listLanguages.json b/lexicons/git/temp/listLanguages.json index 405af2ee..04ab70ba 100644 --- a/lexicons/git/temp/listLanguages.json +++ b/lexicons/git/temp/listLanguages.json @@ -24,7 +24,7 @@ "encoding": "application/json", "schema": { "type": "object", - "required": ["ref", "languages"], + "required": ["ref", "languages", "total"], "properties": { "ref": { "type": "string", @@ -37,13 +37,9 @@ "ref": "#language" } }, - "totalSize": { + "total": { "type": "integer", "description": "Total size of all analyzed files in bytes" - }, - "totalFiles": { - "type": "integer", - "description": "Total number of files analyzed" } } } @@ -78,21 +74,6 @@ "percentage": { "type": "integer", "description": "Percentage of total codebase (0-100)" - }, - "fileCount": { - "type": "integer", - "description": "Number of files in this language" - }, - "color": { - "type": "string", - "description": "Hex color code for this language" - }, - "extensions": { - "type": "array", - "items": { - "type": "string" - }, - "description": "File extensions associated with this language" } } }