Something went wrong. Try again.
Monorepo for Tangled
Something went wrong. Try again.
22 kB · 735 lines
Go
at master
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736package knotserver
import ( "context" "database/sql" "encoding/json" "errors" "fmt" "io" "net/http" "net/url" "path/filepath" "strings"
comatproto "github.com/bluesky-social/indigo/api/atproto" "github.com/bluesky-social/indigo/atproto/syntax" "github.com/bluesky-social/indigo/xrpc" indigoxrpc "github.com/bluesky-social/indigo/xrpc" jmodels "github.com/bluesky-social/jetstream/pkg/models" "tangled.org/core/api/tangled" "tangled.org/core/appview/models" "tangled.org/core/eventstream" "tangled.org/core/knotserver/db" "tangled.org/core/knotserver/git" knotxrpc "tangled.org/core/knotserver/xrpc" "tangled.org/core/log" "tangled.org/core/rbac" "tangled.org/core/tid" "tangled.org/core/workflow")
func (h *Knot) processPublicKey(ctx context.Context, event *jmodels.Event) error { l := log.FromContext(ctx) raw := json.RawMessage(event.Commit.Record) did := event.Did
var record tangled.PublicKey if err := json.Unmarshal(raw, &record); err != nil { return fmt.Errorf("failed to unmarshal record: %w", err) }
pk := db.PublicKey{ Did: did, PublicKey: record, } if err := h.db.AddPublicKey(pk); err != nil { l.Error("failed to add public key", "error", err) return fmt.Errorf("failed to add public key: %w", err) } l.Info("added public key from firehose", "did", did) return nil}
func (h *Knot) processKnotMember(ctx context.Context, event *jmodels.Event) error { did := event.Did rkey := event.Commit.RKey l := log.FromContext(ctx).With("handler", "processKnotMember", "did", did, "rkey", rkey)
switch event.Commit.Operation { case jmodels.CommitOperationCreate, jmodels.CommitOperationUpdate: raw := json.RawMessage(event.Commit.Record) var record tangled.KnotMember if err := json.Unmarshal(raw, &record); err != nil { return fmt.Errorf("failed to unmarshal record: %w", err) }
if record.Domain != h.c.Server.Hostname { return fmt.Errorf("domain mismatch: %s != %s", record.Domain, h.c.Server.Hostname) }
subject, err := syntax.ParseDID(record.Subject) if err != nil { return fmt.Errorf("invalid subject DID %q: %w", record.Subject, err) }
ok, err := h.e.E.Enforce(did, rbac.ThisServer, rbac.ThisServer, "server:invite") if err != nil { return fmt.Errorf("failed to enforce permissions: %w", err) } if !ok { return fmt.Errorf("permission denied for %s", did) }
sqlTx, err := h.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("failed to start txn: %w", err) } committed := false defer func() { if !committed { sqlTx.Rollback() } }()
existing, err := db.GetKnotMember(sqlTx, did, rkey) if err != nil && !errors.Is(err, sql.ErrNoRows) { return fmt.Errorf("failed to look up existing member: %w", err) }
var staleSubject string if existing != nil && existing.Subject != subject { staleSubject = existing.Subject.String() if err := db.RemoveKnotMember(sqlTx, did, rkey); err != nil { return fmt.Errorf("failed to remove stale member row: %w", err) } }
if err := db.AddKnotMember(sqlTx, db.KnotMember{ Did: syntax.DID(did), Rkey: rkey, Subject: subject, }); err != nil { return fmt.Errorf("failed to persist member row: %w", err) }
if err := db.AddDid(sqlTx, subject.String()); err != nil { return fmt.Errorf("failed to add did: %w", err) }
dropStaleAcl := false var staleDidDropped bool if staleSubject != "" { remaining, err := db.CountKnotMembersBySubject(sqlTx, staleSubject) if err != nil { return fmt.Errorf("failed to count stale subject rows: %w", err) } if remaining == 0 { dropStaleAcl = true stillNeeded, err := h.e.WouldHaveAnyPolicyExcludingKnotMember(staleSubject, rbac.ThisServer) if err != nil { return fmt.Errorf("failed to check residual policies for stale subject: %w", err) } if !stillNeeded { if err := db.RemoveDid(sqlTx, staleSubject); err != nil { return fmt.Errorf("failed to remove stale did: %w", err) } staleDidDropped = true } } l.Info("replaced stale knot member", "old_subject", staleSubject, "new_subject", subject, "stale_did_dropped", staleDidDropped) }
if err := sqlTx.Commit(); err != nil { return fmt.Errorf("failed to commit txn: %w", err) } committed = true
if dropStaleAcl { if _, err := h.e.TryRemoveKnotMember(rbac.ThisServer, staleSubject); err != nil { l.Error("post-commit: failed to remove stale ACL", "subject", staleSubject, "err", err) } } if _, err := h.e.TryAddKnotMember(rbac.ThisServer, subject.String()); err != nil { l.Error("post-commit: failed to add member ACL", "subject", subject, "err", err) }
if staleDidDropped { h.jc.RemoveDid(staleSubject) } h.jc.AddDid(subject.String()) l.Info("added member from firehose", "member", subject)
if err := h.fetchAndAddKeys(ctx, subject.String()); err != nil { return fmt.Errorf("failed to fetch and add keys: %w", err) } return nil
case jmodels.CommitOperationDelete: sqlTx, err := h.db.BeginTx(ctx, nil) if err != nil { return fmt.Errorf("failed to start txn: %w", err) } committed := false defer func() { if !committed { sqlTx.Rollback() } }()
member, err := db.GetKnotMember(sqlTx, did, rkey) if errors.Is(err, sql.ErrNoRows) { l.Info("knot member already removed") return nil } if err != nil { return fmt.Errorf("failed to look up knot member: %w", err) }
staleSubject := member.Subject.String()
if err := db.RemoveKnotMember(sqlTx, did, rkey); err != nil { return fmt.Errorf("failed to remove member row: %w", err) }
remaining, err := db.CountKnotMembersBySubject(sqlTx, staleSubject) if err != nil { return fmt.Errorf("failed to count remaining member rows: %w", err) }
dropAcl := false var staleDidDropped bool if remaining == 0 { dropAcl = true stillNeeded, err := h.e.WouldHaveAnyPolicyExcludingKnotMember(staleSubject, rbac.ThisServer) if err != nil { return fmt.Errorf("failed to check residual policies: %w", err) } if !stillNeeded { if err := db.RemoveDid(sqlTx, staleSubject); err != nil { return fmt.Errorf("failed to remove did: %w", err) } staleDidDropped = true } }
if err := sqlTx.Commit(); err != nil { return fmt.Errorf("failed to commit txn: %w", err) } committed = true
if dropAcl { if _, err := h.e.TryRemoveKnotMember(rbac.ThisServer, staleSubject); err != nil { l.Error("post-commit: failed to remove ACL", "subject", staleSubject, "err", err) } }
if staleDidDropped { h.jc.RemoveDid(staleSubject) } l.Info("removed knot member from firehose", "member", member.Subject, "remaining_rows", remaining, "stale_did_dropped", staleDidDropped) return nil }
return nil}
// returns a repo path on disk if present, and error if nottype targetRepo struct { RepoPath string OwnerDid string RepoName string RepoDid string DefaultBranch string // default branch}
func (h *Knot) validatePullRecord(ctx context.Context, record *tangled.RepoPull) (*targetRepo, error) { if record.Target == nil { return nil, fmt.Errorf("ignoring pull record: target repo is nil") }
l := log.FromContext(ctx).With("handler", "validatePullRecord") l = l.With("target_repo", record.Target.Repo) l = l.With("target_branch", record.Target.Branch)
if record.Source == nil { return nil, fmt.Errorf("ignoring pull record: not a branch-based pull request") }
if record.Source.Repo != nil { return nil, fmt.Errorf("ignoring pull record: fork based pull") }
var repoPath, ownerDid, repoName, repoDid string switch { case strings.HasPrefix(record.Target.Repo, "did:"): repoDid = record.Target.Repo var lookupErr error repoPath, ownerDid, repoName, lookupErr = h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, repoDid) if lookupErr != nil { return nil, fmt.Errorf("unknown target repo DID %s: %w", repoDid, lookupErr) }
case strings.Contains(record.Target.Repo, "/"): // TODO: get rid of this PDS fetch once all repos have DIDs repoAt, parseErr := syntax.ParseATURI(record.Target.Repo) if parseErr != nil { return nil, fmt.Errorf("failed to parse ATURI: %w", parseErr) }
ident, resolveErr := h.resolver.ResolveIdent(ctx, repoAt.Authority().String()) if resolveErr != nil || ident.Handle.IsInvalidHandle() { return nil, fmt.Errorf("failed to resolve handle: %w", resolveErr) }
xrpcc := xrpc.Client{ Host: ident.PDSEndpoint(), }
resp, getErr := comatproto.RepoGetRecord(ctx, &xrpcc, "", tangled.RepoNSID, repoAt.Authority().String(), repoAt.RecordKey().String()) if getErr != nil { return nil, fmt.Errorf("failed to resolve repo: %w", getErr) }
repo, ok := resp.Value.Val.(*tangled.Repo) if !ok { return nil, fmt.Errorf("record at %s is not a tangled.Repo", repoAt) }
if repo.Knot != h.c.Server.Hostname { return nil, fmt.Errorf("rejected pull record: not this knot, %s != %s", repo.Knot, h.c.Server.Hostname) }
ownerDid = ident.DID.String() repoName = repoAt.RecordKey().String()
repoDid, didErr := h.db.GetRepoDid(ownerDid, repoName) if didErr != nil { return nil, fmt.Errorf("failed to resolve repo DID for %s/%s: %w", ownerDid, repoName, didErr) }
var lookupErr error repoPath, _, _, lookupErr = h.db.ResolveRepoDIDOnDisk(h.c.Repo.ScanPath, repoDid) if lookupErr != nil { return nil, fmt.Errorf("failed to resolve repo on disk: %w", lookupErr) }
default: return nil, fmt.Errorf("ignoring pull record: target repo has unrecognized format: %s", record.Target.Repo) }
gr, err := git.Open(repoPath, record.Source.Branch) if err != nil { return nil, fmt.Errorf("failed to open git repository: %w", err) }
defaultBranch, _ := gr.FindMainBranch()
return &targetRepo{ RepoPath: repoPath, OwnerDid: ownerDid, RepoName: repoName, RepoDid: repoDid, DefaultBranch: defaultBranch, }, nil}
func (h *Knot) fetchLatestSubmission(ctx context.Context, did, rkey string, record *tangled.RepoPull) (*models.PullSubmission, error) { // resolve the PR owner's identity to fetch the blob from their PDS prOwnerIdent, err := h.resolver.ResolveIdent(ctx, did) if err != nil || prOwnerIdent.Handle.IsInvalidHandle() { return nil, fmt.Errorf("failed to resolve PR owner handle: %w", err) }
if len(record.Rounds) == 0 { return nil, fmt.Errorf("failed to fetch latest submission, no rounds in record") }
roundNumber := len(record.Rounds) - 1 round := record.Rounds[roundNumber]
// fetch the blob from the PR owner's PDS prOwnerPds := prOwnerIdent.PDSEndpoint() blobUrl, err := url.Parse(fmt.Sprintf("%s/xrpc/com.atproto.sync.getBlob", prOwnerPds)) if err != nil { return nil, fmt.Errorf("failed to construct blob URL: %w", err) } q := blobUrl.Query() q.Set("cid", round.PatchBlob.Ref.String()) q.Set("did", did) blobUrl.RawQuery = q.Encode()
req, err := http.NewRequestWithContext(ctx, http.MethodGet, blobUrl.String(), nil) if err != nil { return nil, fmt.Errorf("failed to create blob request: %w", err) } req.Header.Set("Content-Type", "application/json")
blobResp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("failed to fetch blob: %w", err) } defer blobResp.Body.Close()
blob := io.ReadCloser(blobResp.Body) latestSubmission, err := models.PullSubmissionFromRecord(did, rkey, roundNumber, round, &blob) if err != nil { return nil, fmt.Errorf("failed to parse submission: %w", err) }
return latestSubmission, nil}
func (h *Knot) discoverWorkflows(ctx context.Context, repoPath, sha string) (workflow.RawPipeline, error) { gr, err := git.Open(repoPath, sha) if err != nil { return nil, fmt.Errorf("failed to open git repository: %w", err) }
workflowDir, err := gr.FileTree(ctx, workflow.WorkflowDir) if err != nil { return nil, fmt.Errorf("failed to open workflow directory: %w", err) }
var pipeline workflow.RawPipeline for _, e := range workflowDir { if !e.IsFile() { continue }
fpath := filepath.Join(workflow.WorkflowDir, e.Name) contents, err := gr.RawContent(fpath) if err != nil { continue }
pipeline = append(pipeline, workflow.RawWorkflow{ Name: e.Name, Contents: contents, }) }
return pipeline, nil}
func (h *Knot) compilePipeline(ctx context.Context, targetRepo *targetRepo, sourceBranch, sourceSha, targetBranch string, rawPipeline workflow.RawPipeline) tangled.Pipeline { l := log.FromContext(ctx)
trigger := tangled.Pipeline_PullRequestTriggerData{ Action: "create", SourceBranch: sourceBranch, SourceSha: sourceSha, TargetBranch: targetBranch, }
compiler := workflow.Compiler{ Trigger: tangled.Pipeline_TriggerMetadata{ Kind: string(workflow.TriggerKindPullRequest), PullRequest: &trigger, Repo: &tangled.Pipeline_TriggerRepo{ Knot: h.c.Server.Hostname, RepoDid: &targetRepo.RepoDid, Did: targetRepo.OwnerDid, Repo: &targetRepo.RepoName, DefaultBranch: targetRepo.DefaultBranch, }, }, }
l.Info("raw", "raw", rawPipeline) parsed := compiler.Parse(rawPipeline) l.Info("parsed", "parsed", parsed) compiled := compiler.Compile(parsed)
l.Info("compiler diagnostics", "diagnostics", compiler.Diagnostics)
return compiled}
func (h *Knot) processPull(ctx context.Context, event *jmodels.Event) error { raw := json.RawMessage(event.Commit.Record) rkey := event.Commit.RKey did := event.Did
var record tangled.RepoPull if err := json.Unmarshal(raw, &record); err != nil { return fmt.Errorf("failed to unmarshal record: %w", err) }
l := log.FromContext(ctx) l = l.With("handler", "processPull") l = l.With("did", did)
l.Info("validating pull record") targetRepo, err := h.validatePullRecord(ctx, &record) if err != nil { l.Warn("pull record did not validate, skipping...") return err }
l = l.With("target_repo", record.Target.Repo) l = l.With("target_branch", record.Target.Branch)
l.Info("fetching latest submission") latestSubmission, err := h.fetchLatestSubmission(ctx, did, rkey, &record) if err != nil { return err }
sha := latestSubmission.SourceRev if sha == "" { return fmt.Errorf("failed to extract source SHA from pull submission") } l = l.With("sha", sha)
l.Info("discovering workflows", "repo_path", targetRepo.RepoPath) pipeline, err := h.discoverWorkflows(ctx, targetRepo.RepoPath, sha) if err != nil { return err }
l.Info("compiling pipeline", "workflow_count", len(pipeline)) cp := h.compilePipeline(ctx, targetRepo, record.Source.Branch, sha, record.Target.Branch, pipeline)
// do not run empty pipelines if cp.Workflows == nil { l.Info("skipping empty pipeline") return nil }
l.Info("marshaling pipeline event") eventJson, err := json.Marshal(cp) if err != nil { return fmt.Errorf("failed to marshal pipeline event: %w", err) }
ev := eventstream.Event{ Rkey: tid.TID(), Nsid: tangled.PipelineNSID, EventJson: eventJson, }
l.Info("inserting pipeline event") return h.db.InsertEvent(ev, h.n)}
// duplicated from add collaboratorfunc (h *Knot) processCollaborator(ctx context.Context, event *jmodels.Event) error { raw := json.RawMessage(event.Commit.Record) did := event.Did
var record tangled.RepoCollaborator if err := json.Unmarshal(raw, &record); err != nil { return fmt.Errorf("failed to unmarshal record: %w", err) }
subjectId, err := h.resolver.ResolveIdent(ctx, record.Subject) if err != nil || subjectId.Handle.IsInvalidHandle() { return err }
var rbacResource string switch { case strings.HasPrefix(record.Repo, "did:"): ownerDid, _, lookupErr := h.db.GetRepoKeyOwner(record.Repo) if lookupErr != nil { return fmt.Errorf("unknown repo DID %s: %w", record.Repo, lookupErr) } if ownerDid != did { return fmt.Errorf("collaborator record author %s does not own repo %s", did, record.Repo) } rbacResource = record.Repo
case strings.Contains(record.Repo, "/"): // TODO: get rid of this PDS fetch once all repos have DIDs repoAt, parseErr := syntax.ParseATURI(record.Repo) if parseErr != nil { return parseErr }
owner, resolveErr := h.resolver.ResolveIdent(ctx, repoAt.Authority().String()) if resolveErr != nil || owner.Handle.IsInvalidHandle() { return fmt.Errorf("failed to resolve handle: %w", resolveErr) }
xrpcc := xrpc.Client{ Host: owner.PDSEndpoint(), }
resp, getErr := comatproto.RepoGetRecord(ctx, &xrpcc, "", tangled.RepoNSID, repoAt.Authority().String(), repoAt.RecordKey().String()) if getErr != nil { return getErr }
if _, ok := resp.Value.Val.(*tangled.Repo); !ok { return fmt.Errorf("record at %s is not a tangled.Repo", repoAt) } rkey := repoAt.RecordKey().String() repoDid, didErr := h.db.GetRepoDid(owner.DID.String(), rkey) if didErr != nil { return fmt.Errorf("failed to resolve repo DID for %s/%s: %w", owner.DID.String(), rkey, didErr) } rbacResource = repoDid
default: return fmt.Errorf("collaborator record has unrecognized repo format: %s", record.Repo) }
ok, err := h.e.IsCollaboratorInviteAllowed(did, rbac.ThisServer, rbacResource) if err != nil { return fmt.Errorf("failed to check permissions: %w", err) } if !ok { return fmt.Errorf("insufficient permissions: %s, %s, %s", did, "IsCollaboratorInviteAllowed", rbacResource) }
if err := db.AddDid(h.db, subjectId.DID.String()); err != nil { return err } h.jc.AddDid(subjectId.DID.String())
if err := h.e.AddCollaborator(subjectId.DID.String(), rbac.ThisServer, rbacResource); err != nil { return err }
return h.fetchAndAddKeys(ctx, subjectId.DID.String())}
func (h *Knot) fetchAndAddKeys(ctx context.Context, did string) error { l := log.FromContext(ctx)
id, err := h.resolver.Directory().LookupDID(ctx, syntax.DID(did)) if err != nil { return fmt.Errorf("lookup did to fetch keys: %w", err) }
serviceEndpoint, ok := id.Services["atproto_pds"] if !ok { l.Warn("did identity did not contain atproto_pds service while adding their keys", "did", did) return nil }
xrpcc := indigoxrpc.Client{Host: serviceEndpoint.URL} resp, err := comatproto.RepoListRecords(context.Background(), &xrpcc, tangled.PublicKeyNSID, "", 50, did, false) if err != nil { return fmt.Errorf("fetching public keys for did: %w", err) }
for _, record := range resp.Records { if record == nil { continue } key := record.Value.Val.(*tangled.PublicKey) if key == nil { continue } pk := db.PublicKey{ Did: did, PublicKey: *key, } err = h.db.AddPublicKey(pk) if err != nil { return fmt.Errorf("adding public key to db: %w", err) } } return nil}
func (h *Knot) processRepo(ctx context.Context, event *jmodels.Event) error { l := log.FromContext(ctx).With("handler", "processRepo", "did", event.Did, "rkey", event.Commit.RKey)
rkey := strings.TrimSuffix(strings.TrimSpace(event.Commit.RKey), ".git") if rkey == "" { return nil }
if event.Commit.Operation == jmodels.CommitOperationDelete { return nil }
if event.Commit.Operation != jmodels.CommitOperationCreate && event.Commit.Operation != jmodels.CommitOperationUpdate { return nil }
raw := json.RawMessage(event.Commit.Record) var record tangled.Repo if err := json.Unmarshal(raw, &record); err != nil { return fmt.Errorf("failed to unmarshal repo record: %w", err) }
if record.Knot != h.c.Server.Hostname { return nil } if record.RepoDid == nil || *record.RepoDid == "" { l.Info("skipping repo event without repoDid") return nil } repoDid := *record.RepoDid
if err := knotxrpc.ValidateRepoName(rkey); err != nil { l.Warn("skipping repo event with invalid rkey", "repoDid", repoDid, "rkey", rkey, "err", err) return nil }
ownerDid, _, lookupErr := h.db.GetRepoKeyOwner(repoDid) if lookupErr != nil { l.Info("skipping repo event for unknown repoDid", "repoDid", repoDid) return nil } if ownerDid != event.Did { l.Warn("repo event author does not own repoDid", "repoDid", repoDid, "author", event.Did) return nil }
alias := db.RepoAlias{ OwnerDid: event.Did, Rkey: rkey, RepoDid: repoDid, Rev: event.Commit.Rev, } if err := h.db.UpsertRepoAlias(alias); err != nil { l.Warn("failed to upsert repo alias", "err", err) return nil }
l.Info("recorded repo alias", "repoDid", repoDid, "rkey", rkey, "rev", event.Commit.Rev) return nil}
func (h *Knot) processMessages(ctx context.Context, event *jmodels.Event) error { var err error switch event.Kind { case jmodels.EventKindIdentity: err = h.resolver.InvalidateIdent(ctx, event.Did) case jmodels.EventKindCommit: switch event.Commit.Collection { case tangled.PublicKeyNSID: err = h.processPublicKey(ctx, event) case tangled.KnotMemberNSID: err = h.processKnotMember(ctx, event) case tangled.RepoNSID: err = h.processRepo(ctx, event) case tangled.RepoPullNSID: err = h.processPull(ctx, event) case tangled.RepoCollaboratorNSID: err = h.processCollaborator(ctx, event) } default: return nil }
if err != nil { args := []any{"kind", event.Kind, "err", err} if event.Kind == jmodels.EventKindCommit { args = append(args, "nsid", event.Commit.Collection, "did", event.Did, "rkey", event.Commit.RKey) } h.l.Warn("failed to process event, skipping", args...) }
lastTimeUs := event.TimeUS + 1 if saveErr := h.db.SaveLastTimeUs(lastTimeUs); saveErr != nil { h.l.Error("failed to save cursor", "err", saveErr) }
return nil}