diff --git a/appview/accountmigration/accountmigration.go b/appview/accountmigration/accountmigration.go new file mode 100644 index 00000000..d18d4260 --- /dev/null +++ b/appview/accountmigration/accountmigration.go @@ -0,0 +1,298 @@ +package accountmigration + +import ( + "encoding/json" + "fmt" + "log/slog" + "net/http" + "net/url" + "slices" + "strconv" + "strings" + + "github.com/bluesky-social/indigo/atproto/syntax" + "github.com/go-chi/chi/v5" + "tangled.org/core/appview/db" + "tangled.org/core/appview/middleware" + "tangled.org/core/appview/models" + "tangled.org/core/appview/oauth" + "tangled.org/core/appview/pages" + "tangled.org/core/rbac" + "tangled.org/core/sets" +) + +type AccountMigration struct { + logger *slog.Logger + oauth *oauth.OAuth + pages *pages.Pages + db *db.DB + enforcer *rbac.Enforcer +} + +func New( + logger *slog.Logger, + o *oauth.OAuth, + p *pages.Pages, + d *db.DB, + enforcer *rbac.Enforcer, +) *AccountMigration { + return &AccountMigration{ + logger: logger, + oauth: o, + pages: p, + db: d, + enforcer: enforcer, + } +} + +func (s *AccountMigration) Router() http.Handler { + r := chi.NewRouter() + r.Use(middleware.AuthMiddleware(s.oauth)) + + r.Get("/", s.migratePage) + r.Post("/listGitHubRepos", s.listGitHubRepos) + r.Post("/start", s.startMigration) + r.Get("/progress/rows", s.progressRows) + return r +} + +func (s *AccountMigration) migratePage(w http.ResponseWriter, r *http.Request) { + user := s.oauth.GetMultiAccountUser(r) + + service := r.URL.Query().Get("service") + switch service { + case "github": + knots, err := s.enforcer.GetKnotsForUser(user.Did) + if err != nil { + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) + knots = nil + } + s.pages.AccountMigrateFromGitHub(w, pages.AccountMigrateFromGitHubParams{ + BaseParams: pages.BaseParamsFromContext(r.Context()), + Knots: knots, + }) + default: + s.pages.AccountMigrate(w, pages.AccountMigrateParams{ + BaseParams: pages.BaseParamsFromContext(r.Context()), + }) + } +} + +type githubUserRepo struct { + Name string `json:"name"` + CloneUrl string `json:"clone_url"` + DefaultBranch string `json:"default_branch"` + Description string `json:"description"` + Homepage string `json:"homepage"` + Topics []string `json:"topics"` + Fork bool `json:"fork"` + Private bool `json:"private"` + Archived bool `json:"archived"` + Disabled bool `json:"disabled"` +} + +func (s *AccountMigration) listGitHubRepos(w http.ResponseWriter, r *http.Request) { + user := s.oauth.GetMultiAccountUser(r) + + username := strings.TrimSpace(r.FormValue("username")) + if username == "" { + s.pages.Notice(w, "migrate-error", "GitHub username is required.") + return + } + + query := url.Values{} + query.Set("sort", "updated") + query.Set("per_page", "80") + + endpoint := fmt.Sprintf("https://api.github.com/users/%s/repos?%s", url.PathEscape(username), query.Encode()) + req, err := http.NewRequestWithContext(r.Context(), http.MethodGet, endpoint, nil) + if err != nil { + s.pages.Notice(w, "migrate-error", "Failed to build GitHub request.") + return + } + req.Header.Set("Accept", "application/vnd.github+json") + + resp, err := http.DefaultClient.Do(req) + if err != nil { + s.logger.Error("github list repos failed", "username", username, "err", err) + s.pages.Notice(w, "migrate-error", "Failed to reach GitHub. Try again.") + return + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + s.pages.Notice(w, "migrate-error", fmt.Sprintf("GitHub returned %d. Check the username.", resp.StatusCode)) + return + } + + var githubRepos []githubUserRepo + if err := json.NewDecoder(resp.Body).Decode(&githubRepos); err != nil { + s.logger.Error("decode github response failed", "err", err) + s.pages.Notice(w, "migrate-error", "Failed to parse GitHub response.") + return + } + + enqueued, err := db.ListEnqueuedGitRepoNames(r.Context(), s.db, user.Did) + if err != nil { + s.logger.Error("list enqueued names failed", "err", err) + enqueued = map[string]struct{}{} + } + + knots, err := s.enforcer.GetKnotsForUser(user.Did) + if err != nil { + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) + knots = nil + } + + importRepos := make([]pages.RepoImportParams, 0, len(githubRepos)) + for _, ghrepo := range githubRepos { + if ghrepo.Fork || ghrepo.Private || ghrepo.Archived || ghrepo.Disabled { + continue + } + name := strings.ToLower(ghrepo.Name) + _, migrated := enqueued[name] + importRepos = append(importRepos, pages.RepoImportParams{ + SourceKind: pages.RepoImportSourceGitHub, + CloneUrl: ghrepo.CloneUrl, + Name: name, + Description: ghrepo.Description, + Website: ghrepo.Homepage, + Topics: ghrepo.Topics, + Selected: !migrated, + }) + } + + if err := s.pages.AccountMigrateRepoListFragment(w, pages.AccountMigrateRepoListParams{ + Repos: importRepos, + Knots: knots, + }); err != nil { + s.logger.Error("render repo-list fragment failed", "err", err) + } +} + +func (s *AccountMigration) startMigration(w http.ResponseWriter, r *http.Request) { + user := s.oauth.GetMultiAccountUser(r) + + if err := r.ParseForm(); err != nil { + s.pages.Notice(w, "migrate-error", "Invalid form submission.") + return + } + + knots, err := s.enforcer.GetKnotsForUser(user.Did) + if err != nil { + s.logger.Error("knots lookup failed", "did", user.Did, "err", err) + s.pages.Notice(w, "migrate-error", "Failed to look up your knots.") + return + } + allowed := sets.Collect(slices.Values(knots)) + + sessionId := s.oauth.GetSessIdFromCookie(r) + if sessionId == "" { + s.pages.Notice(w, "migrate-error", "Session expired. Log in again.") + return + } + + count, err := strconv.Atoi(r.FormValue("count")) + if err != nil || count <= 0 { + s.pages.Notice(w, "migrate-error", "Invalid form submission.") + return + } + + s.logger.Debug("migrating repos", "count", count) + + rows := make([]db.GitRepoMigration, 0, count) + seen := sets.New[string]() + for i := range count { + if r.FormValue(fmt.Sprintf("selected_%d", i)) == "" { + s.logger.Warn("can't find selected repo", "i", i) + continue + } + s.logger.Info("found selected repo", "i", i) + + cloneUrl := strings.TrimSpace(r.FormValue(fmt.Sprintf("clone_url_%d", i))) + knot := strings.TrimSpace(r.FormValue(fmt.Sprintf("knot_%d", i))) + name := strings.ToLower(strings.TrimSpace(r.FormValue(fmt.Sprintf("name_%d", i)))) + desc := strings.TrimSpace(r.FormValue(fmt.Sprintf("description_%d", i))) + website := strings.TrimSpace(r.FormValue(fmt.Sprintf("website_%d", i))) + topics := r.FormValue(fmt.Sprintf("topics_%d", i)) + + if cloneUrl == "" || knot == "" || name == "" { + s.pages.Notice(w, "migrate-error", "Each selected row needs a name, clone URL, and knot.") + return + } + if err := models.ValidateRepoName(name); err != nil { + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Row %d: %s", i+1, err.Error())) + return + } + if len([]rune(desc)) > 140 { + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Row %d: description must be 140 characters or fewer.", i+1)) + return + } + if !allowed.Contains(knot) { + s.pages.Notice(w, "migrate-error", fmt.Sprintf("You are not a member of knot %q.", knot)) + return + } + if seen.Contains(name) { + s.pages.Notice(w, "migrate-error", fmt.Sprintf("Duplicate repository name %q in selection.", name)) + return + } + seen.Insert(name) + + rows = append(rows, db.GitRepoMigration{ + OwnerDid: syntax.DID(user.Did), + SourceKind: db.GitRepoMigrationSourceGitHub, + CloneUrl: cloneUrl, + Name: name, + Knot: knot, + Description: desc, + Website: website, + SessionID: sessionId, + Topics: func(s string) []string { + if s == "" { + return nil + } + parts := strings.Split(s, ",") + out := parts[:0] + for _, p := range parts { + p = strings.TrimSpace(p) + if p != "" { + out = append(out, p) + } + } + return out + }(topics), + }) + } + + if len(rows) == 0 { + s.pages.Notice(w, "migrate-error", "Pick at least one repo.") + return + } + + s.logger.Info("inserting migrations", "len", len(rows)) + + if err := db.InsertGitRepoMigrations(r.Context(), s.db, rows); err != nil { + s.logger.Error("insert migrations failed", "err", err) + s.pages.Notice(w, "migrate-error", "Failed to enqueue migrations.") + return + } + + w.Header().Set("HX-Redirect", "/settings/migration") + http.Redirect(w, r, "/settings/migration", http.StatusSeeOther) +} + +func (s *AccountMigration) progressRows(w http.ResponseWriter, r *http.Request) { + user := s.oauth.GetMultiAccountUser(r) + + migrations, err := db.ListGitRepoMigrationsForOwner(r.Context(), s.db, user.Did) + if err != nil { + s.logger.Error("list migrations failed", "did", user.Did, "err", err) + } + + if err := s.pages.AccountMigrateProgressRowsFragment(w, pages.AccountMigrateProgressParams{ + BaseParams: pages.BaseParamsFromContext(r.Context()), + Migrations: migrations, + }); err != nil { + s.logger.Error("render progress rows failed", "err", err) + } +} diff --git a/appview/accountmigration/worker.go b/appview/accountmigration/worker.go new file mode 100644 index 00000000..cfee381b --- /dev/null +++ b/appview/accountmigration/worker.go @@ -0,0 +1,193 @@ +package accountmigration + +import ( + "context" + "fmt" + "log/slog" + "sync" + "time" + + comatproto "github.com/bluesky-social/indigo/api/atproto" + "github.com/bluesky-social/indigo/atproto/atclient" + "github.com/bluesky-social/indigo/atproto/syntax" + lexutil "github.com/bluesky-social/indigo/lex/util" + "github.com/bluesky-social/indigo/xrpc" + + "tangled.org/core/api/tangled" + "tangled.org/core/appview/db" + "tangled.org/core/appview/models" + "tangled.org/core/appview/oauth" + "tangled.org/core/xrpc/xrpcclient" + "tangled.org/core/orm" +) + +const defaultConcurrency = 4 + +type Worker struct { + db *db.DB + oauth *oauth.OAuth + dev bool + logger *slog.Logger + concurrency int + + startMu sync.Mutex + started bool +} + +func NewWorker(d *db.DB, o *oauth.OAuth, dev bool, logger *slog.Logger) *Worker { + return &Worker{ + db: d, + oauth: o, + dev: dev, + logger: logger, + concurrency: defaultConcurrency, + } +} + +func (w *Worker) Start(ctx context.Context) { + w.startMu.Lock() + defer w.startMu.Unlock() + if w.started { + return + } + w.started = true + + for i := 0; i < w.concurrency; i++ { + go w.runWorker(ctx, i) + } +} + +func (w *Worker) runWorker(ctx context.Context, id int) { + l := w.logger.With("worker", id) + for { + select { + case <-ctx.Done(): + return + default: + } + + row, ok, err := db.ClaimNextPending(ctx, w.db) + if err != nil { + l.Error("claim failed", "err", err) + time.Sleep(time.Second) + continue + } + if !ok { + time.Sleep(time.Second) + continue + } + l.Info("migrating repo", "row", row) + + w.importRepo(ctx, l, row) + } +} + +func (w *Worker) importRepo(ctx context.Context, l *slog.Logger, row *db.GitRepoMigration) { + l = l.With("id", row.ID, "owner", row.OwnerDid, "name", row.Name, "knot", row.Knot) + + err := w.doImportRepo(ctx, row) + if err == nil { + if err := db.MarkGitRepoMigrationDone(ctx, w.db, row.ID); err != nil { + l.Error("mark done failed", "err", err) + } + return + } + + l.Warn("job failed", "err", err) + + if uerr := db.MarkGitRepoMigrationFailed(ctx, w.db, row.ID, err.Error()); uerr != nil { + l.Error("mark failed failed", "err", uerr) + } +} + +func (w *Worker) doImportRepo(ctx context.Context, row *db.GitRepoMigration) error { + atpClient, err := w.oauth.ClientForDidSession(ctx, row.OwnerDid, row.SessionID) + if err != nil { + return fmt.Errorf("resume session: %w", err) + } + + existing, _ := db.GetRepo(w.db, + orm.FilterEq("did", row.OwnerDid), + orm.FilterEq("rkey", row.Name), + ) + if existing != nil || rkeyOccupied(ctx, atpClient, row.OwnerDid.String(), row.Name) { + return fmt.Errorf("rkey %q already exists", row.Name) + } + + sc, err := w.oauth.ServiceClientForDidSession(ctx, row.OwnerDid, row.SessionID, + oauth.WithService(row.Knot), + oauth.WithLxm(tangled.RepoCreateNSID), + oauth.WithDev(w.dev), + oauth.WithTimeout(10*time.Minute), + ) + if err != nil { + return fmt.Errorf("knot service client: %w", err) + } + + defaultBranch := "main" + createResp, err := tangled.RepoCreate(ctx, sc, &tangled.RepoCreate_Input{ + Rkey: row.Name, + Name: row.Name, + DefaultBranch: &defaultBranch, + Source: &row.CloneUrl, + }) + if xrpcerr := xrpcclient.HandleXrpcErr(err); xrpcerr != nil { + return fmt.Errorf("knot RepoCreate: %w", err) + } + if err != nil { + return fmt.Errorf("knot RepoCreate: %w", err) + } + var repoDid string + if createResp != nil && createResp.RepoDid != nil { + repoDid = *createResp.RepoDid + } + if repoDid == "" { + return fmt.Errorf("knot returned empty repo DID") + } + + repo := &models.Repo{ + Did: row.OwnerDid.String(), + Name: row.Name, + Knot: row.Knot, + Rkey: row.Name, + Description: row.Description, + Created: time.Now(), + RepoDid: repoDid, + } + record := repo.AsRecord() + + rkey := row.Name + _, err = comatproto.RepoCreateRecord(ctx, atpClient, &comatproto.RepoCreateRecord_Input{ + Collection: tangled.RepoNSID, + Repo: row.OwnerDid.String(), + Rkey: &rkey, + Record: &lexutil.LexiconTypeDecoder{ + Val: &record, + }, + }) + if err != nil { + w.cleanupKnotRepo(ctx, sc, row.OwnerDid, row.Knot, row.Name) + return fmt.Errorf("PDS PutRecord: %w", err) + } + + return nil +} + +func (w *Worker) cleanupKnotRepo(ctx context.Context, sc *xrpc.Client, did syntax.DID, knot, name string) { + cctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + if err := tangled.RepoDelete(cctx, sc, &tangled.RepoDelete_Input{ + Did: did.String(), + Name: name, + Rkey: name, + }); err != nil { + w.logger.Error("cleanup: RepoDelete failed", "knot", knot, "name", name, "err", err) + } +} + +func rkeyOccupied(ctx context.Context, client *atclient.APIClient, did, rkey string) bool { + probeCtx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + resp, err := comatproto.RepoGetRecord(probeCtx, client, "", tangled.RepoNSID, did, rkey) + return err == nil && resp != nil +} diff --git a/appview/db/account_migration.go b/appview/db/account_migration.go new file mode 100644 index 00000000..b0e2a658 --- /dev/null +++ b/appview/db/account_migration.go @@ -0,0 +1,224 @@ +package db + +import ( + "context" + "database/sql" + "errors" + "time" + + "github.com/bluesky-social/indigo/atproto/syntax" +) + +type GitRepoMigrationStatus string + +const ( + GitRepoMigrationStatusPending GitRepoMigrationStatus = "pending" + GitRepoMigrationStatusRunning GitRepoMigrationStatus = "running" + GitRepoMigrationStatusDone GitRepoMigrationStatus = "done" + GitRepoMigrationStatusFailed GitRepoMigrationStatus = "failed" +) + +const ( + GitRepoMigrationSourceGitHub = "github" +) + +type GitRepoMigrations []GitRepoMigration + +func (m GitRepoMigrations) AnyActive() bool { + for _, r := range m { + if r.Status == GitRepoMigrationStatusPending || r.Status == GitRepoMigrationStatusRunning { + return true + } + } + return false +} + +type GitRepoMigration struct { + ID int64 + OwnerDid syntax.DID + SourceKind string + CloneUrl string + Name string + Knot string + Description string + Website string + Topics []string + SessionID string + Status GitRepoMigrationStatus + ErrorMsg string + UpdatedAt time.Time +} + +func InsertGitRepoMigrations(ctx context.Context, e *DB, rows []GitRepoMigration) error { + if len(rows) == 0 { + return nil + } + txx, err := e.BeginTx(ctx, nil) + if err != nil { + return err + } + defer txx.Rollback() + + stmt, err := txx.PrepareContext(ctx, ` + insert into gitrepo_migrations + (owner_did, source_kind, clone_url, name, knot, description, session_id, + status, error_msg, updated_at) + values (?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + on conflict(owner_did, name) do update set + source_kind = excluded.source_kind, + clone_url = excluded.clone_url, + knot = excluded.knot, + description = excluded.description, + session_id = excluded.session_id, + status = 'pending', + error_msg = '', + updated_at = excluded.updated_at + `) + if err != nil { + return err + } + defer stmt.Close() + + now := time.Now().UTC().Format(time.RFC3339) + for _, r := range rows { + status := r.Status + if status == "" { + status = GitRepoMigrationStatusPending + } + if _, err := stmt.ExecContext(ctx, + r.OwnerDid, r.SourceKind, r.CloneUrl, r.Name, r.Knot, r.Description, r.SessionID, + status, r.ErrorMsg, now, + ); err != nil { + return err + } + } + return txx.Commit() +} + +func ClaimNextPending(ctx context.Context, e Execer) (*GitRepoMigration, bool, error) { + var ( + m GitRepoMigration + updatedAt string + ) + err := e.QueryRowContext(ctx, ` + update gitrepo_migrations + set status = 'running', + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where id = ( + select id from gitrepo_migrations gm + where status = 'pending' + and not exists ( + select 1 from gitrepo_migrations gm2 + where gm2.owner_did = gm.owner_did + and gm2.status = 'running' + ) + order by id + limit 1 + ) + returning id, owner_did, source_kind, clone_url, name, knot, + description, session_id, status, error_msg, updated_at + `).Scan( + &m.ID, &m.OwnerDid, &m.SourceKind, &m.CloneUrl, &m.Name, &m.Knot, + &m.Description, &m.SessionID, &m.Status, &m.ErrorMsg, + &updatedAt, + ) + if errors.Is(err, sql.ErrNoRows) { + return nil, false, nil + } + if err != nil { + return nil, false, err + } + if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { + m.UpdatedAt = t + } + return &m, true, nil +} + +func MarkGitRepoMigrationDone(ctx context.Context, e Execer, id int64) error { + _, err := e.ExecContext(ctx, ` + update gitrepo_migrations + set status = 'done', + error_msg = '', + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where id = ? + `, id) + return err +} + +func MarkGitRepoMigrationFailed(ctx context.Context, e Execer, id int64, errMsg string) error { + _, err := e.ExecContext(ctx, ` + update gitrepo_migrations + set status = 'failed', + error_msg = ?, + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where id = ? + `, errMsg, id) + return err +} + +func ListGitRepoMigrationsForOwner(ctx context.Context, e Execer, owner string) (GitRepoMigrations, error) { + rows, err := e.QueryContext(ctx, ` + select id, owner_did, source_kind, clone_url, name, knot, + description, session_id, status, coalesce(error_msg, ''), + updated_at + from gitrepo_migrations + where owner_did = ? + order by id desc + `, owner) + if err != nil { + return nil, err + } + defer rows.Close() + + var out GitRepoMigrations + for rows.Next() { + var ( + m GitRepoMigration + updatedAt string + ) + if err := rows.Scan( + &m.ID, &m.OwnerDid, &m.SourceKind, &m.CloneUrl, &m.Name, &m.Knot, + &m.Description, &m.SessionID, &m.Status, &m.ErrorMsg, + &updatedAt, + ); err != nil { + return nil, err + } + if t, err := time.Parse(time.RFC3339, updatedAt); err == nil { + m.UpdatedAt = t + } + out = append(out, m) + } + return out, rows.Err() +} + +func ListEnqueuedGitRepoNames(ctx context.Context, e Execer, owner string) (map[string]struct{}, error) { + rows, err := e.QueryContext(ctx, ` + select name from gitrepo_migrations + where owner_did = ? + and status in ('pending', 'running', 'done') + `, owner) + if err != nil { + return nil, err + } + defer rows.Close() + + out := map[string]struct{}{} + for rows.Next() { + var n string + if err := rows.Scan(&n); err != nil { + return nil, err + } + out[n] = struct{}{} + } + return out, rows.Err() +} + +func ReapStaleRunningGitRepoMigrations(ctx context.Context, e Execer) error { + _, err := e.ExecContext(ctx, ` + update gitrepo_migrations + set status = 'pending', + updated_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now') + where status = 'running' + `) + return err +} diff --git a/appview/db/db.go b/appview/db/db.go index 51357984..6387ea1a 100644 --- a/appview/db/db.go +++ b/appview/db/db.go @@ -2487,6 +2487,30 @@ func Make(ctx context.Context, dbPath string) (*DB, error) { return err }) + orm.RunMigration(conn, logger, "add-gitrepo_migrations", func(tx *sql.Tx) error { + _, err := tx.Exec(` + create table gitrepo_migrations ( + id integer primary key autoincrement, + owner_did text not null, + source_kind text not null, + clone_url text not null, + name text not null, + knot text not null, + description text not null, + session_id text not null default '', + + status text not null default 'pending', + error_msg text not null default '', + updated_at text not null default (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), + + unique(owner_did, name) + ); + create index if not exists idx_gitrepo_migrations_status + on gitrepo_migrations(status); + `) + return err + }) + return &DB{ db, logger, diff --git a/appview/oauth/oauth.go b/appview/oauth/oauth.go index 16eeb82d..57af8b8e 100644 --- a/appview/oauth/oauth.go +++ b/appview/oauth/oauth.go @@ -350,6 +350,14 @@ func (o *OAuth) AuthorizedClient(r *http.Request) (*atclient.APIClient, error) { return session.APIClient(), nil } +func (o *OAuth) ClientForDidSession(ctx context.Context, did syntax.DID, sessionId string) (*atclient.APIClient, error) { + sess, err := o.resumeSession(ctx, did, sessionId) + if err != nil { + return nil, fmt.Errorf("error getting session: %w", err) + } + return sess.APIClient(), nil +} + // this is a higher level abstraction on ServerGetServiceAuth type ServiceClientOpts struct { service string @@ -460,6 +468,38 @@ func (o *OAuth) SpindleServiceClient(r *http.Request, spindle, lxm string) (*xrp ) } +func (o *OAuth) ServiceClientForDidSession(ctx context.Context, did syntax.DID, sessionId string, os ...ServiceClientOpt) (*xrpc.Client, error) { + opts := DefaultServiceClientOpts() + for _, o := range os { + o(&opts) + } + + client, err := o.ClientForDidSession(ctx, did, sessionId) + if err != nil { + return nil, err + } + + sixty := time.Now().Unix() + 60 + if opts.exp < sixty { + opts.exp = sixty + } + + resp, err := comatproto.ServerGetServiceAuth(ctx, client, opts.Audience(), opts.exp, opts.lxm) + if err != nil { + return nil, err + } + + return &xrpc.Client{ + Auth: &xrpc.AuthInfo{ + AccessJwt: resp.Token, + }, + Host: opts.Host(), + Client: &http.Client{ + Timeout: opts.timeout, + }, + }, nil +} + func (o *OAuth) StartElevatedAuthFlow(ctx context.Context, w http.ResponseWriter, r *http.Request, did string, extraScopes []string, returnURL string) (string, error) { parsedDid, err := syntax.ParseDID(did) if err != nil { diff --git a/appview/pages/funcmap.go b/appview/pages/funcmap.go index 0b5fd231..0f951ede 100644 --- a/appview/pages/funcmap.go +++ b/appview/pages/funcmap.go @@ -599,6 +599,7 @@ func (p *Pages) funcMap() template.FuncMap { {"Name": "knots", "Label": "Knots", "Icon": "volleyball"}, {"Name": "spindles", "Label": "Spindles", "Icon": "spool"}, {"Name": "sites", "Label": "Sites", "Icon": "globe"}, + {"Name": "migration", "Label": "Migration", "Icon": "book-down"}, }, "RepoSettingsTabs": []tab{ {"Name": "general", "Label": "General", "Icon": "sliders-horizontal"}, diff --git a/appview/pages/pages.go b/appview/pages/pages.go index f59b8c4e..1bf555b4 100644 --- a/appview/pages/pages.go +++ b/appview/pages/pages.go @@ -2047,6 +2047,69 @@ func (p *Pages) ReplyPlaceholderFragment(w io.Writer, params ReplyPlaceholderFra return p.executePlain("fragments/comment/replyPlaceholder", w, params) } +type AccountMigrateParams struct { + BaseParams +} + +func (p *Pages) AccountMigrate(w io.Writer, params AccountMigrateParams) error { + return p.execute("account/migrate/index", w, params) +} + +type AccountMigrateFromGitHubParams struct { + BaseParams + Knots []string +} + +type RepoImportSource string + +const ( + RepoImportSourceGitHub RepoImportSource = "github" +) + +type RepoImportParams struct { + SourceKind RepoImportSource + CloneUrl string + Name string + Knot string + Description string + Website string + Topics []string + + Selected bool +} + +func (p *Pages) AccountMigrateFromGitHub(w io.Writer, params AccountMigrateFromGitHubParams) error { + return p.execute("account/migrate/fromGitHub", w, params) +} + +type AccountMigrateRepoListParams struct { + Repos []RepoImportParams + Knots []string +} + +func (p *Pages) AccountMigrateRepoListFragment(w io.Writer, params AccountMigrateRepoListParams) error { + return p.executePlain("account/migrate/fragments/repoList", w, params) +} + +type AccountMigrateProgressParams struct { + BaseParams + Migrations db.GitRepoMigrations +} + +func (p *Pages) AccountMigrateProgressRowsFragment(w io.Writer, params AccountMigrateProgressParams) error { + return p.executePlain("account/migrate/fragments/progressRows", w, params) +} + +type UserMigrationSettingsParams struct { + BaseParams + Tab string + Migrations db.GitRepoMigrations +} + +func (p *Pages) UserMigrationSettings(w io.Writer, params UserMigrationSettingsParams) error { + return p.execute("user/settings/migration", w, params) +} + func (p *Pages) Static() http.Handler { if p.dev { return http.StripPrefix("/static/", http.FileServer(http.Dir("appview/pages/static"))) diff --git a/appview/pages/templates/account/migrate/fragments/progressRows.html b/appview/pages/templates/account/migrate/fragments/progressRows.html new file mode 100644 index 00000000..32ec9a67 --- /dev/null +++ b/appview/pages/templates/account/migrate/fragments/progressRows.html @@ -0,0 +1,35 @@ +{{ define "account/migrate/fragments/progressRows" }} + + {{ range .Migrations }} + + {{ .Name }} + {{ .Knot }} + + {{ if eq .Status "done" }} + done + {{ else if eq .Status "failed" }} + failed + {{ else if eq .Status "running" }} + running… + {{ else }} + pending + {{ end }} + {{ if .ErrorMsg }} + ({{ .ErrorMsg }}) + {{ end }} + + + {{ if not .UpdatedAt.IsZero }}{{ .UpdatedAt.Format "2006-01-02 15:04:05" }}{{ end }} + + + {{ end }} + +{{ end }} diff --git a/appview/pages/templates/account/migrate/fragments/repoList.html b/appview/pages/templates/account/migrate/fragments/repoList.html new file mode 100644 index 00000000..8d1e1ed6 --- /dev/null +++ b/appview/pages/templates/account/migrate/fragments/repoList.html @@ -0,0 +1,68 @@ +{{ define "account/migrate/fragments/repoList" }} +{{ if not .Repos }} +

No importable repositories found.

+{{ else }} +
+ + +
+ + +
+ + {{ range $i, $repo := .Repos }} +
+ +
+ +
+
{{ $repo.Name }}
+
{{ $repo.CloneUrl }}
+
+
+ + + + +
+ + + + + + + +
+
+ {{ end }} + +
+ +
+
+{{ end }} +{{ end }} diff --git a/appview/pages/templates/account/migrate/fromGitHub.html b/appview/pages/templates/account/migrate/fromGitHub.html new file mode 100644 index 00000000..0eebb48f --- /dev/null +++ b/appview/pages/templates/account/migrate/fromGitHub.html @@ -0,0 +1,37 @@ +{{ define "title" }}Migrate from GitHub{{ end }} + +{{ define "content" }} +
+

Migrate from GitHub

+

+ Enter a GitHub username to list their public repositories. +

+
+
+
+ + +
+ + + + {{ if not .Knots }} +

+ You aren't a member of any knot yet. Join or register a knot first. +

+ {{ end }} + +
+
+{{ end }} diff --git a/appview/pages/templates/account/migrate/index.html b/appview/pages/templates/account/migrate/index.html new file mode 100644 index 00000000..7df6937a --- /dev/null +++ b/appview/pages/templates/account/migrate/index.html @@ -0,0 +1,15 @@ +{{ define "title" }}Migrate account{{ end }} + +{{ define "content" }} +
+

Migrate repositories

+

+ Pull repositories from another forge into Tangled. +

+
+
+ + Migrate from GitHub + +
+{{ end }} diff --git a/appview/pages/templates/user/settings/migration.html b/appview/pages/templates/user/settings/migration.html new file mode 100644 index 00000000..06ae0184 --- /dev/null +++ b/appview/pages/templates/user/settings/migration.html @@ -0,0 +1,55 @@ +{{ define "title" }}{{ .Tab }} settings{{ end }} + +{{ define "content" }} +
+

Settings

+
+
+
+
+ {{ template "user/settings/fragments/sidebar" . }} +
+
+ {{ template "migrationSettings" . }} +
+
+
+{{ end }} + +{{ define "migrationSettings" }} +
+
+

Repository imports

+

+ Track the status of repositories pulled in from another forge. + Successful imports stay; failed ones can be retried by starting a new migration with the same name. +

+
+
+ + {{ i "plus" "size-4" }} + new migration + +
+
+ +
+ {{ if not .Migrations }} +
+ no migrations yet +
+ {{ else }} + + + + + + + + + + {{ template "account/migrate/fragments/progressRows" . }} +
RepoKnotStatus
+ {{ end }} +
+{{ end }} diff --git a/appview/settings/settings.go b/appview/settings/settings.go index 8a1a25e3..586921f0 100644 --- a/appview/settings/settings.go +++ b/appview/settings/settings.go @@ -79,6 +79,10 @@ func (s *Settings) Router() http.Handler { r.Delete("/", s.releaseSitesDomain) }) + r.Route("/migration", func(r chi.Router) { + r.Get("/", s.migrationSettings) + }) + r.Post("/password/request", s.requestPasswordReset) r.Post("/password/reset", s.resetPassword) r.Post("/deactivate", s.deactivateAccount) @@ -110,6 +114,21 @@ func (s *Settings) isTnglShUser(ctx context.Context, did syntax.DID) (bool, erro return strings.TrimRight(userIdent.PDSEndpoint(), "/") == strings.TrimRight(s.Config.Pds.Host, "/"), nil } +func (s *Settings) migrationSettings(w http.ResponseWriter, r *http.Request) { + user := s.OAuth.GetMultiAccountUser(r) + + migrations, err := db.ListGitRepoMigrationsForOwner(r.Context(), s.Db, user.Did) + if err != nil { + s.Logger.Error("list migrations failed", "did", user.Did, "err", err) + } + + s.Pages.UserMigrationSettings(w, pages.UserMigrationSettingsParams{ + BaseParams: pages.BaseParamsFromContext(r.Context()), + Tab: "migration", + Migrations: migrations, + }) +} + func (s *Settings) sitesSettings(w http.ResponseWriter, r *http.Request) { user := s.OAuth.GetMultiAccountUser(r) diff --git a/appview/state/router.go b/appview/state/router.go index 1c685537..3f6eef21 100644 --- a/appview/state/router.go +++ b/appview/state/router.go @@ -8,6 +8,7 @@ import ( "strings" "github.com/go-chi/chi/v5" + "tangled.org/core/appview/accountmigration" "tangled.org/core/appview/db" "tangled.org/core/appview/focus" "tangled.org/core/appview/issues" @@ -262,6 +263,8 @@ func (s *State) StandardRouter(mw *middleware.Middleware) http.Handler { }) r.Get("/profile/popover", s.ProfilePopover) + r.Mount("/account/migrate", s.AccountMigrationRouter()) + r.Route("/profile", func(r chi.Router) { r.Use(middleware.AuthMiddleware(s.oauth)) r.Get("/edit-bio", s.EditBioFragment) @@ -322,6 +325,17 @@ func (s *State) Core() http.HandlerFunc { } } +func (s *State) AccountMigrationRouter() http.Handler { + am := accountmigration.New( + log.SubLogger(s.logger, "accountmigration"), + s.oauth, + s.pages, + s.db, + s.enforcer, + ) + return am.Router() +} + func (s *State) SettingsRouter() http.Handler { settings := &settings.Settings{ Db: s.db, diff --git a/appview/state/state.go b/appview/state/state.go index ed72bed5..260f457f 100644 --- a/appview/state/state.go +++ b/appview/state/state.go @@ -13,6 +13,7 @@ import ( "tangled.org/core/api/tangled" "tangled.org/core/appview" + "tangled.org/core/appview/accountmigration" "tangled.org/core/appview/bsky" "tangled.org/core/appview/cache" "tangled.org/core/appview/cloudflare" @@ -75,6 +76,8 @@ type State struct { logger *slog.Logger cfClient *cloudflare.Client codesearch *codesearch.CodeSearch + + accountMigrationWorker *accountmigration.Worker } func Make(ctx context.Context, config *config.Config) (*State, error) { @@ -224,6 +227,13 @@ func Make(ctx context.Context, config *config.Config) (*State, error) { } knotstream.Start(ctx) + + if err := db.ReapStaleRunningGitRepoMigrations(ctx, d); err != nil { + logger.Warn("failed to reap stale gitrepo migrations", "err", err) + } + amWorker := accountmigration.NewWorker(d, oauth, config.Core.Dev, log.SubLogger(logger, "accountmigration")) + amWorker.Start(ctx) + state := &State{ db: d, notifier: notifier, @@ -243,6 +253,8 @@ func Make(ctx context.Context, config *config.Config) (*State, error) { logger: logger, cfClient: cfClient, codesearch: &codesearch.CodeSearch{Host: config.CodeSearch.ZoektUrl}, + + accountMigrationWorker: amWorker, } // fetch initial bluesky posts if configured