ContentAPI Crawler
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340package cli
import ( "context" "encoding/json" "errors" "flag" "fmt" "io" "net/url" "os" "strings"
"github.com/darien/qcscrawl/internal/config" "github.com/darien/qcscrawl/internal/contentapi" "github.com/openclaw/crawlkit/control")
type runtime struct { ctx context.Context out, errOut io.Writer configPath string}
// Run executes one read-only control command. init only writes local configuration.func Run(ctx context.Context, args []string, out, errOut io.Writer) error { if out == nil { out = io.Discard } if errOut == nil { errOut = io.Discard } if ctx == nil { ctx = context.Background() } path, args, err := globalConfig(args) if err != nil { return err } if len(args) > 0 && args[0] == "--" { args = args[1:] } r := &runtime{ctx: ctx, out: out, errOut: errOut, configPath: path} if len(args) == 0 { return r.help(nil) } if args[0] == "help" || args[0] == "--help" || args[0] == "-h" { return r.help(args[1:]) } if len(args) > 1 && (args[1] == "--help" || args[1] == "-h") { return r.help(args[:1]) } switch args[0] { case "init": return r.init(args[1:]) case "doctor": return r.doctor(args[1:]) case "metadata": return r.metadata(args[1:]) case "status": return r.status(args[1:]) case "sync": return r.sync(args[1:]) case "search": return r.search(args[1:]) case "messages": return r.messages(args[1:]) case "content": return r.content(args[1:]) case "users": return r.users(args[1:]) case "sql": return r.sql(args[1:]) default: return fmt.Errorf("unknown command %q", args[0]) }}
func globalConfig(args []string) (string, []string, error) { var path string var rest []string for i := 0; i < len(args); i++ { if args[i] == "--config" || args[i] == "-c" { if i+1 >= len(args) { return "", nil, errors.New("--config requires a path") } path = args[i+1] i++ continue } if strings.HasPrefix(args[i], "--config=") { path = strings.TrimPrefix(args[i], "--config=") continue } rest = append(rest, args[i]) } return config.ResolvePath(path), rest, nil}func (r *runtime) help(args []string) error { if len(args) > 0 { if text, ok := commandHelp[args[0]]; ok { _, err := fmt.Fprint(r.out, text) return err } return fmt.Errorf("unknown command %q", args[0]) } _, err := fmt.Fprint(r.out, `qcscrawl — local ContentAPI archive browser
Usage: qcscrawl [--config PATH] COMMAND [options]
Offline archive commands (no provider request; existing database opened read-only): search QUERY [--include-deleted] [--limit N] [--json] messages [--content ID] [--user ID] [--include-deleted] [--limit N] [--json] content ID [--json] users [QUERY] [--json] sql QUERY [--json]
Setup and provider commands: init doctor [--json] sync [--full | --repair] [--json]
Control commands: metadata [--json] status [--json]
Use COMMAND --help for details. token_env is an environment-variable name(such as QCSCRAWL_TOKEN), never a bearer token. Use --config explicitly whenan agent must avoid the default home profile.`) return err}
var commandHelp = map[string]string{ "search": `Usage: qcscrawl [--config PATH] search QUERY [--include-deleted] [--limit N] [--json]Search archived message text locally. No provider request or database write.`, "messages": `Usage: qcscrawl [--config PATH] messages [--content ID] [--user ID] [--include-deleted] [--limit N] [--json]Browse archived messages locally. No provider request or database write.`, "content": `Usage: qcscrawl [--config PATH] content ID [--json]Show one archived content record locally.`, "users": `Usage: qcscrawl [--config PATH] users [QUERY] [--json]List archived users, optionally filtering usernames locally.`, "sql": `Usage: qcscrawl [--config PATH] sql QUERY [--json]Run one local read-only SQL statement. Writes and multiple statements are rejected.`, "init": `Usage: qcscrawl [--config PATH] initCreate local configuration and runtime directories.`, "doctor": `Usage: qcscrawl [--config PATH] doctor [--json]Check configuration and provider connectivity. Makes read-only provider requests.`, "sync": `Usage: qcscrawl [--config PATH] sync [--full | --repair] [--json]Synchronize with the configured provider. This is a network operation.`, "metadata": `Usage: qcscrawl metadata [--json]Print CrawlKit control metadata.`, "status": `Usage: qcscrawl [--config PATH] status [--json]Show local configuration/archive readiness.`,}
func (r *runtime) printJSON(value any) error { enc := json.NewEncoder(r.out) enc.SetEscapeHTML(false) return enc.Encode(value)}func (r *runtime) metadata(args []string) error { fs := flag.NewFlagSet("metadata", flag.ContinueOnError) fs.SetOutput(io.Discard) jsonOut := fs.Bool("json", false, "output JSON") if err := fs.Parse(args); err != nil { return err } if fs.NArg() != 0 { return errors.New("metadata takes flags only") } c := config.Default() m := control.NewManifest(config.AppID, "QCS ContentAPI Crawler", config.BinaryName) m.Description = "Read-only local-first ContentAPI archive crawler." m.Paths = control.Paths{DefaultConfig: r.configPath, ConfigEnv: config.ConfigEnv, DefaultDatabase: c.DBPath, DefaultCache: c.CacheDir, DefaultLogs: c.LogDir, DefaultShare: c.ShareDir} m.Capabilities = []string{"metadata", "status", "diagnostics", "doctor", "sync", "search", "messages", "content", "users", "sql"} m.Privacy = control.Privacy{ContainsPrivateMessages: true, ExportsSecrets: false, LocalOnlyScopes: []string{"contentapi", "sqlite"}} m.Commands = map[string]control.Command{ "init": {Title: "Initialize configuration", Argv: []string{config.BinaryName, "init"}, Mutates: true}, "doctor": {Title: "Diagnostics", Argv: []string{config.BinaryName, "doctor", "--json"}, JSON: true}, "status": {Title: "Archive status", Argv: []string{config.BinaryName, "status", "--json"}, JSON: true}, "sync": {Title: "Synchronize archive", Argv: []string{config.BinaryName, "sync", "--json"}, JSON: true}, } if *jsonOut { return r.printJSON(m) } _, err := fmt.Fprintf(r.out, "%s (%s)\n", m.DisplayName, m.Binary.Name) return err}
type doctorResult struct { SchemaVersion string `json:"schema_version"` AppID string `json:"app_id"` ConfigPath string `json:"config_path"` BaseURL string `json:"base_url,omitempty"` TokenEnv string `json:"token_env,omitempty"` TokenPresent bool `json:"token_present"` Status *contentapi.Status `json:"status,omitempty"` AboutOK bool `json:"about_ok"` TokenOK bool `json:"token_ok,omitempty"` OK bool `json:"ok"` Errors []string `json:"errors,omitempty"`}
func (r *runtime) doctor(args []string) error { fs := flag.NewFlagSet("doctor", flag.ContinueOnError) fs.SetOutput(io.Discard) jsonOut := fs.Bool("json", false, "output JSON") if err := fs.Parse(args); err != nil { return err } if fs.NArg() != 0 { return errors.New("doctor takes flags only") } result := doctorResult{SchemaVersion: control.SchemaVersion, AppID: config.AppID, ConfigPath: r.configPath} cfg, err := config.Load(r.configPath) if err != nil { result.Errors = append(result.Errors, "load config: "+err.Error()) return r.finishDoctor(result, *jsonOut, err) } result.BaseURL, result.TokenEnv, result.TokenPresent = safeURL(cfg.BaseURL), cfg.TokenEnv, cfg.TokenPresent() client, err := contentapi.New(cfg.BaseURL, cfg.Token()) if err != nil { result.Errors = append(result.Errors, "client: "+err.Error()) return r.finishDoctor(result, *jsonOut, err) } status, _, err := client.Status(r.ctx) if err != nil { result.Errors = append(result.Errors, "status: "+err.Error()) } else { result.Status = &status } _, _, err = client.About(r.ctx) if err != nil { result.Errors = append(result.Errors, "about: "+err.Error()) } else { result.AboutOK = true } if result.TokenPresent { _, _, err = client.TokenStatus(r.ctx) if err != nil { result.Errors = append(result.Errors, "token: "+err.Error()) } else { result.TokenOK = true } } result.OK = len(result.Errors) == 0 if err := r.finishDoctor(result, *jsonOut, nil); err != nil { return err } if !result.OK { return errors.New("doctor found problems") } return nil}func (r *runtime) finishDoctor(v doctorResult, jsonOut bool, cause error) error { if jsonOut { if err := r.printJSON(v); err != nil { return err } } else { if v.OK { _, _ = fmt.Fprintln(r.out, "ok: ContentAPI status and about are reachable") } else { _, _ = fmt.Fprintln(r.out, "doctor: problems found") for _, e := range v.Errors { _, _ = fmt.Fprintln(r.out, "-", e) } } } return cause}
func safeURL(raw string) string { u, err := url.Parse(raw) if err != nil { return "" } u.User = nil u.RawQuery, u.Fragment = "", "" return u.String()}
func (r *runtime) status(args []string) error { fs := flag.NewFlagSet("status", flag.ContinueOnError) fs.SetOutput(io.Discard) jsonOut := fs.Bool("json", false, "output JSON") if err := fs.Parse(args); err != nil { return err } if fs.NArg() != 0 { return errors.New("status takes flags only") } cfg, err := config.Load(r.configPath) if err != nil { if !errors.Is(err, os.ErrNotExist) { return err } cfg = config.Default() if err := cfg.Normalize(); err != nil { return err } } state, summary := "uninitialized", "archive has not been initialized" if _, err := os.Stat(cfg.DBPath); err == nil { state, summary = "ready", "archive database present" } else if !errors.Is(err, os.ErrNotExist) { return err } s := control.NewStatus(config.AppID, summary) s.State, s.ConfigPath, s.DatabasePath = state, r.configPath, cfg.DBPath db := control.SQLiteDatabase("primary", "ContentAPI archive", "archive", cfg.DBPath, true, nil) s.DatabaseBytes = db.Bytes s.WALBytes = fileSize(cfg.DBPath + "-wal") s.Databases = []control.Database{db} if *jsonOut { return r.printJSON(s) } _, err = fmt.Fprintf(r.out, "%s: %s\n", s.State, s.Summary) return err}func fileSize(path string) int64 { info, err := os.Stat(path) if err != nil { return 0 } return info.Size()}