Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483// Package fetch is a small HTTP client with an on-disk cache, per-host rate limiting, retries// and forge token injection. Every network access in isomorph goes through it so that// repeated runs are cheap and polite to the (volunteer-run) APIs we depend on.package fetch
import ( "context" "crypto/sha256" "encoding/hex" "encoding/json" "errors" "fmt" "io" "net/http" "net/http/cookiejar" "os" "path/filepath" "strconv" "strings" "sync" "time")
// DefaultUserAgent identifies the client; New appends the operator's contact when configured.const DefaultUserAgent = "isomorph/0.1 (+https://tangled.org/odd.computer/isomorph)"
// Client fetches URLs with caching.type Client struct { HTTP *http.Client CacheDir string TTL time.Duration UserAgent string // MinInterval is the minimum gap between requests to the same host. MinInterval time.Duration // HostInterval overrides MinInterval for hosts with stricter published limits (crates.io // allows crawlers one request per second). A host's interval is never shortened below // MinInterval. HostInterval map[string]time.Duration // Contact (an e-mail address or URL) is appended to the User-Agent so operators can reach // whoever runs the crawl. Read from $ISOMORPH_CONTACT or ~/.config/isomorph/contact. Contact string // RequireContact lists hosts whose crawler policies require contact information in the // User-Agent (crates.io); requests to them fail while Contact is empty. RequireContact map[string]bool // Retries is the number of attempts on 429/5xx/network errors. Retries int // Tokens maps host -> bearer/private token. Populated from env by New. Tokens map[string]string // Offline makes Get fail rather than hit the network on a cache miss. Offline bool // Logf reports waits and other non-fatal events; nil discards them. Logf func(format string, args ...any)
mu sync.Mutex last map[string]time.Time powMu sync.Mutex // serialises proof-of-work answers (pow.go)}
// ErrOffline is returned on a cache miss when Client.Offline is set.var ErrOffline = errors.New("fetch: offline and not cached")
// StatusError is returned for non-2xx responses.type StatusError struct { URL string Status int Body string}
func (e *StatusError) Error() string { return fmt.Sprintf("fetch %s: HTTP %d", e.URL, e.Status)}
// IsNotFound reports whether err is a 404 StatusError.func IsNotFound(err error) bool { var se *StatusError return errors.As(err, &se) && se.Status == http.StatusNotFound}
// New builds a Client. cacheDir may be empty to use $ISOMORPH_CACHE or ~/.cache/isomorph.// Tokens are read from GITHUB_TOKEN, GITLAB_TOKEN (gitlab.com), SALSA_TOKEN, OBS_TOKEN.func New(cacheDir string) *Client { if cacheDir == "" { cacheDir = os.Getenv("ISOMORPH_CACHE") } if cacheDir == "" { if home, err := os.UserCacheDir(); err == nil { cacheDir = filepath.Join(home, "isomorph") } else { cacheDir = filepath.Join(os.TempDir(), "isomorph-cache") } } jar, _ := cookiejar.New(nil) // holds proof-of-work answers (pow.go) c := &Client{ HTTP: &http.Client{Timeout: 60 * time.Second, Jar: jar}, CacheDir: cacheDir, TTL: 7 * 24 * time.Hour, UserAgent: DefaultUserAgent, MinInterval: 250 * time.Millisecond, HostInterval: map[string]time.Duration{ "crates.io": time.Second, "static.crates.io": time.Second, // The anonymous OBS API publishes no limit; bulk spec crawls stay at one a second. "api.opensuse.org": time.Second, }, RequireContact: map[string]bool{"crates.io": true, "static.crates.io": true}, Retries: 3, Tokens: map[string]string{}, last: map[string]time.Time{}, } if t := os.Getenv("GITHUB_TOKEN"); t != "" { c.Tokens["api.github.com"] = t } if t := os.Getenv("GITLAB_TOKEN"); t != "" { c.Tokens["gitlab.com"] = t } if t := os.Getenv("SALSA_TOKEN"); t != "" { c.Tokens["salsa.debian.org"] = t } c.Contact = contactFromEnvOrFile() if c.Contact != "" { c.UserAgent = strings.TrimSuffix(DefaultUserAgent, ")") + "; " + c.Contact + ")" } if t := tokenFromEnvOrFile("ARCH_GITLAB_TOKEN", "gitlab.archlinux.org"); t != "" { c.Tokens["gitlab.archlinux.org"] = t } if os.Getenv("ISOMORPH_OFFLINE") != "" { c.Offline = true } return c}
// contactFromEnvOrFile returns $ISOMORPH_CONTACT, or else ~/.config/isomorph/contact.func contactFromEnvOrFile() string { if v := strings.TrimSpace(os.Getenv("ISOMORPH_CONTACT")); v != "" { return v } home, err := os.UserHomeDir() if err != nil { return "" } b, err := os.ReadFile(filepath.Join(home, ".config", "isomorph", "contact")) if err != nil { return "" } return strings.TrimSpace(string(b))}
// tokenFromEnvOrFile returns $env, or else the contents of ~/.config/isomorph/<host>.token// (so a token need not live in the shell environment).func tokenFromEnvOrFile(env, host string) string { if t := os.Getenv(env); t != "" { return t } home, err := os.UserHomeDir() if err != nil { return "" } b, err := os.ReadFile(filepath.Join(home, ".config", "isomorph", host+".token")) if err != nil { return "" } return strings.TrimSpace(string(b))}
// Option tweaks a single request.type Option func(*reqOpts)
type reqOpts struct { headers map[string]string ttl time.Duration noCache bool}
// WithHeader adds a request header.func WithHeader(k, v string) Option { return func(o *reqOpts) { if o.headers == nil { o.headers = map[string]string{} } o.headers[k] = v }}
// WithTTL overrides the cache TTL for this request (0 = never expire).func WithTTL(d time.Duration) Option { return func(o *reqOpts) { o.ttl = d } }
// NoCache bypasses the cache for this request.func NoCache() Option { return func(o *reqOpts) { o.noCache = true } }
// Get returns the body of url, from cache when fresh.func (c *Client) Get(ctx context.Context, url string, opts ...Option) ([]byte, error) { var o reqOpts o.ttl = c.TTL for _, f := range opts { f(&o) } key := cacheKey(url, o.headers) if !o.noCache { if b, ok := c.readCache(key, o.ttl); ok { return b, nil } } if c.Offline { return nil, fmt.Errorf("%w: %s", ErrOffline, url) } var lastErr error for attempt := 0; attempt <= c.Retries; attempt++ { if attempt > 0 { backoff := time.Duration(1<<uint(attempt-1)) * time.Second select { case <-ctx.Done(): return nil, ctx.Err() case <-time.After(backoff): } } b, status, retry, err := c.do(ctx, url, o.headers) if err == nil && status/100 == 2 { if !o.noCache { c.writeCache(key, b) } return b, nil } if err == nil { lastErr = &StatusError{URL: url, Status: status, Body: truncate(string(b), 512)} if !retry { // Cache 404s briefly so repeated lookups of missing files are cheap. if status == http.StatusNotFound && !o.noCache { c.writeCacheStatus(key, status) } return nil, lastErr } } else { lastErr = err if ctx.Err() != nil { return nil, ctx.Err() } if !retry { break } } } return nil, lastErr}
// GetJSON fetches url and decodes it into v.func (c *Client) GetJSON(ctx context.Context, url string, v any, opts ...Option) error { opts = append([]Option{WithHeader("Accept", "application/json")}, opts...) b, err := c.Get(ctx, url, opts...) if err != nil { return err } if err := json.Unmarshal(b, v); err != nil { return fmt.Errorf("fetch %s: decode: %w", url, err) } return nil}
// GetString fetches url and returns the body as a string.func (c *Client) GetString(ctx context.Context, url string, opts ...Option) (string, error) { b, err := c.Get(ctx, url, opts...) return string(b), err}
func (c *Client) do(ctx context.Context, url string, headers map[string]string) (body []byte, status int, retry bool, err error) { body, status, header, err := c.send(ctx, url, headers) if err == nil && isPoWChallenge(status, body) { if perr := c.payPoW(ctx, origin(url)); perr != nil { return body, status, false, fmt.Errorf("fetch %s: proof-of-work: %w", url, perr) } body, status, header, err = c.send(ctx, url, headers) } if err != nil { return body, status, true, err } retry = status == http.StatusTooManyRequests || status >= 500 if status == http.StatusTooManyRequests || status == http.StatusForbidden { // Honour Retry-After / GitHub secondary limits by sleeping a bit before the next attempt. if ra := header.Get("Retry-After"); ra != "" { if secs, perr := strconv.Atoi(ra); perr == nil && secs > 0 && secs <= 120 { time.Sleep(time.Duration(secs) * time.Second) retry = true } } if remaining := firstHeader(header, "X-RateLimit-Remaining", "RateLimit-Remaining"); remaining == "0" { // GitHub sends X-RateLimit-*, GitLab RateLimit-*. Short windows (GitHub search: one // minute) are waited out; a long one (GitHub's hourly core limit, GitLab's // unauthenticated API window) is the caller's problem, and retrying only adds load. reset := firstHeader(header, "X-RateLimit-Reset", "RateLimit-Reset") if wait := rateLimitWait(reset, time.Now()); wait > 0 && wait <= maxRateLimitWait { c.logf("fetch %s: rate limited, waiting %s for the window to reset", url, wait) select { case <-ctx.Done(): return nil, status, false, ctx.Err() case <-time.After(wait): } retry = true } else { retry = false err = fmt.Errorf("fetch %s: rate limited (X-RateLimit-Reset=%s)", url, reset) } } } return body, status, retry, err}
// send performs one GET with the client's User-Agent, tokens and per-host throttle.func (c *Client) send(ctx context.Context, url string, headers map[string]string) (body []byte, status int, header http.Header, err error) { req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { return nil, 0, nil, err } host := req.URL.Hostname() if c.RequireContact[host] && c.Contact == "" { return nil, 0, nil, fmt.Errorf("fetch %s: %s requires contact information in the User-Agent; set ISOMORPH_CONTACT or ~/.config/isomorph/contact", url, host) } req.Header.Set("User-Agent", c.UserAgent) if tok := c.Tokens[host]; tok != "" { switch host { case "api.github.com": req.Header.Set("Authorization", "Bearer "+tok) default: req.Header.Set("PRIVATE-TOKEN", tok) } } for k, v := range headers { req.Header.Set(k, v) } c.throttle(host) resp, err := c.HTTP.Do(req) if err != nil { return nil, 0, nil, err } defer resp.Body.Close() body, err = io.ReadAll(io.LimitReader(resp.Body, 64<<20)) return body, resp.StatusCode, resp.Header, err}
func firstHeader(h http.Header, names ...string) string { for _, n := range names { if v := h.Get(n); v != "" { return v } } return ""}
// maxRateLimitWait bounds how long a request waits for an exhausted rate-limit window.const maxRateLimitWait = 90 * time.Second
// rateLimitWait is the time until an X-RateLimit-Reset epoch (plus a second of slack), or 0// when the header is missing or malformed.func rateLimitWait(reset string, now time.Time) time.Duration { secs, err := strconv.ParseInt(reset, 10, 64) if err != nil || secs <= 0 { return 0 } return max(time.Unix(secs, 0).Sub(now), 0) + time.Second}
func (c *Client) throttle(host string) { interval := max(c.MinInterval, c.HostInterval[host]) if interval <= 0 { return } c.mu.Lock() last := c.last[host] wait := interval - time.Since(last) if wait > 0 { c.last[host] = time.Now().Add(wait) } else { c.last[host] = time.Now() } c.mu.Unlock() if wait > 0 { time.Sleep(wait) }}
func cacheKey(url string, headers map[string]string) string { h := sha256.New() h.Write([]byte(url)) if a := headers["Accept"]; a != "" { h.Write([]byte("\nAccept:" + a)) } return hex.EncodeToString(h.Sum(nil))}
func (c *Client) cachePath(key string) string { return filepath.Join(c.CacheDir, key[:2], key)}
func (c *Client) readCache(key string, ttl time.Duration) ([]byte, bool) { p := c.cachePath(key) st, err := os.Stat(p) if err != nil { return nil, false } if ttl > 0 && time.Since(st.ModTime()) > ttl { return nil, false } b, err := os.ReadFile(p) if err != nil { return nil, false } if strings.HasPrefix(string(b), "\x00STATUS:") { return nil, false // negative cache entries are handled in Get via readStatus } return b, true}
// writeCache stores b under key, through a temporary file so a reader never sees a partial one.func (c *Client) writeCache(key string, b []byte) { p := c.cachePath(key) if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil { return } tmp := p + ".tmp" if err := os.WriteFile(tmp, b, 0o644); err != nil { return } _ = os.Rename(tmp, p)}
func (c *Client) writeCacheStatus(key string, status int) { c.writeCache(key+".neg", []byte("\x00STATUS:"+strconv.Itoa(status)))}
// CachedStatus returns a negatively cached status (e.g. 404) for url if one is recorded and// younger than an hour.func (c *Client) CachedStatus(url string) (int, bool) { key := cacheKey(url, nil) + ".neg" b, ok := c.readCacheRaw(key, time.Hour) if !ok || !strings.HasPrefix(string(b), "\x00STATUS:") { return 0, false } n, err := strconv.Atoi(strings.TrimPrefix(string(b), "\x00STATUS:")) return n, err == nil}
func (c *Client) readCacheRaw(key string, ttl time.Duration) ([]byte, bool) { p := c.cachePath(key) st, err := os.Stat(p) if err != nil || (ttl > 0 && time.Since(st.ModTime()) > ttl) { return nil, false } b, err := os.ReadFile(p) return b, err == nil}
func truncate(s string, n int) string { if len(s) <= n { return s } return s[:n] + "..."}
// Cached reports whether Get(url, opts...) would be served from the disk cache without a// network round trip. Callers that pace themselves (the .SRCINFO and OBS crawls) use it to// skip the wait for cached responses.func (c *Client) Cached(url string, opts ...Option) bool { var o reqOpts o.ttl = c.TTL for _, f := range opts { f(&o) } if o.noCache { return false } _, ok := c.readCache(cacheKey(url, o.headers), o.ttl) return ok}
// CachedJSON is Cached with the Accept header GetJSON sends.func (c *Client) CachedJSON(url string, opts ...Option) bool { return c.Cached(url, append([]Option{WithHeader("Accept", "application/json")}, opts...)...)}
func (c *Client) logf(format string, args ...any) { if c.Logf != nil { c.Logf(format, args...) }}