// Package fetch is a small HTTP client with an on-disk cache, per-host rate limiting, retries
// and forge token injection. Every network access in isomorph goes through it so that
// repeated runs are cheap and polite to the (volunteer-run) APIs we depend on.
package fetch

import (
	"context"
	"crypto/sha256"
	"encoding/hex"
	"encoding/json"
	"errors"
	"fmt"
	"io"
	"net/http"
	"net/http/cookiejar"
	"os"
	"path/filepath"
	"strconv"
	"strings"
	"sync"
	"time"
)

// DefaultUserAgent identifies the client; New appends the operator's contact when configured.
const DefaultUserAgent = "isomorph/0.1 (+https://tangled.org/odd.computer/isomorph)"

// Client fetches URLs with caching.
type Client struct {
	HTTP      *http.Client
	CacheDir  string
	TTL       time.Duration
	UserAgent string
	// MinInterval is the minimum gap between requests to the same host.
	MinInterval time.Duration
	// HostInterval overrides MinInterval for hosts with stricter published limits (crates.io
	// allows crawlers one request per second). A host's interval is never shortened below
	// MinInterval.
	HostInterval map[string]time.Duration
	// Contact (an e-mail address or URL) is appended to the User-Agent so operators can reach
	// whoever runs the crawl. Read from $ISOMORPH_CONTACT or ~/.config/isomorph/contact.
	Contact string
	// RequireContact lists hosts whose crawler policies require contact information in the
	// User-Agent (crates.io); requests to them fail while Contact is empty.
	RequireContact map[string]bool
	// Retries is the number of attempts on 429/5xx/network errors.
	Retries int
	// Tokens maps host -> bearer/private token. Populated from env by New.
	Tokens map[string]string
	// Offline makes Get fail rather than hit the network on a cache miss.
	Offline bool
	// Logf reports waits and other non-fatal events; nil discards them.
	Logf func(format string, args ...any)

	mu    sync.Mutex
	last  map[string]time.Time
	powMu sync.Mutex // serialises proof-of-work answers (pow.go)
}

// ErrOffline is returned on a cache miss when Client.Offline is set.
var ErrOffline = errors.New("fetch: offline and not cached")

// StatusError is returned for non-2xx responses.
type StatusError struct {
	URL    string
	Status int
	Body   string
}

func (e *StatusError) Error() string {
	return fmt.Sprintf("fetch %s: HTTP %d", e.URL, e.Status)
}

// IsNotFound reports whether err is a 404 StatusError.
func IsNotFound(err error) bool {
	var se *StatusError
	return errors.As(err, &se) && se.Status == http.StatusNotFound
}

// New builds a Client. cacheDir may be empty to use $ISOMORPH_CACHE or ~/.cache/isomorph.
// Tokens are read from GITHUB_TOKEN, GITLAB_TOKEN (gitlab.com), SALSA_TOKEN, OBS_TOKEN.
func New(cacheDir string) *Client {
	if cacheDir == "" {
		cacheDir = os.Getenv("ISOMORPH_CACHE")
	}
	if cacheDir == "" {
		if home, err := os.UserCacheDir(); err == nil {
			cacheDir = filepath.Join(home, "isomorph")
		} else {
			cacheDir = filepath.Join(os.TempDir(), "isomorph-cache")
		}
	}
	jar, _ := cookiejar.New(nil) // holds proof-of-work answers (pow.go)
	c := &Client{
		HTTP:        &http.Client{Timeout: 60 * time.Second, Jar: jar},
		CacheDir:    cacheDir,
		TTL:         7 * 24 * time.Hour,
		UserAgent:   DefaultUserAgent,
		MinInterval: 250 * time.Millisecond,
		HostInterval: map[string]time.Duration{
			"crates.io":        time.Second,
			"static.crates.io": time.Second,
			// The anonymous OBS API publishes no limit; bulk spec crawls stay at one a second.
			"api.opensuse.org": time.Second,
		},
		RequireContact: map[string]bool{"crates.io": true, "static.crates.io": true},
		Retries:        3,
		Tokens:         map[string]string{},
		last:           map[string]time.Time{},
	}
	if t := os.Getenv("GITHUB_TOKEN"); t != "" {
		c.Tokens["api.github.com"] = t
	}
	if t := os.Getenv("GITLAB_TOKEN"); t != "" {
		c.Tokens["gitlab.com"] = t
	}
	if t := os.Getenv("SALSA_TOKEN"); t != "" {
		c.Tokens["salsa.debian.org"] = t
	}
	c.Contact = contactFromEnvOrFile()
	if c.Contact != "" {
		c.UserAgent = strings.TrimSuffix(DefaultUserAgent, ")") + "; " + c.Contact + ")"
	}
	if t := tokenFromEnvOrFile("ARCH_GITLAB_TOKEN", "gitlab.archlinux.org"); t != "" {
		c.Tokens["gitlab.archlinux.org"] = t
	}
	if os.Getenv("ISOMORPH_OFFLINE") != "" {
		c.Offline = true
	}
	return c
}

// contactFromEnvOrFile returns $ISOMORPH_CONTACT, or else ~/.config/isomorph/contact.
func contactFromEnvOrFile() string {
	if v := strings.TrimSpace(os.Getenv("ISOMORPH_CONTACT")); v != "" {
		return v
	}
	home, err := os.UserHomeDir()
	if err != nil {
		return ""
	}
	b, err := os.ReadFile(filepath.Join(home, ".config", "isomorph", "contact"))
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(b))
}

// tokenFromEnvOrFile returns $env, or else the contents of ~/.config/isomorph/<host>.token
// (so a token need not live in the shell environment).
func tokenFromEnvOrFile(env, host string) string {
	if t := os.Getenv(env); t != "" {
		return t
	}
	home, err := os.UserHomeDir()
	if err != nil {
		return ""
	}
	b, err := os.ReadFile(filepath.Join(home, ".config", "isomorph", host+".token"))
	if err != nil {
		return ""
	}
	return strings.TrimSpace(string(b))
}

// Option tweaks a single request.
type Option func(*reqOpts)

type reqOpts struct {
	headers map[string]string
	ttl     time.Duration
	noCache bool
}

// WithHeader adds a request header.
func WithHeader(k, v string) Option {
	return func(o *reqOpts) {
		if o.headers == nil {
			o.headers = map[string]string{}
		}
		o.headers[k] = v
	}
}

// WithTTL overrides the cache TTL for this request (0 = never expire).
func WithTTL(d time.Duration) Option { return func(o *reqOpts) { o.ttl = d } }

// NoCache bypasses the cache for this request.
func NoCache() Option { return func(o *reqOpts) { o.noCache = true } }

// Get returns the body of url, from cache when fresh.
func (c *Client) Get(ctx context.Context, url string, opts ...Option) ([]byte, error) {
	var o reqOpts
	o.ttl = c.TTL
	for _, f := range opts {
		f(&o)
	}
	key := cacheKey(url, o.headers)
	if !o.noCache {
		if b, ok := c.readCache(key, o.ttl); ok {
			return b, nil
		}
	}
	if c.Offline {
		return nil, fmt.Errorf("%w: %s", ErrOffline, url)
	}
	var lastErr error
	for attempt := 0; attempt <= c.Retries; attempt++ {
		if attempt > 0 {
			backoff := time.Duration(1<<uint(attempt-1)) * time.Second
			select {
			case <-ctx.Done():
				return nil, ctx.Err()
			case <-time.After(backoff):
			}
		}
		b, status, retry, err := c.do(ctx, url, o.headers)
		if err == nil && status/100 == 2 {
			if !o.noCache {
				c.writeCache(key, b)
			}
			return b, nil
		}
		if err == nil {
			lastErr = &StatusError{URL: url, Status: status, Body: truncate(string(b), 512)}
			if !retry {
				// Cache 404s briefly so repeated lookups of missing files are cheap.
				if status == http.StatusNotFound && !o.noCache {
					c.writeCacheStatus(key, status)
				}
				return nil, lastErr
			}
		} else {
			lastErr = err
			if ctx.Err() != nil {
				return nil, ctx.Err()
			}
			if !retry {
				break
			}
		}
	}
	return nil, lastErr
}

// GetJSON fetches url and decodes it into v.
func (c *Client) GetJSON(ctx context.Context, url string, v any, opts ...Option) error {
	opts = append([]Option{WithHeader("Accept", "application/json")}, opts...)
	b, err := c.Get(ctx, url, opts...)
	if err != nil {
		return err
	}
	if err := json.Unmarshal(b, v); err != nil {
		return fmt.Errorf("fetch %s: decode: %w", url, err)
	}
	return nil
}

// GetString fetches url and returns the body as a string.
func (c *Client) GetString(ctx context.Context, url string, opts ...Option) (string, error) {
	b, err := c.Get(ctx, url, opts...)
	return string(b), err
}

func (c *Client) do(ctx context.Context, url string, headers map[string]string) (body []byte, status int, retry bool, err error) {
	body, status, header, err := c.send(ctx, url, headers)
	if err == nil && isPoWChallenge(status, body) {
		if perr := c.payPoW(ctx, origin(url)); perr != nil {
			return body, status, false, fmt.Errorf("fetch %s: proof-of-work: %w", url, perr)
		}
		body, status, header, err = c.send(ctx, url, headers)
	}
	if err != nil {
		return body, status, true, err
	}
	retry = status == http.StatusTooManyRequests || status >= 500
	if status == http.StatusTooManyRequests || status == http.StatusForbidden {
		// Honour Retry-After / GitHub secondary limits by sleeping a bit before the next attempt.
		if ra := header.Get("Retry-After"); ra != "" {
			if secs, perr := strconv.Atoi(ra); perr == nil && secs > 0 && secs <= 120 {
				time.Sleep(time.Duration(secs) * time.Second)
				retry = true
			}
		}
		if remaining := firstHeader(header, "X-RateLimit-Remaining", "RateLimit-Remaining"); remaining == "0" {
			// GitHub sends X-RateLimit-*, GitLab RateLimit-*. Short windows (GitHub search: one
			// minute) are waited out; a long one (GitHub's hourly core limit, GitLab's
			// unauthenticated API window) is the caller's problem, and retrying only adds load.
			reset := firstHeader(header, "X-RateLimit-Reset", "RateLimit-Reset")
			if wait := rateLimitWait(reset, time.Now()); wait > 0 && wait <= maxRateLimitWait {
				c.logf("fetch %s: rate limited, waiting %s for the window to reset", url, wait)
				select {
				case <-ctx.Done():
					return nil, status, false, ctx.Err()
				case <-time.After(wait):
				}
				retry = true
			} else {
				retry = false
				err = fmt.Errorf("fetch %s: rate limited (X-RateLimit-Reset=%s)", url, reset)
			}
		}
	}
	return body, status, retry, err
}

// send performs one GET with the client's User-Agent, tokens and per-host throttle.
func (c *Client) send(ctx context.Context, url string, headers map[string]string) (body []byte, status int, header http.Header, err error) {
	req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil)
	if err != nil {
		return nil, 0, nil, err
	}
	host := req.URL.Hostname()
	if c.RequireContact[host] && c.Contact == "" {
		return nil, 0, nil, fmt.Errorf("fetch %s: %s requires contact information in the User-Agent; set ISOMORPH_CONTACT or ~/.config/isomorph/contact", url, host)
	}
	req.Header.Set("User-Agent", c.UserAgent)
	if tok := c.Tokens[host]; tok != "" {
		switch host {
		case "api.github.com":
			req.Header.Set("Authorization", "Bearer "+tok)
		default:
			req.Header.Set("PRIVATE-TOKEN", tok)
		}
	}
	for k, v := range headers {
		req.Header.Set(k, v)
	}
	c.throttle(host)
	resp, err := c.HTTP.Do(req)
	if err != nil {
		return nil, 0, nil, err
	}
	defer resp.Body.Close()
	body, err = io.ReadAll(io.LimitReader(resp.Body, 64<<20))
	return body, resp.StatusCode, resp.Header, err
}

func firstHeader(h http.Header, names ...string) string {
	for _, n := range names {
		if v := h.Get(n); v != "" {
			return v
		}
	}
	return ""
}

// maxRateLimitWait bounds how long a request waits for an exhausted rate-limit window.
const maxRateLimitWait = 90 * time.Second

// rateLimitWait is the time until an X-RateLimit-Reset epoch (plus a second of slack), or 0
// when the header is missing or malformed.
func rateLimitWait(reset string, now time.Time) time.Duration {
	secs, err := strconv.ParseInt(reset, 10, 64)
	if err != nil || secs <= 0 {
		return 0
	}
	return max(time.Unix(secs, 0).Sub(now), 0) + time.Second
}

func (c *Client) throttle(host string) {
	interval := max(c.MinInterval, c.HostInterval[host])
	if interval <= 0 {
		return
	}
	c.mu.Lock()
	last := c.last[host]
	wait := interval - time.Since(last)
	if wait > 0 {
		c.last[host] = time.Now().Add(wait)
	} else {
		c.last[host] = time.Now()
	}
	c.mu.Unlock()
	if wait > 0 {
		time.Sleep(wait)
	}
}

func cacheKey(url string, headers map[string]string) string {
	h := sha256.New()
	h.Write([]byte(url))
	if a := headers["Accept"]; a != "" {
		h.Write([]byte("\nAccept:" + a))
	}
	return hex.EncodeToString(h.Sum(nil))
}

func (c *Client) cachePath(key string) string {
	return filepath.Join(c.CacheDir, key[:2], key)
}

func (c *Client) readCache(key string, ttl time.Duration) ([]byte, bool) {
	p := c.cachePath(key)
	st, err := os.Stat(p)
	if err != nil {
		return nil, false
	}
	if ttl > 0 && time.Since(st.ModTime()) > ttl {
		return nil, false
	}
	b, err := os.ReadFile(p)
	if err != nil {
		return nil, false
	}
	if strings.HasPrefix(string(b), "\x00STATUS:") {
		return nil, false // negative cache entries are handled in Get via readStatus
	}
	return b, true
}

// writeCache stores b under key, through a temporary file so a reader never sees a partial one.
func (c *Client) writeCache(key string, b []byte) {
	p := c.cachePath(key)
	if err := os.MkdirAll(filepath.Dir(p), 0o755); err != nil {
		return
	}
	tmp := p + ".tmp"
	if err := os.WriteFile(tmp, b, 0o644); err != nil {
		return
	}
	_ = os.Rename(tmp, p)
}

func (c *Client) writeCacheStatus(key string, status int) {
	c.writeCache(key+".neg", []byte("\x00STATUS:"+strconv.Itoa(status)))
}

// CachedStatus returns a negatively cached status (e.g. 404) for url if one is recorded and
// younger than an hour.
func (c *Client) CachedStatus(url string) (int, bool) {
	key := cacheKey(url, nil) + ".neg"
	b, ok := c.readCacheRaw(key, time.Hour)
	if !ok || !strings.HasPrefix(string(b), "\x00STATUS:") {
		return 0, false
	}
	n, err := strconv.Atoi(strings.TrimPrefix(string(b), "\x00STATUS:"))
	return n, err == nil
}

func (c *Client) readCacheRaw(key string, ttl time.Duration) ([]byte, bool) {
	p := c.cachePath(key)
	st, err := os.Stat(p)
	if err != nil || (ttl > 0 && time.Since(st.ModTime()) > ttl) {
		return nil, false
	}
	b, err := os.ReadFile(p)
	return b, err == nil
}

func truncate(s string, n int) string {
	if len(s) <= n {
		return s
	}
	return s[:n] + "..."
}

// Cached reports whether Get(url, opts...) would be served from the disk cache without a
// network round trip. Callers that pace themselves (the .SRCINFO and OBS crawls) use it to
// skip the wait for cached responses.
func (c *Client) Cached(url string, opts ...Option) bool {
	var o reqOpts
	o.ttl = c.TTL
	for _, f := range opts {
		f(&o)
	}
	if o.noCache {
		return false
	}
	_, ok := c.readCache(cacheKey(url, o.headers), o.ttl)
	return ok
}

// CachedJSON is Cached with the Accept header GetJSON sends.
func (c *Client) CachedJSON(url string, opts ...Option) bool {
	return c.Cached(url, append([]Option{WithHeader("Accept", "application/json")}, opts...)...)
}

func (c *Client) logf(format string, args ...any) {
	if c.Logf != nil {
		c.Logf(format, args...)
	}
}