Something went wrong. Try again.
Maps Linux distribution source packages to their upstream repositories, and through them to each other
Something went wrong. Try again.
123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221package fetch
import ( "context" "fmt" "net/http" "net/http/httptest" "strconv" "strings" "sync/atomic" "testing" "time")
func TestGetCaches(t *testing.T) { var hits int32 c := New(t.TempDir()) c.MinInterval = 0 // The UA is the default plus the configured contact, if any (ISOMORPH_CONTACT or the // contact file), so compare with the client's own. if !strings.HasPrefix(c.UserAgent, strings.TrimSuffix(DefaultUserAgent, ")")) { t.Fatalf("UA %q does not extend the default", c.UserAgent) } srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) if r.Header.Get("User-Agent") != c.UserAgent { t.Errorf("missing UA: %q", r.Header.Get("User-Agent")) } w.Write([]byte("hello")) })) defer srv.Close() ctx := context.Background() for i := 0; i < 3; i++ { b, err := c.Get(ctx, srv.URL+"/x") if err != nil || string(b) != "hello" { t.Fatalf("get: %v %q", err, b) } } if hits != 1 { t.Fatalf("expected 1 hit, got %d", hits) } if _, err := c.Get(ctx, srv.URL+"/x", NoCache()); err != nil { t.Fatal(err) } if hits != 2 { t.Fatalf("expected 2 hits after NoCache, got %d", hits) }}
func TestGetRetriesAnd404(t *testing.T) { var n int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/flaky": if atomic.AddInt32(&n, 1) < 2 { w.WriteHeader(503) return } w.Write([]byte("ok")) case "/missing": w.WriteHeader(404) } })) defer srv.Close() c := New(t.TempDir()) c.MinInterval = 0 c.Retries = 2 // Shrink backoff by using a context deadline generous enough for one 1s retry. ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) defer cancel() b, err := c.Get(ctx, srv.URL+"/flaky") if err != nil || string(b) != "ok" { t.Fatalf("flaky: %v %q", err, b) } _, err = c.Get(ctx, srv.URL+"/missing") if !IsNotFound(err) { t.Fatalf("expected 404, got %v", err) }}
// TestRateLimitWindow: an exhausted window that resets within maxRateLimitWait is waited// out; one that resets later fails at once, without retries.func TestRateLimitWindow(t *testing.T) { var short, long int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { switch r.URL.Path { case "/search": if atomic.AddInt32(&short, 1) == 1 { w.Header().Set("X-RateLimit-Remaining", "0") w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Unix(), 10)) w.WriteHeader(403) return } w.Write([]byte("ok")) case "/core": atomic.AddInt32(&long, 1) w.Header().Set("X-RateLimit-Remaining", "0") w.Header().Set("X-RateLimit-Reset", strconv.FormatInt(time.Now().Add(time.Hour).Unix(), 10)) w.WriteHeader(403) } })) defer srv.Close() c := New(t.TempDir()) c.MinInterval = 0 c.Retries = 3 ctx, cancel := context.WithTimeout(context.Background(), 20*time.Second) defer cancel() if b, err := c.Get(ctx, srv.URL+"/search"); err != nil || string(b) != "ok" { t.Fatalf("short window: %v %q", err, b) } start := time.Now() if _, err := c.Get(ctx, srv.URL+"/core"); err == nil || !strings.Contains(err.Error(), "rate limited") { t.Fatalf("long window: %v", err) } if n := atomic.LoadInt32(&long); n != 1 || time.Since(start) > 2*time.Second { t.Fatalf("long window retried %d times over %s", n, time.Since(start)) } // GitLab spells the headers without X- and answers 429. var gitlab int32 srv2 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&gitlab, 1) w.Header().Set("RateLimit-Remaining", "0") w.Header().Set("RateLimit-Reset", strconv.FormatInt(time.Now().Add(10*time.Minute).Unix(), 10)) w.WriteHeader(429) })) defer srv2.Close() if _, err := c.Get(ctx, srv2.URL+"/api"); err == nil || !strings.Contains(err.Error(), "rate limited") || atomic.LoadInt32(&gitlab) != 1 { t.Fatalf("GitLab long window: %v after %d requests", err, gitlab) } if w := rateLimitWait("bogus", time.Now()); w != 0 { t.Fatalf("malformed reset: %v", w) }}
func TestOffline(t *testing.T) { c := New(t.TempDir()) c.Offline = true if _, err := c.Get(context.Background(), "http://127.0.0.1:1/never"); err == nil { t.Fatal("expected offline error") }}
func TestCached(t *testing.T) { srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Write([]byte(`{"a":1}`)) })) defer srv.Close() c := New(t.TempDir()) c.MinInterval = 0 if c.CachedJSON(srv.URL + "/j") { t.Fatal("should not be cached before fetch") } var v map[string]int if err := c.GetJSON(context.Background(), srv.URL+"/j", &v); err != nil { t.Fatal(err) } if !c.CachedJSON(srv.URL + "/j") { t.Fatal("expected cached after GetJSON") } if c.Cached(srv.URL + "/j") { t.Fatal("plain Get uses a different cache key than GetJSON") }}
// TestHostInterval: a host with a stricter published limit is paced at its own interval even// when MinInterval is off.func TestHostInterval(t *testing.T) { var hits int32 srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { atomic.AddInt32(&hits, 1) w.Write([]byte("ok")) })) defer srv.Close() c := New(t.TempDir()) c.MinInterval = 0 host := strings.Split(strings.TrimPrefix(srv.URL, "http://"), ":")[0] c.HostInterval = map[string]time.Duration{host: 300 * time.Millisecond} start := time.Now() for i := 0; i < 3; i++ { if _, err := c.Get(context.Background(), fmt.Sprintf("%s/%d", srv.URL, i), NoCache()); err != nil { t.Fatal(err) } } if d := time.Since(start); d < 600*time.Millisecond { t.Fatalf("3 requests took %s, want >= 600ms at a 300ms host interval", d) } if New(t.TempDir()).HostInterval["crates.io"] != time.Second { t.Fatal("crates.io must default to one request per second") }}
// TestRequireContact: hosts whose policies require contact information are refused without one,// and the contact appears in the User-Agent once set.func TestRequireContact(t *testing.T) { var ua string srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { ua = r.Header.Get("User-Agent") w.Write([]byte("ok")) })) defer srv.Close() host := strings.Split(strings.TrimPrefix(srv.URL, "http://"), ":")[0] c := New(t.TempDir()) c.MinInterval, c.Retries = 0, 0 c.HostInterval = nil c.Contact = "" c.RequireContact = map[string]bool{host: true} if _, err := c.Get(context.Background(), srv.URL+"/a", NoCache()); err == nil || !strings.Contains(err.Error(), "ISOMORPH_CONTACT") { t.Fatalf("request without contact: %v", err) } t.Setenv("ISOMORPH_CONTACT", "ops@example.org") c2 := New(t.TempDir()) c2.MinInterval, c2.Retries = 0, 0 c2.RequireContact = map[string]bool{host: true} if _, err := c2.Get(context.Background(), srv.URL+"/b", NoCache()); err != nil { t.Fatal(err) } if !strings.Contains(ua, "ops@example.org") || !strings.HasPrefix(ua, "isomorph/") { t.Fatalf("User-Agent %q", ua) }}