package main import ( "context" "fmt" "io" "net/http" "os" "path/filepath" "time" ) // downloadTimeout caps one download attempt. Geofabrik extracts run // to a few hundred megabytes; half an hour covers them on a slow // link. const downloadTimeout = 30 * time.Minute const attempts = 3 // fetchFile downloads url to dest unless dest already exists with // content. The body streams to a temp file, so a whole file is never // loaded into memory and a failed download never leaves a partial // file behind. Each attempt is bounded by downloadTimeout and a // failure retries up to attempts times. func fetchFile(ctx context.Context, url, dest string) error { return fetchFileAuth(ctx, url, dest, "") } // fetchFileAuth is fetchFile with an Authorization header value, for // sources that require a token. The transport drops the header when a // redirect crosses to another host, so a download that redirects to a // CDN does not send the token there. func fetchFileAuth(ctx context.Context, url, dest, authorization string) error { if info, err := os.Stat(dest); err == nil && info.Size() > 0 { return nil } if err := os.MkdirAll(filepath.Dir(dest), 0o755); err != nil { return err } var lastErr error for attempt := 1; attempt <= attempts; attempt++ { lastErr = download(ctx, url, dest, authorization) if lastErr == nil { return nil } if ctx.Err() != nil { return lastErr } time.Sleep(time.Duration(attempt) * 2 * time.Second) } return fmt.Errorf("downloading %s: %w", url, lastErr) } // httpGet performs a bounded GET and returns the body for streaming. // The caller must close it. A non-200 status is an error. An empty // authorization sends no Authorization header. func httpGet(ctx context.Context, url string, timeout time.Duration, authorization string) (io.ReadCloser, error) { ctx, cancel := context.WithTimeout(ctx, timeout) req, err := http.NewRequestWithContext(ctx, http.MethodGet, url, nil) if err != nil { cancel() return nil, err } if authorization != "" { req.Header.Set("Authorization", authorization) } resp, err := http.DefaultClient.Do(req) if err != nil { cancel() return nil, err } if resp.StatusCode != http.StatusOK { resp.Body.Close() cancel() return nil, fmt.Errorf("GET %s: %s", url, resp.Status) } return &cancelReadCloser{ReadCloser: resp.Body, cancel: cancel}, nil } type cancelReadCloser struct { io.ReadCloser cancel context.CancelFunc } func (c *cancelReadCloser) Close() error { err := c.ReadCloser.Close() c.cancel() return err } func download(ctx context.Context, url, dest, authorization string) error { body, err := httpGet(ctx, url, downloadTimeout, authorization) if err != nil { return err } defer body.Close() tmp, err := os.CreateTemp(filepath.Dir(dest), ".download-*") if err != nil { return err } defer os.Remove(tmp.Name()) if _, err := io.Copy(tmp, body); err != nil { tmp.Close() return err } if err := tmp.Close(); err != nil { return err } return os.Rename(tmp.Name(), dest) }