From 3b10030da0c588a21dc47ceead53ebe72dcd14f3 Mon Sep 17 00:00:00 2001
From: =?UTF-8?q?Daniel=20Mart=C3=AD?= <mvdan@mvdan.cc>
Date: Thu, 3 Jul 2025 11:21:49 +0100
Subject: [PATCH] encoding/jsonschema: parallelize vendor_external.go a bit
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Fetching the zip file from upstream takes ~150ms,
and reading and updating the test data takes ~400ms.
Given that all `go generate ./...` steps happen sequentially,
cutting down the runtime cost even by 150ms is noticeable.

                   │     old     │                new                 │
                   │   sec/op    │   sec/op     vs base               │
    VendorExternal   900.5m ± 8%   751.2m ± 3%  -16.58% (p=0.000 n=8)

Signed-off-by: Daniel Martí <mvdan@mvdan.cc>
Change-Id: If6e23e1fcde00b4ace8e0a18e71826f952ed2b18
Reviewed-on: https://review.gerrithub.io/c/cue-lang/cue/+/1217835
Unity-Result: CUE porcuepine <cue.porcuepine@gmail.com>
Reviewed-by: Roger Peppe <rogpeppe@gmail.com>
TryBot-Result: CUEcueckoo <cueckoo@cuelang.org>
---
 encoding/jsonschema/vendor_external.go | 72 +++++++++++++++-----------
 1 file changed, 42 insertions(+), 30 deletions(-)

diff --git a/encoding/jsonschema/vendor_external.go b/encoding/jsonschema/vendor_external.go
index 4a3758b77..82c269e12 100644
--- a/encoding/jsonschema/vendor_external.go
+++ b/encoding/jsonschema/vendor_external.go
@@ -35,6 +35,8 @@ import (
 	"path/filepath"
 	"strings"
 
+	"golang.org/x/sync/errgroup"
+
 	"cuelang.org/go/encoding/jsonschema/internal/externaltest"
 )
 
@@ -59,24 +61,45 @@ func main() {
 }
 
 func doVendor(commit string) error {
-	// Fetch a commit from GitHub via their archive ZIP endpoint, which is a lot faster
-	// than git cloning just to retrieve a single commit's files.
-	// See: https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#download-a-repository-archive-zip
-	zipURL := fmt.Sprintf("https://github.com/json-schema-org/JSON-Schema-Test-Suite/archive/%s.zip", commit)
-	log.Printf("fetching %s", zipURL)
-	resp, err := http.Get(zipURL)
-	if err != nil {
-		return err
-	}
-	defer resp.Body.Close()
-	zipBytes, err := io.ReadAll(resp.Body)
-	if err != nil {
+	// Reading the old test data and fetching a zip file for the upstream data can be done in parallel.
+	// This is useful as each operation takes hundreds of milliseconds.
+	g := new(errgroup.Group)
+	var oldTests map[string][]*externaltest.Schema
+	g.Go(func() error {
+		log.Printf("reading old test data")
+		var err error
+		oldTests, err = externaltest.ReadTestDir(testDir)
+		if err != nil && !errors.Is(err, externaltest.ErrNotFound) {
+			return err
+		}
+		return nil
+	})
+	var fsys fs.FS
+	g.Go(func() error {
+		// Fetch a commit from GitHub via their archive ZIP endpoint, which is a lot faster
+		// than git cloning just to retrieve a single commit's files.
+		// See: https://docs.github.com/en/rest/repos/contents?apiVersion=2022-11-28#download-a-repository-archive-zip
+		zipURL := fmt.Sprintf("https://github.com/json-schema-org/JSON-Schema-Test-Suite/archive/%s.zip", commit)
+		log.Printf("fetching %s", zipURL)
+		resp, err := http.Get(zipURL)
+		if err != nil {
+			return err
+		}
+		defer resp.Body.Close()
+		zipBytes, err := io.ReadAll(resp.Body)
+		if err != nil {
+			return err
+		}
+		zipr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
+		if err != nil {
+			return err
+		}
+		// Note that GitHub produces archives with a top-level directory representing
+		// the name of the repository and the version which was retrieved.
+		fsys, err = fs.Sub(zipr, fmt.Sprintf("JSON-Schema-Test-Suite-%s/tests", commit))
 		return err
-	}
-
-	log.Printf("reading old test data")
-	oldTests, err := externaltest.ReadTestDir(testDir)
-	if err != nil && !errors.Is(err, externaltest.ErrNotFound) {
+	})
+	if err := g.Wait(); err != nil {
 		return err
 	}
 
@@ -85,17 +108,7 @@ func doVendor(commit string) error {
 	if err := os.RemoveAll(testSubdir); err != nil {
 		return err
 	}
-	zipr, err := zip.NewReader(bytes.NewReader(zipBytes), int64(len(zipBytes)))
-	if err != nil {
-		return err
-	}
-	// Note that GitHub produces archives with a top-level directory representing
-	// the name of the repository and the version which was retrieved.
-	fsys, err := fs.Sub(zipr, fmt.Sprintf("JSON-Schema-Test-Suite-%s/tests", commit))
-	if err != nil {
-		return err
-	}
-	err = fs.WalkDir(fsys, ".", func(filename string, d fs.DirEntry, err error) error {
+	if err := fs.WalkDir(fsys, ".", func(filename string, d fs.DirEntry, err error) error {
 		if err != nil {
 			return err
 		}
@@ -121,8 +134,7 @@ func doVendor(commit string) error {
 			return err
 		}
 		return nil
-	})
-	if err != nil {
+	}); err != nil {
 		return err
 	}
 
-- 
2.51.2