feat: fetch official skills index

cmd/update/update_test.go

@@ -49,12 +49,21 @@ func mockDetectAndNpm(t *testing.T, result selfupdate.DetectResult, npmFn func(s
 		u.DetectOverride = func() selfupdate.DetectResult { return result }
 		u.NpmInstallOverride = npmFn
 		u.VerifyOverride = func(string) error { return nil }
+		u.SkillsIndexFetchOverride = successfulSkillsIndexFetch()
 		u.SkillsCommandOverride = successfulSkillsCommand()
 		return u
 	}
 	t.Cleanup(func() { newUpdater = origNew })
 }
 
+func successfulSkillsIndexFetch() func() *selfupdate.NpmResult {
+	return func() *selfupdate.NpmResult {
+		r := &selfupdate.NpmResult{}
+		r.Stdout.WriteString(`{"skills":[{"name":"lark-calendar"},{"name":"lark-mail"}]}`)
+		return r
+	}
+}
+
 func successfulSkillsCommand() func(args ...string) *selfupdate.NpmResult {
 	return func(args ...string) *selfupdate.NpmResult {
 		r := &selfupdate.NpmResult{}
@@ -478,6 +487,10 @@ func TestUpdateNpmVerifyFail_JSON_NoRestoreHintWhenBackupUnavailable(t *testing.
 		u.NpmInstallOverride = func(version string) *selfupdate.NpmResult { return &selfupdate.NpmResult{} }
 		u.VerifyOverride = func(string) error { return errors.New("bad binary") }
 		u.RestoreAvailableOverride = func() bool { return false }
+		u.SkillsIndexFetchOverride = func() *selfupdate.NpmResult {
+			t.Fatal("skills sync should not run when binary verification fails")
+			return nil
+		}
 		u.SkillsCommandOverride = func(args ...string) *selfupdate.NpmResult {
 			t.Fatal("skills sync should not run when binary verification fails")
 			return nil
@@ -810,6 +823,11 @@ func TestUpdateNpm_SkillsFail_JSON(t *testing.T) {
 		}
 		u.NpmInstallOverride = func(version string) *selfupdate.NpmResult { return &selfupdate.NpmResult{} }
 		u.VerifyOverride = func(string) error { return nil }
+		u.SkillsIndexFetchOverride = func() *selfupdate.NpmResult {
+			r := &selfupdate.NpmResult{}
+			r.Err = fmt.Errorf("index unavailable")
+			return r
+		}
 		u.SkillsCommandOverride = func(args ...string) *selfupdate.NpmResult {
 			r := &selfupdate.NpmResult{}
 			r.Stderr.WriteString("npx: command not found")
@@ -862,6 +880,11 @@ func TestUpdateNpm_SkillsFail_Human(t *testing.T) {
 		}
 		u.NpmInstallOverride = func(version string) *selfupdate.NpmResult { return &selfupdate.NpmResult{} }
 		u.VerifyOverride = func(string) error { return nil }
+		u.SkillsIndexFetchOverride = func() *selfupdate.NpmResult {
+			r := &selfupdate.NpmResult{}
+			r.Err = fmt.Errorf("index unavailable")
+			return r
+		}
 		u.SkillsCommandOverride = func(args ...string) *selfupdate.NpmResult {
 			r := &selfupdate.NpmResult{}
 			r.Stderr.WriteString("npx: command not found")
@@ -1006,6 +1029,7 @@ func TestUpdateRun_AlreadyLatest_RunsSkillsSync(t *testing.T) {
 	t.Cleanup(func() { newUpdater = origNew })
 	newUpdater = func() *selfupdate.Updater {
 		return &selfupdate.Updater{
+			SkillsIndexFetchOverride: successfulSkillsIndexFetch(),
 			SkillsCommandOverride: func(args ...string) *selfupdate.NpmResult {
 				skillsCalled = true
 				return successfulSkillsCommand()(args...)
@@ -1044,6 +1068,7 @@ func TestUpdateRun_Manual_RunsSkillsSync(t *testing.T) {
 	t.Cleanup(func() { newUpdater = origNew })
 	newUpdater = func() *selfupdate.Updater {
 		return &selfupdate.Updater{
+			SkillsIndexFetchOverride: successfulSkillsIndexFetch(),
 			DetectOverride: func() selfupdate.DetectResult {
 				return selfupdate.DetectResult{
 					Method:       selfupdate.InstallManual,
@@ -1088,6 +1113,7 @@ func TestUpdateRun_Npm_RunsSkillsSync_WritesLatestState(t *testing.T) {
 	t.Cleanup(func() { newUpdater = origNew })
 	newUpdater = func() *selfupdate.Updater {
 		return &selfupdate.Updater{
+			SkillsIndexFetchOverride: successfulSkillsIndexFetch(),
 			DetectOverride: func() selfupdate.DetectResult {
 				return selfupdate.DetectResult{
 					Method: selfupdate.InstallNpm, NpmAvailable: true,
@@ -1147,6 +1173,10 @@ func TestUpdateRun_CheckIncludesSkillsStatus(t *testing.T) {
 			DetectOverride: func() selfupdate.DetectResult {
 				return selfupdate.DetectResult{Method: selfupdate.InstallNpm, NpmAvailable: true}
 			},
+			SkillsIndexFetchOverride: func() *selfupdate.NpmResult {
+				skillsCalled = true
+				return successfulSkillsIndexFetch()()
+			},
 			SkillsCommandOverride: func(args ...string) *selfupdate.NpmResult {
 				skillsCalled = true
 				return successfulSkillsCommand()(args...)
@@ -1196,6 +1226,10 @@ func TestUpdateRun_CheckAlreadyLatest_NoSideEffect(t *testing.T) {
 	t.Cleanup(func() { newUpdater = origNew })
 	newUpdater = func() *selfupdate.Updater {
 		return &selfupdate.Updater{
+			SkillsIndexFetchOverride: func() *selfupdate.NpmResult {
+				skillsCalled = true
+				return successfulSkillsIndexFetch()()
+			},
 			SkillsCommandOverride: func(args ...string) *selfupdate.NpmResult {
 				skillsCalled = true
 				return successfulSkillsCommand()(args...)

internal/selfupdate/updater.go

@@ -10,10 +10,13 @@
 	"bytes"
 	"context"
 	"fmt"
+	"io"
+	"net/http"
 	"os/exec"
 	"strings"
 	"time"
 
+	"github.com/larksuite/cli/internal/transport"
 	"github.com/larksuite/cli/internal/vfs"
 )
 
@@ -37,9 +40,15 @@
 )
 
 const (
-	npmInstallTimeout   = 10 * time.Minute
-	skillsUpdateTimeout = 2 * time.Minute
-	verifyTimeout       = 10 * time.Second
+	npmInstallTimeout      = 10 * time.Minute
+	skillsUpdateTimeout    = 2 * time.Minute
+	skillsIndexMaxBodySize = 1 << 20
+	verifyTimeout          = 10 * time.Second
+)
+
+var (
+	skillsIndexFetchTimeout = 10 * time.Second
+	officialSkillsIndexURL  = "https://open.feishu.cn/.well-known/skills/index.json"
 )
 
 // DetectResult holds installation detection results.
@@ -83,6 +92,7 @@
 type Updater struct {
 	DetectOverride           func() DetectResult
 	NpmInstallOverride       func(version string) *NpmResult
+	SkillsIndexFetchOverride func() *NpmResult
 	SkillsCommandOverride    func(args ...string) *NpmResult
 	VerifyOverride           func(expectedVersion string) error
 	RestoreAvailableOverride func() bool
@@ -153,6 +163,53 @@
 	return r
 }
 
+func (u *Updater) ListOfficialSkillsIndex() *NpmResult {
+	if u.SkillsIndexFetchOverride != nil {
+		return u.SkillsIndexFetchOverride()
+	}
+
+	r := &NpmResult{}
+	ctx, cancel := context.WithTimeout(context.Background(), skillsIndexFetchTimeout)
+	defer cancel()
+
+	req, err := http.NewRequestWithContext(ctx, http.MethodGet, officialSkillsIndexURL, nil)
+	if err != nil {
+		r.Err = err
+		return r
+	}
+
+	client := transport.NewHTTPClient(0)
+	client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		if req.URL.Scheme != "https" {
+			return fmt.Errorf("official skills index redirected to non-HTTPS URL: %s", req.URL.Redacted())
+		}
+		return nil
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		r.Err = err
+		return r
+	}
+	defer resp.Body.Close()
+
+	if resp.StatusCode < http.StatusOK || resp.StatusCode >= http.StatusMultipleChoices {
+		r.Err = fmt.Errorf("official skills index returned HTTP %d", resp.StatusCode)
+		return r
+	}
+
+	limited := io.LimitReader(resp.Body, skillsIndexMaxBodySize+1)
+	if _, err := io.Copy(&r.Stdout, limited); err != nil {
+		r.Err = err
+		return r
+	}
+	if r.Stdout.Len() > skillsIndexMaxBodySize {
+		r.Stdout.Reset()
+		r.Err = fmt.Errorf("official skills index exceeds %d bytes", skillsIndexMaxBodySize)
+		return r
+	}
+	return r
+}
+
 func (u *Updater) ListOfficialSkills() *NpmResult {
 	r := u.runSkillsListOfficial("https://open.feishu.cn")
 	if r.Err != nil {

internal/selfupdate/updater_test.go

@@ -4,12 +4,18 @@
 package selfupdate
 
 import (
+	"context"
+	"errors"
 	"fmt"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"runtime"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/larksuite/cli/internal/vfs"
 )
@@ -232,6 +238,113 @@ func TestSkillsCommandsUseExpectedArgs(t *testing.T) {
 	}
 }
 
+func TestListOfficialSkillsIndexSuccess(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, `{"skills":[{"name":"lark-calendar"}]}`)
+	}))
+	defer server.Close()
+
+	oldURL := officialSkillsIndexURL
+	officialSkillsIndexURL = server.URL
+	t.Cleanup(func() { officialSkillsIndexURL = oldURL })
+
+	result := New().ListOfficialSkillsIndex()
+	if result.Err != nil {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want nil", result.Err)
+	}
+	if got := result.Stdout.String(); !strings.Contains(got, "lark-calendar") {
+		t.Fatalf("ListOfficialSkillsIndex() stdout = %q, want skill JSON", got)
+	}
+}
+
+func TestListOfficialSkillsIndexHTTPError(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Error(w, "not found", http.StatusNotFound)
+	}))
+	defer server.Close()
+
+	oldURL := officialSkillsIndexURL
+	officialSkillsIndexURL = server.URL
+	t.Cleanup(func() { officialSkillsIndexURL = oldURL })
+
+	result := New().ListOfficialSkillsIndex()
+	if result.Err == nil || !strings.Contains(result.Err.Error(), "HTTP 404") {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want HTTP 404", result.Err)
+	}
+}
+
+func TestListOfficialSkillsIndexBodyTooLarge(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		fmt.Fprint(w, strings.Repeat("x", skillsIndexMaxBodySize+1))
+	}))
+	defer server.Close()
+
+	oldURL := officialSkillsIndexURL
+	officialSkillsIndexURL = server.URL
+	t.Cleanup(func() { officialSkillsIndexURL = oldURL })
+
+	result := New().ListOfficialSkillsIndex()
+	if result.Err == nil || !strings.Contains(result.Err.Error(), "exceeds") {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want exceeds", result.Err)
+	}
+	if result.Stdout.Len() != 0 {
+		t.Fatalf("ListOfficialSkillsIndex() stdout len = %d, want 0", result.Stdout.Len())
+	}
+}
+
+func TestListOfficialSkillsIndexTimeout(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		time.Sleep(200 * time.Millisecond)
+		fmt.Fprint(w, `{"skills":[{"name":"lark-calendar"}]}`)
+	}))
+	defer server.Close()
+
+	oldURL := officialSkillsIndexURL
+	oldTimeout := skillsIndexFetchTimeout
+	officialSkillsIndexURL = server.URL
+	skillsIndexFetchTimeout = 50 * time.Millisecond
+	t.Cleanup(func() {
+		officialSkillsIndexURL = oldURL
+		skillsIndexFetchTimeout = oldTimeout
+	})
+
+	result := New().ListOfficialSkillsIndex()
+	var netErr net.Error
+	if result.Err == nil || (!errors.Is(result.Err, context.DeadlineExceeded) && !(errors.As(result.Err, &netErr) && netErr.Timeout())) {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want timeout error", result.Err)
+	}
+}
+
+func TestListOfficialSkillsIndexRejectsNonHTTPSRedirect(t *testing.T) {
+	server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		http.Redirect(w, r, "http://example.com/skills.json", http.StatusFound)
+	}))
+	defer server.Close()
+
+	oldURL := officialSkillsIndexURL
+	officialSkillsIndexURL = server.URL
+	t.Cleanup(func() { officialSkillsIndexURL = oldURL })
+
+	result := New().ListOfficialSkillsIndex()
+	if result.Err == nil || !strings.Contains(result.Err.Error(), "non-HTTPS") {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want non-HTTPS redirect", result.Err)
+	}
+}
+
+func TestListOfficialSkillsIndexUsesOverride(t *testing.T) {
+	result := (&Updater{SkillsIndexFetchOverride: func() *NpmResult {
+		r := &NpmResult{}
+		r.Stdout.WriteString(`{"skills":[{"name":"override-skill"}]}`)
+		return r
+	}}).ListOfficialSkillsIndex()
+	if result.Err != nil {
+		t.Fatalf("ListOfficialSkillsIndex() err = %v, want nil", result.Err)
+	}
+	if !strings.Contains(result.Stdout.String(), "override-skill") {
+		t.Fatalf("ListOfficialSkillsIndex() stdout = %q, want override result", result.Stdout.String())
+	}
+}
+
 func TestListOfficialSkillsFallsBack(t *testing.T) {
 	called := []string{}
 	updater := &Updater{