Skip to content

codeintel: allow configuring max concurrency in uploads #1023

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
Strum355 merged 2 commits into from
Aug 28, 2023
+277 −355
Merged

codeintel: allow configuring max concurrency in uploads #1023

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
6b72a07
codeintel: allow configuring max concurrency in uploads
Strum355 Aug 22, 2023
55a541a
pin to main
Strum355 Aug 28, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions cmd/src/code_intel_upload.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -172,6 +172,7 @@ func codeintelUploadOptions(out *output.Output, isSCIPAvailable bool) upload.Upl
RetryInterval: time.Second,
Path: path,
MaxPayloadSizeBytes: codeintelUploadFlags.maxPayloadSizeMb * 1000 * 1000,
MaxConcurrency: codeintelUploadFlags.maxConcurrency,
GitHubToken: codeintelUploadFlags.gitHubToken,
GitLabToken: codeintelUploadFlags.gitLabToken,
},
Expand Down
2 changes: 2 additions & 0 deletions cmd/src/code_intel_upload_flags.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,7 @@ var codeintelUploadFlags struct {
// SourcegraphInstanceOptions
uploadRoute string
maxPayloadSizeMb int64
maxConcurrency int

// Codehost authorization secrets
gitHubToken string
Expand Down Expand Up @@ -77,6 +78,7 @@ func init() {
// SourcegraphInstanceOptions
codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.uploadRoute, "upload-route", "/.api/lsif/upload", "The path of the upload route. For internal use only.")
codeintelUploadFlagSet.Int64Var(&codeintelUploadFlags.maxPayloadSizeMb, "max-payload-size", 100, `The maximum upload size (in megabytes). Indexes exceeding this limit will be uploaded over multiple HTTP requests.`)
codeintelUploadFlagSet.IntVar(&codeintelUploadFlags.maxConcurrency, "max-concurrency", -1, "The maximum number of concurrent uploads. Only relevant for multipart uploads. Defaults to all parts concurrently.")

// Codehost authorization secrets
codeintelUploadFlagSet.StringVar(&codeintelUploadFlags.gitHubToken, "github-token", "", `A GitHub access token with 'public_repo' scope that Sourcegraph uses to verify you have access to the repository.`)
Expand Down
106 changes: 61 additions & 45 deletions go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@ module github.com/sourcegraph/src-cli
go 1.18

require (
cloud.google.com/go/storage v1.28.0
cloud.google.com/go/storage v1.30.1
github.com/atotto/clipboard v0.1.4
github.com/aws/aws-sdk-go-v2/config v1.18.13
github.com/aws/aws-sdk-go-v2/service/ec2 v1.86.0
Expand All @@ -12,30 +12,30 @@ require (
github.com/charmbracelet/bubbles v0.15.0
github.com/charmbracelet/lipgloss v0.7.1
github.com/creack/goselect v0.1.2
github.com/derision-test/glock v0.0.0-20210316032053-f5b74334bb29
github.com/derision-test/glock v1.0.0
github.com/dineshappavoo/basex v0.0.0-20170425072625-481a6f6dc663
github.com/dustin/go-humanize v1.0.1
github.com/gobwas/glob v0.2.3
github.com/google/go-cmp v0.5.9
github.com/grafana/regexp v0.0.0-20221123153739-15dc172cd2db
github.com/hexops/autogold v1.3.0
github.com/hexops/autogold v1.3.1
github.com/jedib0t/go-pretty/v6 v6.3.7
github.com/jig/teereadcloser v0.0.0-20181016160506-953720c48e05
github.com/json-iterator/go v1.1.12
github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51
github.com/mattn/go-isatty v0.0.17
github.com/mattn/go-isatty v0.0.19
github.com/neelance/parallel v0.0.0-20160708114440-4de9ce63d14c
github.com/pkg/browser v0.0.0-20210911075715-681adbf594b8
github.com/sourcegraph/conc v0.3.0
github.com/sourcegraph/go-diff v0.6.2-0.20221123165719-f8cd299c40f3
github.com/sourcegraph/jsonx v0.0.0-20200629203448-1a936bd500cf
github.com/sourcegraph/scip v0.2.4-0.20221213205653-aa0e511dcfef
github.com/sourcegraph/sourcegraph/lib v0.0.0-20230316093010-26299ec302d0
github.com/stretchr/testify v1.8.1
golang.org/x/net v0.8.0
golang.org/x/sync v0.1.0
google.golang.org/api v0.110.0
google.golang.org/protobuf v1.28.1
github.com/sourcegraph/scip v0.3.1-0.20230627154934-45df7f6d33fc
github.com/sourcegraph/sourcegraph/lib v0.0.0-20230822113036-acc34d9b2e95
github.com/stretchr/testify v1.8.4
golang.org/x/net v0.12.0
golang.org/x/sync v0.3.0
google.golang.org/api v0.132.0
google.golang.org/protobuf v1.31.0
gopkg.in/yaml.v3 v3.0.1
jaytaylor.com/html2text v0.0.0-20200412013138-3577fbdbcff7
k8s.io/api v0.27.1
Expand All @@ -45,10 +45,26 @@ require (
)

require (
cloud.google.com/go v0.107.0 // indirect
cloud.google.com/go/compute v1.18.0 // indirect
cloud.google.com/go/compute/metadata v0.2.3 // indirect
cloud.google.com/go/iam v0.12.0 // indirect
github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect
github.com/google/s2a-go v0.1.4 // indirect
github.com/jackc/chunkreader/v2 v2.0.1 // indirect
github.com/jackc/pgconn v1.12.1 // indirect
github.com/jackc/pgio v1.0.0 // indirect
github.com/jackc/pgpassfile v1.0.0 // indirect
github.com/jackc/pgproto3/v2 v2.3.0 // indirect
github.com/jackc/pgservicefile v0.0.0-20200714003250-2b9c44734f2b // indirect
github.com/onsi/ginkgo/v2 v2.9.7 // indirect
github.com/onsi/gomega v1.27.8 // indirect
go.uber.org/goleak v1.2.1 // indirect
google.golang.org/genproto/googleapis/api v0.0.0-20230717213848-3f92550aa753 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20230717213848-3f92550aa753 // indirect
)

require (
cloud.google.com/go v0.110.4 // indirect
cloud.google.com/go/compute v1.22.0 // indirect
cloud.google.com/go/compute/metadata v0.2.4-0.20230617002413-005d2dfb6b68 // indirect
cloud.google.com/go/iam v1.1.1 // indirect
github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 // indirect
github.com/Masterminds/goutils v1.1.1 // indirect
github.com/Masterminds/semver v1.5.0 // indirect
Expand All @@ -70,19 +86,19 @@ require (
github.com/bufbuild/buf v1.4.0 // indirect
github.com/charmbracelet/bubbletea v0.23.1 // direct
github.com/charmbracelet/glamour v0.5.0 // indirect
github.com/cockroachdb/errors v1.9.0 // indirect
github.com/cockroachdb/logtags v0.0.0-20211118104740-dabe8e521a4f // indirect
github.com/cockroachdb/redact v1.1.3 // indirect
github.com/cockroachdb/errors v1.10.0 // indirect
github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect
github.com/cockroachdb/redact v1.1.5 // indirect
github.com/containerd/console v1.0.3 // indirect
github.com/cpuguy83/go-md2man/v2 v2.0.2 // indirect
github.com/davecgh/go-spew v1.1.1 // indirect
github.com/dlclark/regexp2 v1.7.0 // indirect
github.com/dlclark/regexp2 v1.8.0 // indirect
github.com/emicklei/go-restful/v3 v3.9.0 // indirect
github.com/envoyproxy/protoc-gen-validate v0.9.1 // indirect
github.com/fatih/color v1.13.0 // indirect
github.com/getsentry/sentry-go v0.15.0 // indirect
github.com/envoyproxy/protoc-gen-validate v0.10.1 // indirect
github.com/fatih/color v1.15.0 // indirect
github.com/getsentry/sentry-go v0.22.0 // indirect
github.com/ghodss/yaml v1.0.0 // indirect
github.com/go-logr/logr v1.2.3 // indirect
github.com/go-logr/logr v1.2.4 // indirect
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.1 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
Expand All @@ -92,22 +108,22 @@ require (
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/protobuf v1.5.3 // indirect
github.com/google/gnostic v0.5.7-v3refs // indirect
github.com/google/gofuzz v1.1.0 // indirect
github.com/google/gofuzz v1.2.0 // indirect
github.com/google/uuid v1.3.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect
github.com/googleapis/gax-go/v2 v2.7.0 // indirect
github.com/googleapis/enterprise-certificate-proxy v0.2.5 // indirect
github.com/googleapis/gax-go/v2 v2.12.0 // indirect
github.com/gorilla/css v1.0.0 // indirect
github.com/hexops/gotextdiff v1.0.3 // indirect
github.com/hexops/valast v1.4.3 // indirect
github.com/huandu/xstrings v1.3.2 // indirect
github.com/imdario/mergo v0.3.13 // indirect
github.com/inconshreveable/mousetrap v1.0.1 // indirect
github.com/imdario/mergo v0.3.16 // indirect
github.com/inconshreveable/mousetrap v1.1.0 // indirect
github.com/jdxcode/netrc v0.0.0-20210204082910-926c7f70242a // indirect
github.com/jhump/protocompile v0.0.0-20220216033700-d705409f108f // indirect
github.com/jhump/protoreflect v1.12.1-0.20220417024638-438db461d753 // indirect
github.com/jmespath/go-jmespath v0.4.0 // indirect
github.com/josharian/intern v1.0.0 // indirect
github.com/klauspost/compress v1.15.11 // indirect
github.com/klauspost/compress v1.16.7 // indirect
github.com/klauspost/pgzip v1.2.5 // indirect
github.com/kr/pretty v0.3.1 // indirect
github.com/kr/text v0.2.0 // indirect
Expand All @@ -116,7 +132,7 @@ require (
github.com/mattn/go-colorable v0.1.13 // indirect
github.com/mattn/go-localereader v0.0.1 // indirect
github.com/mattn/go-runewidth v0.0.14 // indirect
github.com/microcosm-cc/bluemonday v1.0.22 // indirect
github.com/microcosm-cc/bluemonday v1.0.23 // indirect
github.com/mitchellh/copystructure v1.2.0 // indirect
github.com/mitchellh/reflectwalk v1.0.2 // indirect
github.com/moby/spdystream v0.2.0 // indirect
Expand All @@ -136,11 +152,11 @@ require (
github.com/pmezard/go-difflib v1.0.0 // indirect
github.com/pseudomuto/protoc-gen-doc v1.5.1 // indirect
github.com/pseudomuto/protokit v0.2.0 // indirect
github.com/rivo/uniseg v0.4.2 // indirect
github.com/rivo/uniseg v0.4.3 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/russross/blackfriday/v2 v2.1.0 // indirect
github.com/sourcegraph/log v0.0.0-20221206163500-7d93c6ad7037 // indirect
github.com/spf13/cobra v1.6.0 // indirect
github.com/sourcegraph/log v0.0.0-20230711093019-40c57b632cca // indirect
github.com/spf13/cobra v1.7.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/ssor/bom v0.0.0-20170718123548-6386211fdfcf // indirect
github.com/stretchr/objx v0.5.0 // indirect
Expand All @@ -150,21 +166,21 @@ require (
github.com/yuin/goldmark v1.5.2 // indirect
github.com/yuin/goldmark-emoji v1.0.1 // indirect
go.opencensus.io v0.24.0 // indirect
go.uber.org/atomic v1.10.0 // indirect
go.uber.org/multierr v1.9.0 // indirect
go.uber.org/atomic v1.11.0 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.24.0 // indirect
golang.org/x/crypto v0.3.0 // indirect
golang.org/x/mod v0.9.0 // indirect
golang.org/x/oauth2 v0.5.0 // indirect
golang.org/x/sys v0.6.0 // indirect
golang.org/x/term v0.6.0 // indirect
golang.org/x/text v0.8.0 // indirect
golang.org/x/time v0.0.0-20220922220347-f3bd1da661af // indirect
golang.org/x/tools v0.7.0 // indirect
golang.org/x/crypto v0.11.0 // indirect
golang.org/x/mod v0.11.0 // indirect
golang.org/x/oauth2 v0.10.0 // indirect
golang.org/x/sys v0.10.0 // indirect
golang.org/x/term v0.10.0 // indirect
golang.org/x/text v0.11.0 // indirect
golang.org/x/time v0.3.0 // indirect
golang.org/x/tools v0.10.0 // indirect
golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
google.golang.org/appengine v1.6.7 // indirect
google.golang.org/genproto v0.0.0-20230216225411-c8e22ba71e44 // indirect
google.golang.org/grpc v1.53.0 // indirect
google.golang.org/genproto v0.0.0-20230717213848-3f92550aa753 // indirect
google.golang.org/grpc v1.56.2 // indirect
gopkg.in/inf.v0 v0.9.1 // direct
gopkg.in/yaml.v2 v2.4.0 // indirect
k8s.io/klog/v2 v2.90.1 // indirect
Expand Down
Loading