Skip to content

fix: use separate HTTP clients in scale test load generators #21288

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
spikecurtis wants to merge 1 commit into
base: main
Choose a base branch
from
Open

fix: use separate HTTP clients in scale test load generators #21288

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
82 changes: 38 additions & 44 deletions cli/exp_scaletest.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,8 @@ import (

const scaletestTracerName = "coder_scaletest"

var BypassHeader = map[string][]string{codersdk.BypassRatelimitHeader: {"true"}}

func (r *RootCmd) scaletestCmd() *serpent.Command {
cmd := &serpent.Command{
Use: "scaletest",
Expand Down Expand Up @@ -690,15 +692,6 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

if count <= 0 {
return xerrors.Errorf("--count is required and must be greater than 0")
}
Expand Down Expand Up @@ -810,7 +803,13 @@ func (r *RootCmd) scaletestCreateWorkspaces() *serpent.Command {
return xerrors.Errorf("validate config: %w", err)
}

var runner harness.Runnable = createworkspaces.NewRunner(client, config)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = createworkspaces.NewRunner(runnerClient, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down Expand Up @@ -1011,15 +1010,6 @@ func (r *RootCmd) scaletestWorkspaceUpdates() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

if workspaceCount <= 0 {
return xerrors.Errorf("--workspace-count must be greater than 0")
}
Expand Down Expand Up @@ -1158,7 +1148,14 @@ func (r *RootCmd) scaletestWorkspaceUpdates() *serpent.Command {
for i, config := range configs {
name := fmt.Sprintf("workspaceupdates-%dw", config.WorkspaceCount)
id := strconv.Itoa(i)
var runner harness.Runnable = workspaceupdates.NewRunner(client, config)

// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = workspaceupdates.NewRunner(runnerClient, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down Expand Up @@ -1315,16 +1312,6 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *serpent.Command {
prometheusSrvClose := ServeHandler(ctx, logger, promhttp.HandlerFor(reg, promhttp.HandlerOpts{}), prometheusFlags.Address, "prometheus")
defer prometheusSrvClose()

// Bypass rate limiting
client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

workspaces, err := targetFlags.getTargetedWorkspaces(ctx, client, me.OrganizationIDs, inv.Stdout)
if err != nil {
return err
Expand Down Expand Up @@ -1421,7 +1408,13 @@ func (r *RootCmd) scaletestWorkspaceTraffic() *serpent.Command {
if err := config.Validate(); err != nil {
return xerrors.Errorf("validate config: %w", err)
}
var runner harness.Runnable = workspacetraffic.NewRunner(client, config)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = workspacetraffic.NewRunner(runnerClient, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down Expand Up @@ -1609,9 +1602,13 @@ func (r *RootCmd) scaletestDashboard() *serpent.Command {
return xerrors.Errorf("create token for user: %w", err)
}

userClient := codersdk.New(client.URL,
codersdk.WithSessionToken(userTokResp.Key),
)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
userClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
codersdk.WithSessionToken(userTokResp.Key)(userClient)

config := dashboard.Config{
Interval: interval,
Expand Down Expand Up @@ -1758,15 +1755,6 @@ func (r *RootCmd) scaletestAutostart() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

if workspaceCount <= 0 {
return xerrors.Errorf("--workspace-count must be greater than zero")
}
Expand Down Expand Up @@ -1832,7 +1820,13 @@ func (r *RootCmd) scaletestAutostart() *serpent.Command {
if err := config.Validate(); err != nil {
return xerrors.Errorf("validate config: %w", err)
}
var runner harness.Runnable = autostart.NewRunner(client, config)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = autostart.NewRunner(runnerClient, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down
21 changes: 9 additions & 12 deletions cli/exp_scaletest_dynamicparameters.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,18 +4,18 @@ package cli

import (
"fmt"
"net/http"
"time"

"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/scaletest/loadtestutil"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
"github.com/coder/serpent"

"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/dynamicparameters"
"github.com/coder/coder/v2/scaletest/harness"
)
Expand Down Expand Up @@ -72,15 +72,6 @@ func (r *RootCmd) scaletestDynamicParameters() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

reg := prometheus.NewRegistry()
metrics := dynamicparameters.NewMetrics(reg, "concurrent_evaluations")

Expand Down Expand Up @@ -122,7 +113,13 @@ func (r *RootCmd) scaletestDynamicParameters() *serpent.Command {
Metrics: metrics,
MetricLabelValues: []string{fmt.Sprintf("%d", part.ConcurrentEvaluations)},
}
var runner harness.Runnable = dynamicparameters.NewRunner(client, cfg)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = dynamicparameters.NewRunner(runnerClient, cfg)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down
19 changes: 9 additions & 10 deletions cli/exp_scaletest_notifications.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/scaletest/loadtestutil"

"cdr.dev/slog"

notificationsLib "github.com/coder/coder/v2/coderd/notifications"
Expand Down Expand Up @@ -66,15 +68,6 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

if userCount <= 0 {
return xerrors.Errorf("--user-count must be greater than 0")
}
Expand Down Expand Up @@ -206,7 +199,13 @@ func (r *RootCmd) scaletestNotifications() *serpent.Command {
for i, config := range configs {
id := strconv.Itoa(i)
name := fmt.Sprintf("notifications-%s", id)
var runner harness.Runnable = notifications.NewRunner(client, config)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = notifications.NewRunner(runnerClient, config)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down
20 changes: 9 additions & 11 deletions cli/exp_scaletest_prebuilds.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ package cli

import (
"fmt"
"net/http"
"os/signal"
"strconv"
"sync"
Expand All @@ -14,6 +13,8 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/scaletest/loadtestutil"

"github.com/coder/coder/v2/codersdk"
"github.com/coder/coder/v2/scaletest/harness"
"github.com/coder/coder/v2/scaletest/prebuilds"
Expand Down Expand Up @@ -56,15 +57,6 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
return err
}

client.HTTPClient = &http.Client{
Transport: &codersdk.HeaderTransport{
Transport: http.DefaultTransport,
Header: map[string][]string{
codersdk.BypassRatelimitHeader: {"true"},
},
},
}

if numTemplates <= 0 {
return xerrors.Errorf("--num-templates must be greater than 0")
}
Expand Down Expand Up @@ -140,7 +132,13 @@ func (r *RootCmd) scaletestPrebuilds() *serpent.Command {
return xerrors.Errorf("validate config: %w", err)
}

var runner harness.Runnable = prebuilds.NewRunner(client, cfg)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = prebuilds.NewRunner(runnerClient, cfg)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down
10 changes: 9 additions & 1 deletion cli/exp_scaletest_taskstatus.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,8 @@ import (
"github.com/prometheus/client_golang/prometheus/promhttp"
"golang.org/x/xerrors"

"github.com/coder/coder/v2/scaletest/loadtestutil"

"cdr.dev/slog"
"cdr.dev/slog/sloggers/sloghuman"
"github.com/coder/serpent"
Expand Down Expand Up @@ -143,7 +145,13 @@ After all runners connect, it waits for the baseline duration before triggering
return xerrors.Errorf("validate config for runner %d: %w", i, err)
}

var runner harness.Runnable = taskstatus.NewRunner(client, cfg)
// use an independent client for each Runner, so they don't reuse TCP connections. This can lead to
// requests being unbalanced among Coder instances.
runnerClient, err := loadtestutil.DupClientCopyingHeaders(client, BypassHeader)
if err != nil {
return xerrors.Errorf("create runner client: %w", err)
}
var runner harness.Runnable = taskstatus.NewRunner(runnerClient, cfg)
if tracingEnabled {
runner = &runnableTraceWrapper{
tracer: tracer,
Expand Down
45 changes: 45 additions & 0 deletions scaletest/loadtestutil/client.go
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
package loadtestutil

import (
"maps"
"net/http"

"golang.org/x/xerrors"

"github.com/coder/coder/v2/codersdk"
)

// DupClientCopyingHeaders duplicates the Client, but with an independent underlying HTTP transport, so that it will not
// share connections with the client being duplicated. It copies any headers already on the existing transport as
// [codersdk.HeaderTransport] and add the headers in the argument.
func DupClientCopyingHeaders(client *codersdk.Client, header http.Header) (*codersdk.Client, error) {
nc := codersdk.New(client.URL, codersdk.WithLogger(client.Logger()))
nc.SessionTokenProvider = client.SessionTokenProvider
newHeader, t, err := extractHeaderAndInnerTransport(client.HTTPClient.Transport)
if err != nil {
return nil, xerrors.Errorf("extract headers: %w", err)
}
maps.Copy(newHeader, header)

nc.HTTPClient.Transport = &codersdk.HeaderTransport{
Transport: t.Clone(),
Header: newHeader,
}
return nc, nil
}

func extractHeaderAndInnerTransport(rt http.RoundTripper) (http.Header, *http.Transport, error) {
if t, ok := rt.(*http.Transport); ok {
// base case
return make(http.Header), t, nil
}
if ht, ok := rt.(*codersdk.HeaderTransport); ok {
headers, t, err := extractHeaderAndInnerTransport(ht.Transport)
if err != nil {
return nil, nil, err
}
maps.Copy(headers, ht.Header)
return headers, t, nil
}
return nil, nil, xerrors.New("round tripper is neither HeaderTransport nor Transport")
}
Loading
Loading