Skip to content

Commit 9280052

Browse files
easwarseaswars
authored
balancergroup: Propagate balancer.BuildOptions to child policies (#4184)
1 parent b753f49 commit 9280052

File tree

12 files changed

+175
-85
lines changed

12 files changed

+175
-85
lines changed

‎xds/internal/balancer/balancergroup/balancergroup.go

Lines changed: 9 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,8 @@ type subBalancerWrapper struct {
6060
// The static part of sub-balancer. Keeps balancerBuilders and addresses.
6161
// To be used when restarting sub-balancer.
6262
builder balancer.Builder
63+
// Options to be passed to sub-balancer at the time of creation.
64+
buildOpts balancer.BuildOptions
6365
// ccState is a cache of the addresses/balancer config, so when the balancer
6466
// is restarted after close, it will get the previous update. It's a pointer
6567
// and is set to nil at init, so when the balancer is built for the first
@@ -94,7 +96,7 @@ func (sbc *subBalancerWrapper) updateBalancerStateWithCachedPicker() {
9496
}
9597

9698
func (sbc *subBalancerWrapper) startBalancer() {
97-
b := sbc.builder.Build(sbc, balancer.BuildOptions{})
99+
b := sbc.builder.Build(sbc, sbc.buildOpts)
98100
sbc.group.logger.Infof("Created child policy %p of type %v", b, sbc.builder.Name())
99101
sbc.balancer = b
100102
if sbc.ccState != nil {
@@ -179,6 +181,7 @@ func (sbc *subBalancerWrapper) stopBalancer() {
179181
// balancer group.
180182
type BalancerGroup struct {
181183
cc balancer.ClientConn
184+
buildOpts balancer.BuildOptions
182185
logger *grpclog.PrefixLogger
183186
loadStore load.PerClusterReporter
184187

@@ -235,9 +238,12 @@ var DefaultSubBalancerCloseTimeout = 15 * time.Minute
235238

236239
// New creates a new BalancerGroup. Note that the BalancerGroup
237240
// needs to be started to work.
238-
func New(cc balancer.ClientConn, stateAggregator BalancerStateAggregator, loadStore load.PerClusterReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
241+
//
242+
// TODO(easwars): Pass an options struct instead of N args.
243+
func New(cc balancer.ClientConn, bOpts balancer.BuildOptions, stateAggregator BalancerStateAggregator, loadStore load.PerClusterReporter, logger *grpclog.PrefixLogger) *BalancerGroup {
239244
return &BalancerGroup{
240245
cc: cc,
246+
buildOpts: bOpts,
241247
logger: logger,
242248
loadStore: loadStore,
243249

@@ -305,6 +311,7 @@ func (bg *BalancerGroup) Add(id string, builder balancer.Builder) {
305311
id: id,
306312
group: bg,
307313
builder: builder,
314+
buildOpts: bg.buildOpts,
308315
}
309316
if bg.outgoingStarted {
310317
// Only start the balancer if bg is started. Otherwise, we only keep the

‎xds/internal/balancer/balancergroup/balancergroup_test.go

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,8 @@ import (
3838
"google.golang.org/grpc/balancer"
3939
"google.golang.org/grpc/balancer/roundrobin"
4040
"google.golang.org/grpc/connectivity"
41+
"google.golang.org/grpc/credentials/insecure"
42+
"google.golang.org/grpc/internal/balancer/stub"
4143
"google.golang.org/grpc/resolver"
4244
"google.golang.org/grpc/xds/internal/balancer/weightedtarget/weightedaggregator"
4345
"google.golang.org/grpc/xds/internal/client/load"
@@ -74,7 +76,7 @@ func newTestBalancerGroup(t *testing.T, loadStore load.PerClusterReporter) (*tes
7476
cc := testutils.NewTestClientConn(t)
7577
gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
7678
gator.Start()
77-
bg := New(cc, gator, loadStore, nil)
79+
bg := New(cc, balancer.BuildOptions{}, gator, loadStore, nil)
7880
bg.Start()
7981
return cc, gator, bg
8082
}
@@ -501,7 +503,7 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
501503
cc := testutils.NewTestClientConn(t)
502504
gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
503505
gator.Start()
504-
bg := New(cc, gator, nil, nil)
506+
bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
505507

506508
// Add two balancers to group and send two resolved addresses to both
507509
// balancers.
@@ -590,16 +592,20 @@ func (s) TestBalancerGroup_start_close(t *testing.T) {
590592
// whenever it gets an address update. It's expected that start() doesn't block
591593
// because of deadlock.
592594
func (s) TestBalancerGroup_start_close_deadlock(t *testing.T) {
595+
const balancerName = "stub-TestBalancerGroup_start_close_deadlock"
596+
stub.Register(balancerName, stub.BalancerFuncs{})
597+
builder := balancer.Get(balancerName)
598+
593599
cc := testutils.NewTestClientConn(t)
594600
gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
595601
gator.Start()
596-
bg := New(cc, gator, nil, nil)
602+
bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
597603

598604
gator.Add(testBalancerIDs[0], 2)
599-
bg.Add(testBalancerIDs[0], &testutils.TestConstBalancerBuilder{})
605+
bg.Add(testBalancerIDs[0], builder)
600606
bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[0:2]}})
601607
gator.Add(testBalancerIDs[1], 1)
602-
bg.Add(testBalancerIDs[1], &testutils.TestConstBalancerBuilder{})
608+
bg.Add(testBalancerIDs[1], builder)
603609
bg.UpdateClientConnState(testBalancerIDs[1], balancer.ClientConnState{ResolverState: resolver.State{Addresses: testBackendAddrs[2:4]}})
604610

605611
bg.Start()
@@ -695,7 +701,7 @@ func initBalancerGroupForCachingTest(t *testing.T) (*weightedaggregator.Aggregat
695701
cc := testutils.NewTestClientConn(t)
696702
gator := weightedaggregator.New(cc, nil, testutils.NewTestWRR)
697703
gator.Start()
698-
bg := New(cc, gator, nil, nil)
704+
bg := New(cc, balancer.BuildOptions{}, gator, nil, nil)
699705

700706
// Add two balancers to group and send two resolved addresses to both
701707
// balancers.
@@ -931,3 +937,43 @@ func (s) TestBalancerGroup_locality_caching_readd_with_different_builder(t *test
931937
t.Fatalf("want %v, got %v", want, err)
932938
}
933939
}
940+
941+
// TestBalancerGroupBuildOptions verifies that the balancer.BuildOptions passed
942+
// to the balancergroup at creation time is passed to child policies.
943+
func (s) TestBalancerGroupBuildOptions(t *testing.T) {
944+
const (
945+
balancerName = "stubBalancer-TestBalancerGroupBuildOptions"
946+
parent = int64(1234)
947+
userAgent = "ua"
948+
defaultTestTimeout = 1 * time.Second
949+
)
950+
951+
// Setup the stub balancer such that we can read the build options passed to
952+
// it in the UpdateClientConnState method.
953+
bOpts := balancer.BuildOptions{
954+
DialCreds: insecure.NewCredentials(),
955+
ChannelzParentID: parent,
956+
CustomUserAgent: userAgent,
957+
}
958+
stub.Register(balancerName, stub.BalancerFuncs{
959+
UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
960+
if !cmp.Equal(bd.BuildOptions, bOpts) {
961+
return fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
962+
}
963+
return nil
964+
},
965+
})
966+
cc := testutils.NewTestClientConn(t)
967+
bg := New(cc, bOpts, nil, nil, nil)
968+
bg.Start()
969+
970+
// Add the stub balancer build above as a child policy.
971+
balancerBuilder := balancer.Get(balancerName)
972+
bg.Add(testBalancerIDs[0], balancerBuilder)
973+
974+
// Send an empty clientConn state change. This should trigger the
975+
// verification of the buildOptions being passed to the child policy.
976+
if err := bg.UpdateClientConnState(testBalancerIDs[0], balancer.ClientConnState{}); err != nil {
977+
t.Fatal(err)
978+
}
979+
}

‎xds/internal/balancer/clustermanager/clustermanager.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,12 +40,12 @@ func init() {
4040

4141
type builder struct{}
4242

43-
func (builder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
43+
func (builder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
4444
b := &bal{}
4545
b.logger = prefixLogger(b)
4646
b.stateAggregator = newBalancerStateAggregator(cc, b.logger)
4747
b.stateAggregator.start()
48-
b.bg = balancergroup.New(cc, b.stateAggregator, nil, b.logger)
48+
b.bg = balancergroup.New(cc, opts, b.stateAggregator, nil, b.logger)
4949
b.bg.Start()
5050
b.logger.Infof("Created")
5151
return b

‎xds/internal/balancer/clustermanager/clustermanager_test.go

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -29,8 +29,11 @@ import (
2929
"google.golang.org/grpc/balancer/roundrobin"
3030
"google.golang.org/grpc/codes"
3131
"google.golang.org/grpc/connectivity"
32+
"google.golang.org/grpc/credentials/insecure"
33+
"google.golang.org/grpc/internal/balancer/stub"
3234
"google.golang.org/grpc/internal/grpctest"
3335
"google.golang.org/grpc/internal/hierarchy"
36+
itestutils "google.golang.org/grpc/internal/testutils"
3437
"google.golang.org/grpc/resolver"
3538
"google.golang.org/grpc/status"
3639
"google.golang.org/grpc/xds/internal/balancer/balancergroup"
@@ -510,3 +513,55 @@ func TestRoutingConfigUpdateDeleteAll(t *testing.T) {
510513
testPick(t, p3, tt.pickInfo, tt.wantSC, tt.wantErr)
511514
}
512515
}
516+
517+
func TestClusterManagerForwardsBalancerBuildOptions(t *testing.T) {
518+
const (
519+
balancerName = "stubBalancer-TestClusterManagerForwardsBalancerBuildOptions"
520+
parent = int64(1234)
521+
userAgent = "ua"
522+
defaultTestTimeout = 1 * time.Second
523+
)
524+
525+
// Setup the stub balancer such that we can read the build options passed to
526+
// it in the UpdateClientConnState method.
527+
ccsCh := itestutils.NewChannel()
528+
bOpts := balancer.BuildOptions{
529+
DialCreds: insecure.NewCredentials(),
530+
ChannelzParentID: parent,
531+
CustomUserAgent: userAgent,
532+
}
533+
stub.Register(balancerName, stub.BalancerFuncs{
534+
UpdateClientConnState: func(bd *stub.BalancerData, _ balancer.ClientConnState) error {
535+
if !cmp.Equal(bd.BuildOptions, bOpts) {
536+
err := fmt.Errorf("buildOptions in child balancer: %v, want %v", bd, bOpts)
537+
ccsCh.Send(err)
538+
return err
539+
}
540+
ccsCh.Send(nil)
541+
return nil
542+
},
543+
})
544+
545+
cc := testutils.NewTestClientConn(t)
546+
rtb := rtBuilder.Build(cc, bOpts)
547+
548+
configJSON1 := fmt.Sprintf(`{
549+
"children": {
550+
"cds:cluster_1":{ "childPolicy": [{"%s":""}] }
551+
}
552+
}`, balancerName)
553+
config1, err := rtParser.ParseConfig([]byte(configJSON1))
554+
if err != nil {
555+
t.Fatalf("failed to parse balancer config: %v", err)
556+
}
557+
558+
if err := rtb.UpdateClientConnState(balancer.ClientConnState{BalancerConfig: config1}); err != nil {
559+
t.Fatalf("failed to update ClientConn state: %v", err)
560+
}
561+
ctx, cancel := context.WithTimeout(context.Background(), defaultTestTimeout)
562+
defer cancel()
563+
if v, err := ccsCh.Receive(ctx); err != nil {
564+
err2 := v.(error)
565+
t.Fatal(err2)
566+
}
567+
}

‎xds/internal/balancer/edsbalancer/eds.go

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -48,8 +48,8 @@ type xdsClientInterface interface {
4848
}
4949

5050
var (
51-
newEDSBalancer = func(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lw load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
52-
return newEDSBalancerImpl(cc, enqueueState, lw, logger)
51+
newEDSBalancer = func(cc balancer.ClientConn, opts balancer.BuildOptions, enqueueState func(priorityType, balancer.State), lw load.PerClusterReporter, logger *grpclog.PrefixLogger) edsBalancerImplInterface {
52+
return newEDSBalancerImpl(cc, opts, enqueueState, lw, logger)
5353
}
5454
newXDSClient = func() (xdsClientInterface, error) { return xdsclient.New() }
5555
)
@@ -61,7 +61,7 @@ func init() {
6161
type edsBalancerBuilder struct{}
6262

6363
// Build helps implement the balancer.Builder interface.
64-
func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptions) balancer.Balancer {
64+
func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Balancer {
6565
x := &edsBalancer{
6666
cc: cc,
6767
closed: grpcsync.NewEvent(),
@@ -80,7 +80,7 @@ func (b *edsBalancerBuilder) Build(cc balancer.ClientConn, _ balancer.BuildOptio
8080
}
8181

8282
x.xdsClient = client
83-
x.edsImpl = newEDSBalancer(x.cc, x.enqueueChildBalancerState, x.lsw, x.logger)
83+
x.edsImpl = newEDSBalancer(x.cc, opts, x.enqueueChildBalancerState, x.lsw, x.logger)
8484
x.logger.Infof("Created")
8585
go x.run()
8686
return x

‎xds/internal/balancer/edsbalancer/eds_impl.go

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ import (
2323
"time"
2424

2525
"github.com/google/go-cmp/cmp"
26+
2627
"google.golang.org/grpc/balancer"
2728
"google.golang.org/grpc/balancer/base"
2829
"google.golang.org/grpc/balancer/roundrobin"
@@ -65,6 +66,7 @@ type balancerGroupWithConfig struct {
6566
// policy is used to manage endpoints in each locality.
6667
type edsBalancerImpl struct {
6768
cc balancer.ClientConn
69+
buildOpts balancer.BuildOptions
6870
logger *grpclog.PrefixLogger
6971
loadReporter load.PerClusterReporter
7072

@@ -102,9 +104,10 @@ type edsBalancerImpl struct {
102104
}
103105

104106
// newEDSBalancerImpl create a new edsBalancerImpl.
105-
func newEDSBalancerImpl(cc balancer.ClientConn, enqueueState func(priorityType, balancer.State), lr load.PerClusterReporter, logger *grpclog.PrefixLogger) *edsBalancerImpl {
107+
func newEDSBalancerImpl(cc balancer.ClientConn, bOpts balancer.BuildOptions, enqueueState func(priorityType, balancer.State), lr load.PerClusterReporter, logger *grpclog.PrefixLogger) *edsBalancerImpl {
106108
edsImpl := &edsBalancerImpl{
107109
cc: cc,
110+
buildOpts: bOpts,
108111
logger: logger,
109112
subBalancerBuilder: balancer.Get(roundrobin.Name),
110113
loadReporter: lr,
@@ -248,7 +251,7 @@ func (edsImpl *edsBalancerImpl) handleEDSResponse(edsResp xdsclient.EndpointsUpd
248251
ccPriorityWrapper := edsImpl.ccWrapperWithPriority(priority)
249252
stateAggregator := weightedaggregator.New(ccPriorityWrapper, edsImpl.logger, newRandomWRR)
250253
bgwc = &balancerGroupWithConfig{
251-
bg: balancergroup.New(ccPriorityWrapper, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
254+
bg: balancergroup.New(ccPriorityWrapper, edsImpl.buildOpts, stateAggregator, edsImpl.loadReporter, edsImpl.logger),
252255
stateAggregator: stateAggregator,
253256
configs: make(map[internal.LocalityID]*localityConfig),
254257
}

‎xds/internal/balancer/edsbalancer/eds_impl_priority_test.go

Lines changed: 10 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -35,7 +35,7 @@ import (
3535
// Init 0 and 1; 0 is up, use 0; add 2, use 0; remove 2, use 0.
3636
func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
3737
cc := testutils.NewTestClientConn(t)
38-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
38+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
3939
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
4040

4141
// Two localities, with priorities [0, 1], each with one backend.
@@ -101,7 +101,7 @@ func (s) TestEDSPriority_HighPriorityReady(t *testing.T) {
101101
// down, use 2; remove 2, use 1.
102102
func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
103103
cc := testutils.NewTestClientConn(t)
104-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
104+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
105105
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
106106

107107
// Two localities, with priorities [0, 1], each with one backend.
@@ -208,7 +208,7 @@ func (s) TestEDSPriority_SwitchPriority(t *testing.T) {
208208
// Init 0 and 1; 0 and 1 both down; add 2, use 2.
209209
func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
210210
cc := testutils.NewTestClientConn(t)
211-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
211+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
212212
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
213213

214214
// Two localities, with different priorities, each with one backend.
@@ -271,7 +271,7 @@ func (s) TestEDSPriority_HigherDownWhileAddingLower(t *testing.T) {
271271
// Init 0,1,2; 0 and 1 down, use 2; 0 up, close 1 and 2.
272272
func (s) TestEDSPriority_HigherReadyCloseAllLower(t *testing.T) {
273273
cc := testutils.NewTestClientConn(t)
274-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
274+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
275275
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
276276

277277
// Two localities, with priorities [0,1,2], each with one backend.
@@ -353,7 +353,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
353353
}()()
354354

355355
cc := testutils.NewTestClientConn(t)
356-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
356+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
357357
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
358358

359359
// Two localities, with different priorities, each with one backend.
@@ -403,7 +403,7 @@ func (s) TestEDSPriority_InitTimeout(t *testing.T) {
403403
// - add localities to existing p0 and p1
404404
func (s) TestEDSPriority_MultipleLocalities(t *testing.T) {
405405
cc := testutils.NewTestClientConn(t)
406-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
406+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
407407
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
408408

409409
// Two localities, with different priorities, each with one backend.
@@ -514,7 +514,7 @@ func (s) TestEDSPriority_RemovesAllLocalities(t *testing.T) {
514514
}()()
515515

516516
cc := testutils.NewTestClientConn(t)
517-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
517+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
518518
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
519519

520520
// Two localities, with different priorities, each with one backend.
@@ -698,7 +698,7 @@ func (s) TestPriorityTypeEqual(t *testing.T) {
698698
// will be used.
699699
func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
700700
cc := testutils.NewTestClientConn(t)
701-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
701+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
702702
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
703703

704704
// Two localities, with priorities [0, 1], each with one backend.
@@ -757,7 +757,7 @@ func (s) TestEDSPriority_HighPriorityNoEndpoints(t *testing.T) {
757757
// priority will be used.
758758
func (s) TestEDSPriority_HighPriorityAllUnhealthy(t *testing.T) {
759759
cc := testutils.NewTestClientConn(t)
760-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
760+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
761761
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
762762

763763
// Two localities, with priorities [0, 1], each with one backend.
@@ -823,7 +823,7 @@ func (s) TestEDSPriority_FirstPriorityUnavailable(t *testing.T) {
823823
defaultPriorityInitTimeout = testPriorityInitTimeout
824824

825825
cc := testutils.NewTestClientConn(t)
826-
edsb := newEDSBalancerImpl(cc, nil, nil, nil)
826+
edsb := newEDSBalancerImpl(cc, balancer.BuildOptions{}, nil, nil, nil)
827827
edsb.enqueueChildBalancerStateUpdate = edsb.updateState
828828

829829
// One localities, with priorities [0], each with one backend.

0 commit comments

Comments
 (0)