Skip to content

Commit b1418a6

Browse files
authored
xds: export XDSClient interface and use it in balancer tests (#4510)
- xdsclient.New returns the interface now - xdsclient.SetClient and xdsclient.FromResolverState take and return the interface now - cleanup xds balancer tests to pass xds_client in resolver state
1 parent 7301a31 commit b1418a6

26 files changed

+173
-304
lines changed

xds/csds/csds.go

Lines changed: 2 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -38,33 +38,17 @@ import (
3838
"google.golang.org/grpc/grpclog"
3939
"google.golang.org/grpc/status"
4040
"google.golang.org/grpc/xds/internal/xdsclient"
41-
"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
4241
"google.golang.org/protobuf/types/known/timestamppb"
4342

4443
_ "google.golang.org/grpc/xds/internal/xdsclient/v2" // Register v2 xds_client.
4544
_ "google.golang.org/grpc/xds/internal/xdsclient/v3" // Register v3 xds_client.
4645
)
4746

48-
// xdsClient contains methods from xdsClient.Client which are used by
49-
// the server. This is useful for overriding in unit tests.
50-
type xdsClient interface {
51-
DumpLDS() (string, map[string]xdsclient.UpdateWithMD)
52-
DumpRDS() (string, map[string]xdsclient.UpdateWithMD)
53-
DumpCDS() (string, map[string]xdsclient.UpdateWithMD)
54-
DumpEDS() (string, map[string]xdsclient.UpdateWithMD)
55-
BootstrapConfig() *bootstrap.Config
56-
Close()
57-
}
58-
5947
var (
6048
logger = grpclog.Component("xds")
61-
newXDSClient = func() xdsClient {
49+
newXDSClient = func() xdsclient.XDSClient {
6250
c, err := xdsclient.New()
6351
if err != nil {
64-
// If err is not nil, c is a typed nil (of type *xdsclient.Client).
65-
// If c is returned and assigned to the xdsClient field in the CSDS
66-
// server, the nil checks in the handlers will not handle it
67-
// properly.
6852
logger.Warningf("failed to create xds client: %v", err)
6953
return nil
7054
}
@@ -76,7 +60,7 @@ var (
7660
type ClientStatusDiscoveryServer struct {
7761
// xdsClient will always be the same in practice. But we keep a copy in each
7862
// server instance for testing.
79-
xdsClient xdsClient
63+
xdsClient xdsclient.XDSClient
8064
}
8165

8266
// NewClientStatusDiscoveryServer returns an implementation of the CSDS server that can be

xds/csds/csds_test.go

Lines changed: 3 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -59,13 +59,6 @@ const (
5959
defaultTestTimeout = 10 * time.Second
6060
)
6161

62-
type xdsClientWithWatch interface {
63-
WatchListener(string, func(xdsclient.ListenerUpdate, error)) func()
64-
WatchRouteConfig(string, func(xdsclient.RouteConfigUpdate, error)) func()
65-
WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
66-
WatchEndpoints(string, func(xdsclient.EndpointsUpdate, error)) func()
67-
}
68-
6962
var cmpOpts = cmp.Options{
7063
cmpopts.EquateEmpty(),
7164
cmp.Comparer(func(a, b *timestamppb.Timestamp) bool { return true }),
@@ -250,7 +243,7 @@ func TestCSDS(t *testing.T) {
250243
}
251244
}
252245

253-
func commonSetup(t *testing.T) (xdsClientWithWatch, *e2e.ManagementServer, string, v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient, func()) {
246+
func commonSetup(t *testing.T) (xdsclient.XDSClient, *e2e.ManagementServer, string, v3statuspbgrpc.ClientStatusDiscoveryService_StreamClientStatusClient, func()) {
254247
t.Helper()
255248

256249
// Spin up a xDS management server on a local port.
@@ -275,7 +268,7 @@ func commonSetup(t *testing.T) (xdsClientWithWatch, *e2e.ManagementServer, strin
275268
t.Fatalf("failed to create xds client: %v", err)
276269
}
277270
oldNewXDSClient := newXDSClient
278-
newXDSClient = func() xdsClient { return xdsC }
271+
newXDSClient = func() xdsclient.XDSClient { return xdsC }
279272

280273
// Initialize an gRPC server and register CSDS on it.
281274
server := grpc.NewServer()
@@ -635,7 +628,7 @@ func protoToJSON(p proto.Message) string {
635628

636629
func TestCSDSNoXDSClient(t *testing.T) {
637630
oldNewXDSClient := newXDSClient
638-
newXDSClient = func() xdsClient { return nil }
631+
newXDSClient = func() xdsclient.XDSClient { return nil }
639632
defer func() { newXDSClient = oldNewXDSClient }()
640633

641634
// Initialize an gRPC server and register CSDS on it.

xds/googledirectpath/googlec2p.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -62,15 +62,11 @@ const (
6262
dnsName, xdsName = "dns", "xds"
6363
)
6464

65-
type xdsClient interface {
66-
Close()
67-
}
68-
6965
// For overriding in unittests.
7066
var (
7167
onGCE = googlecloud.OnGCE
7268

73-
newClientWithConfig = func(config *bootstrap.Config) (xdsClient, error) {
69+
newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
7470
return xdsclient.NewWithConfig(config)
7571
}
7672

@@ -139,7 +135,7 @@ func (c2pResolverBuilder) Scheme() string {
139135

140136
type c2pResolver struct {
141137
resolver.Resolver
142-
client xdsClient
138+
client xdsclient.XDSClient
143139
}
144140

145141
func (r *c2pResolver) Close() {

xds/googledirectpath/googlec2p_test.go

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@ import (
3232
"google.golang.org/grpc/internal/xds/env"
3333
"google.golang.org/grpc/resolver"
3434
"google.golang.org/grpc/xds/internal/version"
35+
"google.golang.org/grpc/xds/internal/xdsclient"
3536
"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
3637
"google.golang.org/protobuf/testing/protocmp"
3738
"google.golang.org/protobuf/types/known/structpb"
@@ -130,6 +131,7 @@ func TestBuildNotOnGCE(t *testing.T) {
130131
}
131132

132133
type testXDSClient struct {
134+
xdsclient.XDSClient
133135
closed chan struct{}
134136
}
135137

@@ -177,7 +179,7 @@ func TestBuildXDS(t *testing.T) {
177179

178180
configCh := make(chan *bootstrap.Config, 1)
179181
oldNewClient := newClientWithConfig
180-
newClientWithConfig = func(config *bootstrap.Config) (xdsClient, error) {
182+
newClientWithConfig = func(config *bootstrap.Config) (xdsclient.XDSClient, error) {
181183
configCh <- config
182184
return tXDSClient, nil
183185
}

xds/internal/balancer/cdsbalancer/cdsbalancer.go

Lines changed: 2 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,6 @@ import (
3636
"google.golang.org/grpc/serviceconfig"
3737
"google.golang.org/grpc/xds/internal/balancer/edsbalancer"
3838
"google.golang.org/grpc/xds/internal/xdsclient"
39-
"google.golang.org/grpc/xds/internal/xdsclient/bootstrap"
4039
)
4140

4241
const (
@@ -59,7 +58,6 @@ var (
5958
// not deal with subConns.
6059
return builder.Build(cc, opts), nil
6160
}
62-
newXDSClient func() (xdsClient, error)
6361
buildProvider = buildProviderFunc
6462
)
6563

@@ -84,17 +82,6 @@ func (bb) Build(cc balancer.ClientConn, opts balancer.BuildOptions) balancer.Bal
8482
}
8583
b.logger = prefixLogger((b))
8684
b.logger.Infof("Created")
87-
88-
if newXDSClient != nil {
89-
// For tests
90-
client, err := newXDSClient()
91-
if err != nil {
92-
b.logger.Errorf("failed to create xds-client: %v", err)
93-
return nil
94-
}
95-
b.xdsClient = client
96-
}
97-
9885
var creds credentials.TransportCredentials
9986
switch {
10087
case opts.DialCreds != nil:
@@ -137,14 +124,6 @@ func (bb) ParseConfig(c json.RawMessage) (serviceconfig.LoadBalancingConfig, err
137124
return &cfg, nil
138125
}
139126

140-
// xdsClient contains methods from xdsClient.Client which are used by
141-
// the cdsBalancer. This will be faked out in unittests.
142-
type xdsClient interface {
143-
WatchCluster(string, func(xdsclient.ClusterUpdate, error)) func()
144-
BootstrapConfig() *bootstrap.Config
145-
Close()
146-
}
147-
148127
// ccUpdate wraps a clientConn update received from gRPC (pushed from the
149128
// xdsResolver). A valid clusterName causes the cdsBalancer to register a CDS
150129
// watcher with the xdsClient, while a non-nil error causes it to cancel the
@@ -184,7 +163,7 @@ type cdsBalancer struct {
184163
ccw *ccWrapper // ClientConn interface passed to child LB.
185164
bOpts balancer.BuildOptions // BuildOptions passed to child LB.
186165
updateCh *buffer.Unbounded // Channel for gRPC and xdsClient updates.
187-
xdsClient xdsClient // xDS client to watch Cluster resource.
166+
xdsClient xdsclient.XDSClient // xDS client to watch Cluster resource.
188167
cancelWatch func() // Cluster watch cancel func.
189168
edsLB balancer.Balancer // EDS child policy.
190169
clusterToWatch string
@@ -361,15 +340,8 @@ func (b *cdsBalancer) handleWatchUpdate(update *watchUpdate) {
361340
lbCfg.LrsLoadReportingServerName = new(string)
362341

363342
}
364-
resolverState := resolver.State{}
365-
// Include the xds client for the child LB policies to use. For unit
366-
// tests, b.xdsClient may not be a full *xdsclient.Client, but it will
367-
// always be in production.
368-
if c, ok := b.xdsClient.(*xdsclient.Client); ok {
369-
resolverState = xdsclient.SetClient(resolverState, c)
370-
}
371343
ccState := balancer.ClientConnState{
372-
ResolverState: resolverState,
344+
ResolverState: xdsclient.SetClient(resolver.State{}, b.xdsClient),
373345
BalancerConfig: lbCfg,
374346
}
375347
if err := b.edsLB.UpdateClientConnState(ccState); err != nil {
@@ -407,9 +379,6 @@ func (b *cdsBalancer) run() {
407379
b.edsLB.Close()
408380
b.edsLB = nil
409381
}
410-
if newXDSClient != nil {
411-
b.xdsClient.Close()
412-
}
413382
if b.cachedRoot != nil {
414383
b.cachedRoot.Close()
415384
}

xds/internal/balancer/cdsbalancer/cdsbalancer_security_test.go

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -133,11 +133,7 @@ func (p *fakeProvider) Close() {
133133
// xDSCredentials.
134134
func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
135135
t.Helper()
136-
137136
xdsC := fakeclient.NewClient()
138-
oldNewXDSClient := newXDSClient
139-
newXDSClient = func() (xdsClient, error) { return xdsC, nil }
140-
141137
builder := balancer.Get(cdsName)
142138
if builder == nil {
143139
t.Fatalf("balancer.Get(%q) returned nil", cdsName)
@@ -164,7 +160,7 @@ func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDS
164160
}
165161

166162
// Push a ClientConnState update to the CDS balancer with a cluster name.
167-
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
163+
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
168164
t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
169165
}
170166

@@ -181,8 +177,8 @@ func setupWithXDSCreds(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDS
181177
}
182178

183179
return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
184-
newXDSClient = oldNewXDSClient
185180
newEDSBalancer = oldEDSBalancerBuilder
181+
xdsC.Close()
186182
}
187183
}
188184

xds/internal/balancer/cdsbalancer/cdsbalancer_test.go

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,6 @@ import (
2828

2929
"github.com/google/go-cmp/cmp"
3030
"github.com/google/go-cmp/cmp/cmpopts"
31-
"google.golang.org/grpc/attributes"
3231
"google.golang.org/grpc/balancer"
3332
"google.golang.org/grpc/connectivity"
3433
"google.golang.org/grpc/internal"
@@ -129,7 +128,10 @@ func (tb *testEDSBalancer) waitForClientConnUpdate(ctx context.Context, wantCCS
129128
return err
130129
}
131130
gotCCS := ccs.(balancer.ClientConnState)
132-
if !cmp.Equal(gotCCS, wantCCS, cmpopts.IgnoreUnexported(attributes.Attributes{})) {
131+
if xdsclient.FromResolverState(gotCCS.ResolverState) == nil {
132+
return fmt.Errorf("want resolver state with XDSClient attached, got one without")
133+
}
134+
if !cmp.Equal(gotCCS, wantCCS, cmpopts.IgnoreFields(resolver.State{}, "Attributes")) {
133135
return fmt.Errorf("received ClientConnState: %+v, want %+v", gotCCS, wantCCS)
134136
}
135137
return nil
@@ -173,7 +175,7 @@ func (tb *testEDSBalancer) waitForClose(ctx context.Context) error {
173175

174176
// cdsCCS is a helper function to construct a good update passed from the
175177
// xdsResolver to the cdsBalancer.
176-
func cdsCCS(cluster string) balancer.ClientConnState {
178+
func cdsCCS(cluster string, xdsC xdsclient.XDSClient) balancer.ClientConnState {
177179
const cdsLBConfig = `{
178180
"loadBalancingConfig":[
179181
{
@@ -185,9 +187,9 @@ func cdsCCS(cluster string) balancer.ClientConnState {
185187
}`
186188
jsonSC := fmt.Sprintf(cdsLBConfig, cluster)
187189
return balancer.ClientConnState{
188-
ResolverState: resolver.State{
190+
ResolverState: xdsclient.SetClient(resolver.State{
189191
ServiceConfig: internal.ParseServiceConfigForTesting.(func(string) *serviceconfig.ParseResult)(jsonSC),
190-
},
192+
}, xdsC),
191193
BalancerConfig: &lbConfig{ClusterName: clusterName},
192194
}
193195
}
@@ -211,11 +213,7 @@ func edsCCS(service string, countMax *uint32, enableLRS bool) balancer.ClientCon
211213
// newEDSBalancer function to return it), and also returns a cleanup function.
212214
func setup(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *xdstestutils.TestClientConn, func()) {
213215
t.Helper()
214-
215216
xdsC := fakeclient.NewClient()
216-
oldNewXDSClient := newXDSClient
217-
newXDSClient = func() (xdsClient, error) { return xdsC, nil }
218-
219217
builder := balancer.Get(cdsName)
220218
if builder == nil {
221219
t.Fatalf("balancer.Get(%q) returned nil", cdsName)
@@ -232,7 +230,7 @@ func setup(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBalancer, *x
232230

233231
return xdsC, cdsB.(*cdsBalancer), edsB, tcc, func() {
234232
newEDSBalancer = oldEDSBalancerBuilder
235-
newXDSClient = oldNewXDSClient
233+
xdsC.Close()
236234
}
237235
}
238236

@@ -242,7 +240,7 @@ func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBal
242240
t.Helper()
243241

244242
xdsC, cdsB, edsB, tcc, cancel := setup(t)
245-
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
243+
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
246244
t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
247245
}
248246

@@ -262,6 +260,9 @@ func setupWithWatch(t *testing.T) (*fakeclient.Client, *cdsBalancer, *testEDSBal
262260
// cdsBalancer with different inputs and verifies that the CDS watch API on the
263261
// provided xdsClient is invoked appropriately.
264262
func (s) TestUpdateClientConnState(t *testing.T) {
263+
xdsC := fakeclient.NewClient()
264+
defer xdsC.Close()
265+
265266
tests := []struct {
266267
name string
267268
ccs balancer.ClientConnState
@@ -280,14 +281,14 @@ func (s) TestUpdateClientConnState(t *testing.T) {
280281
},
281282
{
282283
name: "happy-good-case",
283-
ccs: cdsCCS(clusterName),
284+
ccs: cdsCCS(clusterName, xdsC),
284285
wantCluster: clusterName,
285286
},
286287
}
287288

288289
for _, test := range tests {
289290
t.Run(test.name, func(t *testing.T) {
290-
xdsC, cdsB, _, _, cancel := setup(t)
291+
_, cdsB, _, _, cancel := setup(t)
291292
defer func() {
292293
cancel()
293294
cdsB.Close()
@@ -324,7 +325,7 @@ func (s) TestUpdateClientConnStateWithSameState(t *testing.T) {
324325
}()
325326

326327
// This is the same clientConn update sent in setupWithWatch().
327-
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != nil {
328+
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != nil {
328329
t.Fatalf("cdsBalancer.UpdateClientConnState failed with error: %v", err)
329330
}
330331
// The above update should not result in a new watch being registered.
@@ -660,7 +661,7 @@ func (s) TestClose(t *testing.T) {
660661

661662
// Make sure that the UpdateClientConnState() method on the CDS balancer
662663
// returns error.
663-
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName)); err != errBalancerClosed {
664+
if err := cdsB.UpdateClientConnState(cdsCCS(clusterName, xdsC)); err != errBalancerClosed {
664665
t.Fatalf("UpdateClientConnState() after close returned %v, want %v", err, errBalancerClosed)
665666
}
666667

xds/internal/balancer/cdsbalancer/cluster_handler.go

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ type clusterHandler struct {
4040
// CDS Balancer cares about is the most recent update.
4141
updateChannel chan clusterHandlerUpdate
4242

43-
xdsClient xdsClient
43+
xdsClient xdsclient.XDSClient
4444
}
4545

4646
func (ch *clusterHandler) updateRootCluster(rootClusterName string) {
@@ -112,7 +112,7 @@ type clusterNode struct {
112112

113113
// CreateClusterNode creates a cluster node from a given clusterName. This will
114114
// also start the watch for that cluster.
115-
func createClusterNode(clusterName string, xdsClient xdsClient, topLevelHandler *clusterHandler) *clusterNode {
115+
func createClusterNode(clusterName string, xdsClient xdsclient.XDSClient, topLevelHandler *clusterHandler) *clusterNode {
116116
c := &clusterNode{
117117
clusterHandler: topLevelHandler,
118118
}

0 commit comments

Comments
 (0)