Skip to content

Commit 8a497b9

Browse files
yuntanghsuyuntanghsu
committed
Add Inventory Controller watchers and VM tag management for VKS IDPS support
Add NSXServiceAccount and VirtualMachine watchers to the Inventory Controller, along with add/remove VM tag functionality in the Inventory Service. This enables tagging NSX Inventory VMs with nsx-op/cluster-name to support IDPS correlation of Kubernetes objects to IPs in VKS clusters. Key changes: - Add SupervisorClusterName field to NSXServiceAccountStatus and CRD schema, with backfill logic for already-realized ServiceAccounts - Implement VirtualMachine informer that enqueues running VKS VMs for tag processing, with a dedicated delete handler to ensure taggedVMs store cleanup (avoiding memory leak) - Implement NSXServiceAccount informer that enqueues VMs on SA realization and deletion for tag add/remove - Add SyncVirtualMachineTag with idempotent add/remove logic using NSX Fabric API (POST update_tags), backed by in-memory taggedVMs store rehydrated from NSX on startup via initTaggedVMs - Fix HandleHTTPResponse to accept 204 No Content and empty bodies - Add comprehensive unit tests for all handlers and service logic VGL-51541 Signed-off-by: Yun-Tang Hsu <yun-tang.hsu@broadcom.com>
1 parent 6f7ddad commit 8a497b9

18 files changed

+2370
-33
lines changed

‎build/yaml/crd/legacy/nsx.vmware.com_nsxserviceaccounts.yaml‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -154,6 +154,8 @@ spec:
154154
type: object
155155
reason:
156156
type: string
157+
supervisorClusterName:
158+
type: string
157159
secrets:
158160
items:
159161
properties:

‎pkg/apis/legacy/v1alpha1/nsxserviceaccount_types.go‎

Lines changed: 9 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -66,14 +66,15 @@ type NSXServiceAccountStatus struct {
6666
Reason string `json:"reason,omitempty"`
6767
// Represents the realization status of a NSXServiceAccount's current state.
6868
// Known .status.conditions.type is: "Realized"
69-
Conditions []metav1.Condition `json:"conditions,omitempty"`
70-
VPCPath string `json:"vpcPath,omitempty"`
71-
NSXManagers []string `json:"nsxManagers,omitempty"`
72-
ProxyEndpoints NSXProxyEndpoint `json:"proxyEndpoints,omitempty"`
73-
ClusterID string `json:"clusterID,omitempty"`
74-
ClusterName string `json:"clusterName,omitempty"`
75-
Secrets []NSXSecret `json:"secrets,omitempty"`
76-
NSXRestoreStatus *NSXRestoreStatus `json:"nsxRestoreStatus,omitempty"`
69+
Conditions []metav1.Condition `json:"conditions,omitempty"`
70+
VPCPath string `json:"vpcPath,omitempty"`
71+
NSXManagers []string `json:"nsxManagers,omitempty"`
72+
ProxyEndpoints NSXProxyEndpoint `json:"proxyEndpoints,omitempty"`
73+
ClusterID string `json:"clusterID,omitempty"`
74+
ClusterName string `json:"clusterName,omitempty"`
75+
SupervisorClusterName string `json:"supervisorClusterName,omitempty"`
76+
Secrets []NSXSecret `json:"secrets,omitempty"`
77+
NSXRestoreStatus *NSXRestoreStatus `json:"nsxRestoreStatus,omitempty"`
7778
}
7879

7980
// +genclient

‎pkg/controllers/inventory/inventory_controller.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,8 @@ var (
4545
watchIngress,
4646
watchNode,
4747
watchNetworkPolicy,
48+
watchNSXServiceAccount,
49+
watchVirtualMachine,
4850
}
4951
)
5052

‎pkg/controllers/inventory/inventory_controller_test.go‎

Lines changed: 13 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -53,17 +53,27 @@ func (m *MockCache) List(ctx context.Context, list client.ObjectList, opts ...cl
5353
}
5454
func (m *MockCache) GetInformer(ctx context.Context, obj client.Object, opts ...cache.InformerGetOption) (cache.Informer, error) {
5555
args := m.Called(ctx, obj)
56-
return &MockInformer{}, args.Error(1)
56+
informer, _ := args.Get(0).(cache.Informer)
57+
return informer, args.Error(1)
5758
}
5859

5960
type MockInformer struct {
6061
mock.Mock
61-
handlers toolscache.ResourceEventHandlerFuncs
62+
handlers toolscache.ResourceEventHandlerFuncs
63+
registeredHandler toolscache.ResourceEventHandler
64+
addHandlerErr error
6265
cache.Informer
6366
}
6467

6568
func (m *MockInformer) AddEventHandler(handler toolscache.ResourceEventHandler) (toolscache.ResourceEventHandlerRegistration, error) {
66-
if m != nil && m.handlers.AddFunc != nil {
69+
if m == nil {
70+
return nil, nil
71+
}
72+
if m.addHandlerErr != nil {
73+
return nil, m.addHandlerErr
74+
}
75+
m.registeredHandler = handler
76+
if m.handlers.AddFunc != nil {
6777
m.handlers.AddFunc(handler)
6878
}
6979
return m, nil

‎pkg/controllers/inventory/nsxserviceaccount_handler.go‎

Lines changed: 126 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,126 @@
1+
package inventory
2+
3+
import (
4+
"context"
5+
"fmt"
6+
"strings"
7+
8+
vmv1alpha1 "github.com/vmware-tanzu/vm-operator/api/v1alpha1"
9+
"k8s.io/apimachinery/pkg/labels"
10+
"k8s.io/client-go/tools/cache"
11+
ctrl "sigs.k8s.io/controller-runtime"
12+
"sigs.k8s.io/controller-runtime/pkg/client"
13+
14+
nsxvmwarecomv1alpha1 "github.com/vmware-tanzu/nsx-operator/pkg/apis/legacy/v1alpha1"
15+
"github.com/vmware-tanzu/nsx-operator/pkg/nsx/services/inventory"
16+
)
17+
18+
func watchNSXServiceAccount(c *InventoryController, mgr ctrl.Manager) error {
19+
nsxSAInformer, err := mgr.GetCache().GetInformer(context.Background(), &nsxvmwarecomv1alpha1.NSXServiceAccount{})
20+
if err != nil {
21+
log.Error(err, "Failed to create NSXServiceAccount informer")
22+
return err
23+
}
24+
25+
_, err = nsxSAInformer.AddEventHandler(cache.ResourceEventHandlerFuncs{
26+
AddFunc: func(obj interface{}) {
27+
c.handleNSXServiceAccount(obj)
28+
},
29+
UpdateFunc: func(oldObj, newObj interface{}) {
30+
c.handleNSXServiceAccount(newObj)
31+
},
32+
DeleteFunc: func(obj interface{}) {
33+
c.handleNSXServiceAccountDelete(obj)
34+
},
35+
})
36+
if err != nil {
37+
log.Error(err, "Failed to add NSXServiceAccount event handler")
38+
return err
39+
}
40+
return nil
41+
}
42+
43+
func (c *InventoryController) handleNSXServiceAccount(obj interface{}) {
44+
var nsxSA *nsxvmwarecomv1alpha1.NSXServiceAccount
45+
switch v := obj.(type) {
46+
case *nsxvmwarecomv1alpha1.NSXServiceAccount:
47+
nsxSA = v
48+
case cache.DeletedFinalStateUnknown:
49+
var ok bool
50+
nsxSA, ok = v.Obj.(*nsxvmwarecomv1alpha1.NSXServiceAccount)
51+
if !ok {
52+
err := fmt.Errorf("obj is not valid *NSXServiceAccount")
53+
log.Error(err, "DeletedFinalStateUnknown Obj is not *NSXServiceAccount")
54+
return
55+
}
56+
}
57+
58+
if nsxSA.Status.Phase != nsxvmwarecomv1alpha1.NSXServiceAccountPhaseRealized {
59+
log.Debug("Skip NSXServiceAccount not yet realized", "namespace", nsxSA.Namespace, "name", nsxSA.Name)
60+
return
61+
}
62+
63+
c.enqueueVMsForCluster(nsxSA)
64+
}
65+
66+
func (c *InventoryController) handleNSXServiceAccountDelete(obj interface{}) {
67+
var nsxSA *nsxvmwarecomv1alpha1.NSXServiceAccount
68+
switch v := obj.(type) {
69+
case *nsxvmwarecomv1alpha1.NSXServiceAccount:
70+
nsxSA = v
71+
case cache.DeletedFinalStateUnknown:
72+
var ok bool
73+
nsxSA, ok = v.Obj.(*nsxvmwarecomv1alpha1.NSXServiceAccount)
74+
if !ok {
75+
err := fmt.Errorf("obj is not valid *NSXServiceAccount")
76+
log.Error(err, "DeletedFinalStateUnknown Obj is not *NSXServiceAccount")
77+
return
78+
}
79+
}
80+
81+
log.Info("NSXServiceAccount deleted, enqueuing VMs for tag removal", "namespace", nsxSA.Namespace, "name", nsxSA.Name)
82+
c.enqueueVMsForCluster(nsxSA)
83+
}
84+
85+
// enqueueVMsForCluster lists VirtualMachines belonging to the NSXServiceAccount's
86+
// CAPI cluster and enqueues them to the inventory queue for VM tag processing.
87+
func (c *InventoryController) enqueueVMsForCluster(nsxSA *nsxvmwarecomv1alpha1.NSXServiceAccount) {
88+
clusterName := getClusterNameFromSA(nsxSA)
89+
if clusterName == "" {
90+
log.Info("NSXServiceAccount has no Cluster OwnerReference, skipping VM enqueue",
91+
"namespace", nsxSA.Namespace, "name", nsxSA.Name)
92+
return
93+
}
94+
95+
vmList := &vmv1alpha1.VirtualMachineList{}
96+
if err := c.Client.List(context.Background(), vmList, &client.ListOptions{
97+
Namespace: nsxSA.Namespace,
98+
LabelSelector: labels.SelectorFromSet(labels.Set{inventory.CAPIClusterNameLabel: clusterName}),
99+
}); err != nil {
100+
log.Error(err, "Failed to list VirtualMachines for cluster",
101+
"namespace", nsxSA.Namespace, "cluster", clusterName)
102+
return
103+
}
104+
105+
for i := range vmList.Items {
106+
vm := &vmList.Items[i]
107+
log.Debug("Enqueuing VM from NSXServiceAccount event",
108+
"namespace", vm.Namespace, "name", vm.Name, "cluster", clusterName)
109+
key, _ := keyFunc(vm)
110+
c.inventoryObjectQueue.Add(inventory.InventoryKey{
111+
InventoryType: inventory.InventoryVirtualMachine,
112+
ExternalId: vm.Status.InstanceUUID,
113+
Key: key,
114+
})
115+
}
116+
}
117+
118+
// getClusterNameFromSA extracts the CAPI Cluster name from the NSXServiceAccount's OwnerReferences.
119+
func getClusterNameFromSA(nsxSA *nsxvmwarecomv1alpha1.NSXServiceAccount) string {
120+
for _, ref := range nsxSA.OwnerReferences {
121+
if ref.Kind == "Cluster" && strings.Contains(ref.APIVersion, "cluster.x-k8s.io") {
122+
return ref.Name
123+
}
124+
}
125+
return ""
126+
}

0 commit comments

Comments
 (0)