Merge pull request #1116 from shvbsle/tagging_delay_metrics

feat: add tagging controller delays and work queue size metrics

Author: k8s-ci-robot

pkg/controllers/tagging/metrics.go

@@ -20,6 +20,10 @@ import (
 	"k8s.io/component-base/metrics/legacyregistry"
 )
 
+const (
+	metricsSubsystem = "tagging_controller"
+)
+
 var register sync.Once
 
 var (
@@ -30,12 +34,23 @@ var (
 			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"error_type", "instance_id"})
+
+	initialNodeTaggingDelay = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Subsystem:      metricsSubsystem,
+			Name:           "inital_node_tagging_delay_seconds",
+			Help:           "Latency (in seconds) between node creation and its first successful tagging by TaggingController.",
+			Buckets:        metrics.ExponentialBuckets(1, 4, 6), // 1s -> ~17m
+			StabilityLevel: metrics.ALPHA,
+		},
+	)
 )
 
 // registerMetrics registers tagging-controller metrics.
 func registerMetrics() {
 	register.Do(func() {
 		legacyregistry.MustRegister(workItemError)
+		legacyregistry.MustRegister(initialNodeTaggingDelay)
 	})
 }
 

pkg/controllers/tagging/tagging_controller.go

@@ -347,6 +347,10 @@ func (tc *Controller) tagEc2Instance(node *v1.Node) error {
 
 	klog.Infof("Successfully labeled node %s with %v.", node.GetName(), labels)
 
+	if tc.isInitialTag(node) {
+		initialNodeTaggingDelay.Observe(time.Since(node.CreationTimestamp.Time).Seconds())
+	}
+
 	return nil
 }
 
@@ -403,6 +407,11 @@ func (tc *Controller) enqueueNode(node *v1.Node, action string) {
 	}
 }
 
+func (tc *Controller) isInitialTag(node *v1.Node) bool {
+	_, ok := node.Labels[taggingControllerLabelKey]
+	return !ok
+}
+
 func (tc *Controller) isTaggingRequired(node *v1.Node) bool {
 	if node.Labels == nil {
 		return true

pkg/controllers/tagging/tagging_controller_test.go

@@ -347,3 +347,69 @@ func syncNodeStore(nodeinformer coreinformers.NodeInformer, f *fake.Clientset) e
 	}
 	return nodeinformer.Informer().GetStore().Replace(newElems, "newRV")
 }
+
+func Test_isInitialTag(t *testing.T) {
+	testcases := []struct {
+		name          string
+		node          *v1.Node
+		expectedValue bool
+	}{
+		{
+			name: "node0 is recently created with no labels and will be tagged for the first time",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "node0",
+				},
+			},
+			expectedValue: true,
+		},
+		{
+			name: "node0 has other labels but no taggingControllerLabelKey and will be tagged for the first time",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "node0",
+					Labels: map[string]string{
+						"some-other-label": "value",
+					},
+				},
+			},
+			expectedValue: true,
+		},
+		{
+			name: "node0 with taggingControllerLabelKey implies that the node was already tagged",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "node0",
+					Labels: map[string]string{
+						taggingControllerLabelKey: "9767c4972ba72e87ab553bad2afde741", // MD5 for key1=value1
+					},
+				},
+			},
+			expectedValue: false,
+		},
+		{
+			name: "node0 with taggingControllerLabelKey and other labels should not be initial tag",
+			node: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name: "node0",
+					Labels: map[string]string{
+						taggingControllerLabelKey: "9767c4972ba72e87ab553bad2afde741", // MD5 for key1=value1
+						"some-other-label":        "value",
+					},
+				},
+			},
+			expectedValue: false,
+		},
+	}
+
+	tc := &Controller{}
+
+	for _, testcase := range testcases {
+		t.Run(testcase.name, func(t *testing.T) {
+			result := tc.isInitialTag(testcase.node)
+			if result != testcase.expectedValue {
+				t.Errorf("isInitialTag() = %v, want %v", result, testcase.expectedValue)
+			}
+		})
+	}
+}