OTA-1643: Each CO must go Progressing during upgrade

Author: hongkailiu

pkg/monitortests/clusterversionoperator/legacycvomonitortests/monitortest.go

@@ -45,7 +45,7 @@ func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.C
 	isUpgrade := platformidentification.DidUpgradeHappenDuringCollection(finalIntervals, time.Time{}, time.Time{})
 	if isUpgrade {
 		junits = append(junits, testUpgradeOperatorStateTransitions(finalIntervals, w.adminRESTConfig)...)
-		junits = append(junits, clusterOperatorIsNotProgressingWhenMachineConfigIs(finalIntervals)...)
+		junits = append(junits, testUpgradeOperatorProgressingStateTransitions(finalIntervals)...)
 	} else {
 		junits = append(junits, testStableSystemOperatorStateTransitions(finalIntervals, w.adminRESTConfig)...)
 	}

pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go

@@ -599,13 +599,11 @@ func testOperatorStateTransitions(events monitorapi.Intervals, conditionTypes []
 	return ret
 }
 
-func clusterOperatorIsNotProgressingWhenMachineConfigIs(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
+func testUpgradeOperatorProgressingStateTransitions(events monitorapi.Intervals) []*junitapi.JUnitTestCase {
 	var ret []*junitapi.JUnitTestCase
 	upgradeWindows := getUpgradeWindows(events)
 
-	var machineConfigProgressing time.Time
 	var eventsInUpgradeWindows monitorapi.Intervals
-
 	var start, stop time.Time
 	for _, event := range events {
 		if !isInUpgradeWindow(upgradeWindows, event) {
@@ -622,31 +620,50 @@ func clusterOperatorIsNotProgressingWhenMachineConfigIs(events monitorapi.Interv
 	duration := stop.Sub(start).Seconds()
 
 	eventsByOperator := getEventsByOperator(eventsInUpgradeWindows)
-	for _, mcEvent := range eventsByOperator["machine-config"] {
-		condition := monitorapi.GetOperatorConditionStatus(mcEvent)
-		if condition == nil {
-			continue // ignore non-condition intervals
-		}
-		if condition.Type == configv1.OperatorProgressing && condition.Status == configv1.ConditionTrue {
-			machineConfigProgressing = mcEvent.To
-			break
+	coProgressing := map[string]time.Time{}
+	for _, operatorName := range platformidentification.KnownOperators.List() {
+		for _, mcEvent := range eventsByOperator[operatorName] {
+			condition := monitorapi.GetOperatorConditionStatus(mcEvent)
+			if condition == nil {
+				continue // ignore non-condition intervals
+			}
+			if condition.Type == configv1.OperatorProgressing && condition.Status == configv1.ConditionTrue {
+				coProgressing[operatorName] = mcEvent.To
+				break
+			}
 		}
 	}
 
-	mcTestCase := &junitapi.JUnitTestCase{
-		Name:     fmt.Sprintf("[bz-Machine Config Operator] clusteroperator/machine-config must go Progressing=True during an upgrade test"),
-		Duration: duration,
+	// Each cluster operator must report Progressing=True during cluster upgrade
+	var machineConfigProgressing time.Time
+	for _, operatorName := range platformidentification.KnownOperators.List() {
+		bzComponent := platformidentification.GetBugzillaComponentForOperator(operatorName)
+		if bzComponent == "Unknown" {
+			bzComponent = operatorName
+		}
+		mcTestCase := &junitapi.JUnitTestCase{
+			Name:     fmt.Sprintf("[bz-%s] clusteroperator/%s must go Progressing=True during an upgrade test", bzComponent, operatorName),
+			Duration: duration,
+		}
+
+		if t, ok := coProgressing[operatorName]; !ok || t.IsZero() {
+			mcTestCase.FailureOutput = &junitapi.FailureOutput{
+				Output: fmt.Sprintf("clusteroperator/%s was never Progressing=True during the upgrade window from %s to %s", operatorName, start.Format(time.RFC3339), stop.Format(time.RFC3339)),
+			}
+		} else {
+			if operatorName == "machine-config" {
+				machineConfigProgressing = t
+			}
+			mcTestCase.SystemOut = fmt.Sprintf("clusteroperator/%s became Progressing=True at %s during the upgrade window from %s to %s", operatorName, t.Format(time.RFC3339), start.Format(time.RFC3339), stop.Format(time.RFC3339))
+		}
+		ret = append(ret, mcTestCase)
 	}
+
 	if machineConfigProgressing.IsZero() {
-		mcTestCase.FailureOutput = &junitapi.FailureOutput{
-			Output: fmt.Sprintf("machine-config was never Progressing=True during the upgrade window from %s to %s", start.Format(time.RFC3339), stop.Format(time.RFC3339)),
-		}
-		return []*junitapi.JUnitTestCase{mcTestCase}
-	} else {
-		mcTestCase.SystemOut = fmt.Sprintf("machine-config became Progressing=True at %s during the upgrade window from %s to %s", machineConfigProgressing.Format(time.RFC3339), start.Format(time.RFC3339), stop.Format(time.RFC3339))
+		return ret
 	}
-	ret = append(ret, mcTestCase)
 
+	// No cluster operator report Progressing=True when machine-config does
 	for _, operatorName := range platformidentification.KnownOperators.Difference(sets.NewString("machine-config")).List() {
 		bzComponent := platformidentification.GetBugzillaComponentForOperator(operatorName)
 		if bzComponent == "Unknown" {