Merge branch 'main' into add-cvo-test-extension

Author: DavidHurta

pkg/defaultmonitortests/types.go

@@ -144,6 +144,7 @@ func newDefaultMonitorTests(info monitortestframework.MonitorTestInitializationI
 	monitorTestRegistry.AddMonitorTestOrDie(faultyloadbalancer.MonitorName, "kube-apiserver", faultyloadbalancer.NewMonitorTest())
 	monitorTestRegistry.AddMonitorTestOrDie(staticpodinstall.MonitorName, "kube-apiserver", staticpodinstall.NewStaticPodInstallMonitorTest())
 	monitorTestRegistry.AddMonitorTestOrDie(containerfailures.MonitorName, "Node / Kubelet", containerfailures.NewContainerFailuresTests())
+	monitorTestRegistry.AddMonitorTestOrDie(legacytestframeworkmonitortests.PathologicalMonitorName, "Test Framework", legacytestframeworkmonitortests.NewLegacyPathologicalMonitorTests(info))
 
 	return monitorTestRegistry
 }
@@ -193,7 +194,7 @@ func newUniversalMonitorTests(info monitortestframework.MonitorTestInitializatio
 
 	monitorTestRegistry.AddMonitorTestOrDie("legacy-storage-invariants", "Storage", legacystoragemonitortests.NewLegacyTests())
 
-	monitorTestRegistry.AddMonitorTestOrDie("legacy-test-framework-invariants", "Test Framework", legacytestframeworkmonitortests.NewLegacyTests(info))
+	monitorTestRegistry.AddMonitorTestOrDie(legacytestframeworkmonitortests.AlertsMonitorName, "Test Framework", legacytestframeworkmonitortests.NewLegacyAlertsMonitorTests(info))
 	monitorTestRegistry.AddMonitorTestOrDie("timeline-serializer", "Test Framework", timelineserializer.NewTimelineSerializer())
 	monitorTestRegistry.AddMonitorTestOrDie("interval-serializer", "Test Framework", intervalserializer.NewIntervalSerializer())
 	monitorTestRegistry.AddMonitorTestOrDie("tracked-resources-serializer", "Test Framework", trackedresourcesserializer.NewTrackedResourcesSerializer())

pkg/monitortestlibrary/platformidentification/operator_mapping.go

@@ -111,6 +111,7 @@ var (
 		"monitoring",
 		"network",
 		"node-tuning",
+		"olm",
 		"openshift-apiserver",
 		"openshift-controller-manager",
 		"openshift-samples",
@@ -154,6 +155,7 @@ func init() {
 	utilruntime.Must(addOperatorMapping("monitoring", "Monitoring"))
 	utilruntime.Must(addOperatorMapping("network", "Networking"))
 	utilruntime.Must(addOperatorMapping("node-tuning", "Node Tuning Operator"))
+	utilruntime.Must(addOperatorMapping("olm", "OLM"))
 	utilruntime.Must(addOperatorMapping("openshift-apiserver", "openshift-apiserver"))
 	utilruntime.Must(addOperatorMapping("openshift-controller-manager", "openshift-controller-manager"))
 	utilruntime.Must(addOperatorMapping("openshift-samples", "Samples"))

pkg/monitortests/cli/adm_upgrade/status/monitortest.go

@@ -162,12 +162,13 @@ func (w *monitor) CollectData(ctx context.Context, storageDir string, beginning,
 
 	// TODO: Maybe utilize Intervals somehow and do tests in ComputeComputedIntervals and EvaluateTestsFromConstructedIntervals
 
-	wasUpdated := func() (bool, error) {
+	updateCount := func() (int, error) {
 		cv, err := w.configv1client.ConfigV1().ClusterVersions().Get(ctx, "version", metav1.GetOptions{})
 		if err != nil {
-			return false, fmt.Errorf("failed to get cluster version: %w", err)
+			return 0, fmt.Errorf("failed to get cluster version: %w", err)
 		}
-		return len(cv.Status.History) > len(w.initialClusterVersion.Status.History), nil
+		hops := len(cv.Status.History) - len(w.initialClusterVersion.Status.History)
+		return hops, nil
 	}
 
 	return nil, []*junitapi.JUnitTestCase{
@@ -176,7 +177,7 @@ func (w *monitor) CollectData(ctx context.Context, storageDir string, beginning,
 		w.controlPlane(),
 		w.workers(),
 		w.health(),
-		w.updateLifecycle(wasUpdated),
+		w.updateLifecycle(updateCount),
 	}, nil
 }
 

pkg/monitortests/cli/adm_upgrade/status/updatelifecycle.go

@@ -8,17 +8,25 @@ import (
 	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
 )
 
-type wasUpdatedFn func() (bool, error)
+// wasUpdatedFn returns how many times was the cluster updated while the test was running
+type wasUpdatedFn func() (int, error)
 
 func (w *monitor) updateLifecycle(wasUpdated wasUpdatedFn) *junitapi.JUnitTestCase {
 	health := &junitapi.JUnitTestCase{
 		Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 	}
 
-	clusterUpdated, err := wasUpdated()
+	clusterUpdateCount, err := wasUpdated()
 	if err != nil {
 		health.FailureOutput = &junitapi.FailureOutput{
-			Message: fmt.Sprintf("failed to get cluster version: %v", err),
+			Message: fmt.Sprintf("failed to determined whether the cluster was updated: %v", err),
+		}
+		return health
+	}
+
+	if clusterUpdateCount > 1 {
+		health.SkipMessage = &junitapi.SkipMessage{
+			Message: fmt.Sprintf("Cluster updated more than once (%d times)", clusterUpdateCount),
 		}
 		return health
 	}
@@ -95,7 +103,7 @@ func (w *monitor) updateLifecycle(wasUpdated wasUpdatedFn) *junitapi.JUnitTestCa
 			}
 		}
 
-		if !clusterUpdated {
+		if clusterUpdateCount == 0 {
 			// TODO: MCO churn sometimes briefly tricks our code into thinking the cluster is updating, we'll tolerate for
 			// now but we should try fixing this
 			// if observed.output.updating || observed.output.controlPlane != nil || observed.output.workers != nil || observed.output.health != nil {

pkg/monitortests/cli/adm_upgrade/status/updatelifecycle_test.go

@@ -105,10 +105,10 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 	t.Parallel()
 
 	testCases := []struct {
-		name       string
-		snapshots  []snapshot
-		wasUpdated bool
-		expected   *junitapi.JUnitTestCase
+		name        string
+		snapshots   []snapshot
+		updateCount int
+		expected    *junitapi.JUnitTestCase
 	}{
 		{
 			name: "no snapshots -> test skipped",
@@ -128,7 +128,7 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 				{when: time.Now(), out: lifecycle04controlPlaneUpdated},
 				{when: time.Now(), out: lifecycle05after},
 			},
-			wasUpdated: true,
+			updateCount: 1,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 			},
@@ -140,7 +140,7 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 				{when: time.Now(), out: lifecycle02updating},
 				{when: time.Now(), out: lifecycle05after},
 			},
-			wasUpdated: true,
+			updateCount: 1,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 			},
@@ -154,7 +154,7 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 				{when: time.Now(), out: lifecycle02updating},
 				{when: time.Now(), out: lifecycle05after},
 			},
-			wasUpdated: true,
+			updateCount: 1,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 				FailureOutput: &junitapi.FailureOutput{
@@ -169,19 +169,38 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 				{when: time.Now(), out: lifecycle01before},
 				{when: time.Now(), out: lifecycle01before},
 			},
-			wasUpdated: false,
+			updateCount: 0,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 			},
 		},
+		{
+			name: "test is skipped when multiple hops were observed",
+			snapshots: []snapshot{
+				{when: time.Now(), out: lifecycle01before},
+				{when: time.Now(), out: lifecycle02updating},
+				{when: time.Now(), out: lifecycle05after},
+				{when: time.Now(), out: lifecycle02updating},
+				{when: time.Now(), out: lifecycle05after},
+				{when: time.Now(), out: lifecycle02updating},
+				{when: time.Now(), out: lifecycle05after},
+			},
+			updateCount: 3,
+			expected: &junitapi.JUnitTestCase{
+				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
+				SkipMessage: &junitapi.SkipMessage{
+					Message: "Cluster updated more than once (3 times)",
+				},
+			},
+		},
 		{
 			name: "update observed when cluster was not updated",
 			snapshots: []snapshot{
 				{when: time.Now(), out: lifecycle01before},
 				{when: time.Now(), out: lifecycle02updating},
 				{when: time.Now(), out: lifecycle01before},
 			},
-			wasUpdated: false,
+			updateCount: 0,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 				// TODO: MCO churn sometimes briefly tricks our code into thinking the cluster is updating, we'll tolerate for
@@ -192,14 +211,13 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 			},
 		},
 		{
-
 			name: "completed update goes back to updating",
 			snapshots: []snapshot{
 				{when: time.Now(), out: lifecycle03controlPlaneNodesUpdated},
 				{when: time.Now(), out: lifecycle05after},
 				{when: time.Now(), out: lifecycle03controlPlaneNodesUpdated},
 			},
-			wasUpdated: true,
+			updateCount: 1,
 			expected: &junitapi.JUnitTestCase{
 				Name: "[sig-cli][OCPFeatureGate:UpgradeStatus] oc adm upgrade status snapshots reflect the cluster upgrade lifecycle",
 				// TODO: MCO churn sometimes briefly tricks our code into thinking the cluster is updating, we'll tolerate for
@@ -223,8 +241,8 @@ func TestMonitor_UpdateLifecycle(t *testing.T) {
 			// Process snapshots into models for the health check to work with
 			_ = m.expectedLayout()
 
-			wasUpdated := func() (bool, error) {
-				return tc.wasUpdated, nil
+			wasUpdated := func() (int, error) {
+				return tc.updateCount, nil
 			}
 
 			result := m.updateLifecycle(wasUpdated)

pkg/monitortests/clusterversionoperator/legacycvomonitortests/operators.go

@@ -401,6 +401,13 @@ func testUpgradeOperatorStateTransitions(events monitorapi.Intervals, clientConf
 			if condition.Type == configv1.OperatorDegraded && condition.Status == configv1.ConditionTrue {
 				return "https://issues.redhat.com/browse/OCPBUGS-39026", nil
 			}
+		case "olm":
+			if condition.Type == configv1.OperatorAvailable &&
+				condition.Status == configv1.ConditionFalse &&
+				(condition.Reason == "OperatorcontrollerDeploymentOperatorControllerControllerManager_Deploying" ||
+					condition.Reason == "CatalogdDeploymentCatalogdControllerManager_Deploying") {
+				return "https://issues.redhat.com/browse/OCPBUGS-62517", nil
+			}
 		case "openshift-apiserver":
 			if condition.Type == configv1.OperatorAvailable && condition.Status == configv1.ConditionFalse &&
 				(condition.Reason == "APIServerDeployment_NoDeployment" ||

pkg/monitortests/testframework/legacytestframeworkmonitortests/monitortest.go renamed to pkg/monitortests/testframework/legacytestframeworkmonitortests/alerts_monitortest.go

@@ -6,7 +6,6 @@ import (
 
 	"github.com/openshift/origin/pkg/monitortestframework"
 
-	"github.com/openshift/origin/pkg/monitortestlibrary/pathologicaleventlibrary"
 	"github.com/sirupsen/logrus"
 
 	"github.com/openshift/origin/pkg/alerts"
@@ -16,37 +15,41 @@ import (
 	"k8s.io/client-go/rest"
 )
 
-type legacyMonitorTests struct {
+const (
+	AlertsMonitorName = "legacy-test-framework-invariants-alerts"
+)
+
+type legacyAlertsMonitorTests struct {
 	adminRESTConfig            *rest.Config
 	duration                   time.Duration
 	recordedResources          monitorapi.ResourcesMap
 	clusterStabilityDuringTest *monitortestframework.ClusterStabilityDuringTest
 }
 
-func NewLegacyTests(info monitortestframework.MonitorTestInitializationInfo) monitortestframework.MonitorTest {
-	return &legacyMonitorTests{clusterStabilityDuringTest: &info.ClusterStabilityDuringTest}
+func NewLegacyAlertsMonitorTests(info monitortestframework.MonitorTestInitializationInfo) monitortestframework.MonitorTest {
+	return &legacyAlertsMonitorTests{clusterStabilityDuringTest: &info.ClusterStabilityDuringTest}
 }
 
-func (w *legacyMonitorTests) PrepareCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
+func (w *legacyAlertsMonitorTests) PrepareCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
 	return nil
 }
 
-func (w *legacyMonitorTests) StartCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
+func (w *legacyAlertsMonitorTests) StartCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
 	w.adminRESTConfig = adminRESTConfig
 	return nil
 }
 
-func (w *legacyMonitorTests) CollectData(ctx context.Context, storageDir string, beginning, end time.Time) (monitorapi.Intervals, []*junitapi.JUnitTestCase, error) {
+func (w *legacyAlertsMonitorTests) CollectData(ctx context.Context, storageDir string, beginning, end time.Time) (monitorapi.Intervals, []*junitapi.JUnitTestCase, error) {
 	w.duration = end.Sub(beginning)
 	return nil, nil, nil
 }
 
-func (w *legacyMonitorTests) ConstructComputedIntervals(ctx context.Context, startingIntervals monitorapi.Intervals, recordedResources monitorapi.ResourcesMap, beginning, end time.Time) (monitorapi.Intervals, error) {
+func (w *legacyAlertsMonitorTests) ConstructComputedIntervals(ctx context.Context, startingIntervals monitorapi.Intervals, recordedResources monitorapi.ResourcesMap, beginning, end time.Time) (monitorapi.Intervals, error) {
 	w.recordedResources = recordedResources
 	return nil, nil
 }
 
-func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.Context, finalIntervals monitorapi.Intervals) ([]*junitapi.JUnitTestCase, error) {
+func (w *legacyAlertsMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.Context, finalIntervals monitorapi.Intervals) ([]*junitapi.JUnitTestCase, error) {
 	jobType, err := platformidentification.GetJobType(context.TODO(), w.adminRESTConfig)
 	if err != nil {
 		// JobType will be nil here, but we want test cases to all fail if this is the case, so we rely on them to nil check
@@ -57,22 +60,20 @@ func (w *legacyMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.C
 
 	isUpgrade := platformidentification.DidUpgradeHappenDuringCollection(finalIntervals, time.Time{}, time.Time{})
 	if isUpgrade {
-		junits = append(junits, pathologicaleventlibrary.TestDuplicatedEventForUpgrade(finalIntervals, w.adminRESTConfig)...)
 		junits = append(junits, testAlerts(finalIntervals, alerts.AllowedAlertsDuringUpgrade, jobType, w.clusterStabilityDuringTest,
 			w.adminRESTConfig, w.duration, w.recordedResources)...)
 	} else {
-		junits = append(junits, pathologicaleventlibrary.TestDuplicatedEventForStableSystem(finalIntervals, w.adminRESTConfig)...)
 		junits = append(junits, testAlerts(finalIntervals, alerts.AllowedAlertsDuringConformance, jobType, w.clusterStabilityDuringTest,
 			w.adminRESTConfig, w.duration, w.recordedResources)...)
 	}
 
 	return junits, nil
 }
 
-func (*legacyMonitorTests) WriteContentToStorage(ctx context.Context, storageDir, timeSuffix string, finalIntervals monitorapi.Intervals, finalResourceState monitorapi.ResourcesMap) error {
+func (*legacyAlertsMonitorTests) WriteContentToStorage(ctx context.Context, storageDir, timeSuffix string, finalIntervals monitorapi.Intervals, finalResourceState monitorapi.ResourcesMap) error {
 	return nil
 }
 
-func (*legacyMonitorTests) Cleanup(ctx context.Context) error {
+func (*legacyAlertsMonitorTests) Cleanup(ctx context.Context) error {
 	return nil
 }

pkg/monitortests/testframework/legacytestframeworkmonitortests/pathological_events.go

@@ -0,0 +1,64 @@
+package legacytestframeworkmonitortests
+
+import (
+	"context"
+	"time"
+
+	"github.com/openshift/origin/pkg/monitortestframework"
+
+	"github.com/openshift/origin/pkg/monitor/monitorapi"
+	"github.com/openshift/origin/pkg/monitortestlibrary/pathologicaleventlibrary"
+	"github.com/openshift/origin/pkg/monitortestlibrary/platformidentification"
+	"github.com/openshift/origin/pkg/test/ginkgo/junitapi"
+	"k8s.io/client-go/rest"
+)
+
+const (
+	PathologicalMonitorName = "legacy-test-framework-invariants-pathological"
+)
+
+type legacyPathologicalMonitorTests struct {
+	adminRESTConfig *rest.Config
+	duration        time.Duration
+}
+
+func NewLegacyPathologicalMonitorTests(info monitortestframework.MonitorTestInitializationInfo) monitortestframework.MonitorTest {
+	return &legacyPathologicalMonitorTests{}
+}
+
+func (w *legacyPathologicalMonitorTests) PrepareCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
+	return nil
+}
+
+func (w *legacyPathologicalMonitorTests) StartCollection(ctx context.Context, adminRESTConfig *rest.Config, recorder monitorapi.RecorderWriter) error {
+	return nil
+}
+
+func (w *legacyPathologicalMonitorTests) CollectData(ctx context.Context, storageDir string, beginning, end time.Time) (monitorapi.Intervals, []*junitapi.JUnitTestCase, error) {
+	w.duration = end.Sub(beginning)
+	return nil, nil, nil
+}
+
+func (w *legacyPathologicalMonitorTests) ConstructComputedIntervals(ctx context.Context, startingIntervals monitorapi.Intervals, recordedResources monitorapi.ResourcesMap, beginning, end time.Time) (monitorapi.Intervals, error) {
+	return nil, nil
+}
+
+func (w *legacyPathologicalMonitorTests) EvaluateTestsFromConstructedIntervals(ctx context.Context, finalIntervals monitorapi.Intervals) ([]*junitapi.JUnitTestCase, error) {
+	junits := []*junitapi.JUnitTestCase{}
+	isUpgrade := platformidentification.DidUpgradeHappenDuringCollection(finalIntervals, time.Time{}, time.Time{})
+	if isUpgrade {
+		junits = append(junits, pathologicaleventlibrary.TestDuplicatedEventForUpgrade(finalIntervals, w.adminRESTConfig)...)
+	} else {
+		junits = append(junits, pathologicaleventlibrary.TestDuplicatedEventForStableSystem(finalIntervals, w.adminRESTConfig)...)
+	}
+
+	return junits, nil
+}
+
+func (*legacyPathologicalMonitorTests) WriteContentToStorage(ctx context.Context, storageDir, timeSuffix string, finalIntervals monitorapi.Intervals, finalResourceState monitorapi.ResourcesMap) error {
+	return nil
+}
+
+func (*legacyPathologicalMonitorTests) Cleanup(ctx context.Context) error {
+	return nil
+}