WIP: OCPEDGE-1486 CP Node replacement test for TNF

go.mod

@@ -53,6 +53,7 @@ require (
 	github.com/lestrrat/go-jsschema v0.0.0-20181205002244-5c81c58ffcc3
 	github.com/lithammer/dedent v1.1.0
 	github.com/mattn/go-sqlite3 v1.14.30
+	github.com/metal3-io/baremetal-operator/apis v0.11.0
 	github.com/metallb/frr-k8s v0.0.15
 	github.com/microsoftgraph/msgraph-sdk-go v1.81.0
 	github.com/onsi/ginkgo/v2 v2.23.3
@@ -78,7 +79,7 @@ require (
 	github.com/spf13/pflag v1.0.6-0.20210604193023-d5e0c0615ace
 	github.com/spf13/viper v1.8.1
 	github.com/stretchr/objx v0.5.2
-	github.com/stretchr/testify v1.10.0
+	github.com/stretchr/testify v1.11.1
 	github.com/tecbiz-ch/nutanix-go-sdk v0.1.15
 	github.com/tidwall/gjson v1.18.0
 	github.com/tidwall/pretty v1.2.0
@@ -100,9 +101,9 @@ require (
 	gopkg.in/src-d/go-git.v4 v4.13.1
 	gopkg.in/yaml.v2 v2.4.0
 	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.33.4
+	k8s.io/api v0.33.5
 	k8s.io/apiextensions-apiserver v0.33.4
-	k8s.io/apimachinery v0.33.4
+	k8s.io/apimachinery v0.33.5
 	k8s.io/apiserver v0.33.4
 	k8s.io/cli-runtime v0.33.4
 	k8s.io/client-go v0.33.4
@@ -398,7 +399,7 @@ require (
 	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.29 // indirect
 	sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.16 // indirect
-	sigs.k8s.io/controller-runtime v0.19.0 // indirect
+	sigs.k8s.io/controller-runtime v0.21.0 // indirect
 	sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect
 	sigs.k8s.io/kube-storage-version-migrator v0.0.6-0.20230721195810-5c8923c5ff96 // indirect
 	sigs.k8s.io/kustomize/api v0.19.0 // indirect

go.sum

@@ -707,6 +707,8 @@ github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWE
 github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y=
 github.com/mattn/go-sqlite3 v1.14.30 h1:bVreufq3EAIG1Quvws73du3/QgdeZ3myglJlrzSYYCY=
 github.com/mattn/go-sqlite3 v1.14.30/go.mod h1:Uh1q+B4BYcTPb+yiD3kU8Ct7aC0hY9fxUwlHK0RXw+Y=
+github.com/metal3-io/baremetal-operator/apis v0.11.0 h1:sYxjnObegWnDyz028m5Rc6gVYxbSLlvjOAqo6Iq1vOE=
+github.com/metal3-io/baremetal-operator/apis v0.11.0/go.mod h1:T0v/wKeJeUfVFeTq1sihv76IZEE2Bn+hqKkh5kGMxA4=
 github.com/metallb/frr-k8s v0.0.15 h1:6M3UGhovX1EFoaSGjrRD7djUAx3w2I+g81FH8OVtHkM=
 github.com/metallb/frr-k8s v0.0.15/go.mod h1:TjrGoAf+v00hYGlI8jUdyDxY5udMAOs2GWwrvLWnA4E=
 github.com/microsoft/kiota-abstractions-go v1.9.3 h1:cqhbqro+VynJ7kObmo7850h3WN2SbvoyhypPn8uJ1SE=
@@ -986,8 +988,8 @@ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/
 github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
 github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
 github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
-github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
-github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
+github.com/stretchr/testify v1.11.1 h1:7s2iGBzp5EwR7/aIZr8ao5+dra3wiQyKjjFuvgVKu7U=
+github.com/stretchr/testify v1.11.1/go.mod h1:wZwfW3scLgRK+23gO65QZefKpKQRnfz6sD981Nm4B6U=
 github.com/subosito/gotenv v1.2.0 h1:Slr1R9HxAlEKefgq5jn9U+DnETlIUa6HfgEzj0g5d7s=
 github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw=
 github.com/tecbiz-ch/nutanix-go-sdk v0.1.15 h1:ZT5I6OFGswvMceujUE10ZXPNnT5UQIW9gAX4FEFK6Ds=
@@ -1592,8 +1594,8 @@ sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.29 h1:qiifAaaBqV3d/EcN9dKJaJI
 sigs.k8s.io/cloud-provider-azure/pkg/azclient v0.0.29/go.mod h1:ZFAt0qF1kR+w8nBVJK56s6CFvLrlosN1i2c+Sxb7LBk=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.16 h1:Fm/Yjv4nXjUtJ90uXKSKwPwaTWYuDFMhDNNOd77PlOg=
 sigs.k8s.io/cloud-provider-azure/pkg/azclient/configloader v0.0.16/go.mod h1:+kl90flu4+WCP6HBGVYbKVQR+5ztDzUNrWJz8rsnvRU=
-sigs.k8s.io/controller-runtime v0.19.0 h1:nWVM7aq+Il2ABxwiCizrVDSlmDcshi9llbaFbC0ji/Q=
-sigs.k8s.io/controller-runtime v0.19.0/go.mod h1:iRmWllt8IlaLjvTTDLhRBXIEtkCK6hwVBJJsYS9Ajf4=
+sigs.k8s.io/controller-runtime v0.21.0 h1:CYfjpEuicjUecRk+KAeyYh+ouUBn4llGyDYytIGcJS8=
+sigs.k8s.io/controller-runtime v0.21.0/go.mod h1:OSg14+F65eWqIu4DceX7k/+QRAbTTvxeQSNSOQpukWM=
 sigs.k8s.io/gateway-api v1.2.1 h1:fZZ/+RyRb+Y5tGkwxFKuYuSRQHu9dZtbjenblleOLHM=
 sigs.k8s.io/gateway-api v1.2.1/go.mod h1:EpNfEXNjiYfUJypf0eZ0P5iXA9ekSGWaS1WgPaM42X0=
 sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE=

pkg/clioptions/clusterdiscovery/cluster.go

@@ -30,6 +30,13 @@ import (
 	"github.com/openshift/origin/test/extended/util/azure"
 )
 
+// HypervisorConfig contains configuration for hypervisor-based recovery operations
+type HypervisorConfig struct {
+	HypervisorIP   string `json:"hypervisorIP"`
+	SSHUser        string `json:"sshUser"`
+	PrivateKeyPath string `json:"privateKeyPath"`
+}
+
 type ClusterConfiguration struct {
 	ProviderName string `json:"type"`
 
@@ -76,6 +83,9 @@ type ClusterConfiguration struct {
 	// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
 	HasNoOptionalCapabilities bool
 
+	// HypervisorConfig contains SSH configuration for hypervisor-based recovery operations
+	HypervisorConfig *HypervisorConfig
+
 	// APIGroups contains the set of API groups available in the cluster
 	APIGroups sets.Set[string] `json:"-"`
 	// EnabledFeatureGates contains the set of enabled feature gates in the cluster

pkg/cmd/openshift-tests/run/flags.go

@@ -1,6 +1,8 @@
 package run
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
@@ -28,9 +30,6 @@ type RunSuiteFlags struct {
 	ToImage      string
 	TestOptions  []string
 
-	// Shared by initialization code
-	config *clusterdiscovery.ClusterConfiguration
-
 	genericclioptions.IOStreams
 }
 
@@ -84,7 +83,7 @@ func (f *RunSuiteFlags) ToOptions(args []string, availableSuites []*testginkgo.T
 	// shallow copy to mutate
 	ginkgoOptions := f.GinkgoRunSuiteOptions
 
-	providerConfig, err := f.SuiteWithKubeTestInitializationPreSuite()
+	clusterConfig, err := f.SuiteWithKubeTestInitializationPreSuite()
 	if err != nil {
 		return nil, err
 	}
@@ -95,13 +94,39 @@ func (f *RunSuiteFlags) ToOptions(args []string, availableSuites []*testginkgo.T
 		return nil, err
 	}
 
+	// Parse hypervisor configuration if provided and set it in environment for test context
+	if f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON != "" {
+		// Validate the JSON format
+		var hypervisorConfig clusterdiscovery.HypervisorConfig
+		if err := json.Unmarshal([]byte(f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON), &hypervisorConfig); err != nil {
+			return nil, fmt.Errorf("failed to parse hypervisor configuration JSON: %v", err)
+		}
+
+		// Validate required fields
+		if hypervisorConfig.HypervisorIP == "" {
+			return nil, fmt.Errorf("hypervisorIP is required in hypervisor configuration")
+		}
+		if hypervisorConfig.SSHUser == "" {
+			return nil, fmt.Errorf("sshUser is required in hypervisor configuration")
+		}
+		if hypervisorConfig.PrivateKeyPath == "" {
+			return nil, fmt.Errorf("privateKey is required in hypervisor configuration")
+		}
+
+		// Set the hypervisor configuration in the cluster config
+		clusterConfig.HypervisorConfig = &hypervisorConfig
+
+		// Also set it in environment for test context access
+		os.Setenv("HYPERVISOR_CONFIG", f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON)
+	}
+
 	o := &RunSuiteOptions{
 		GinkgoRunSuiteOptions: ginkgoOptions,
 		Suite:                 suite,
 		Extension:             internalExtension,
-		ClusterConfig:         providerConfig,
+		ClusterConfig:         clusterConfig,
 		FromRepository:        f.FromRepository,
-		CloudProviderJSON:     providerConfig.ToJSONString(),
+		CloudProviderJSON:     clusterConfig.ToJSONString(),
 		CloseFn:               closeFn,
 		IOStreams:             f.IOStreams,
 	}

pkg/cmd/openshift-tests/run/options.go

@@ -31,6 +31,11 @@ type RunSuiteOptions struct {
 	CloseFn iooptions.CloseFunc
 	genericclioptions.IOStreams
 
+	// HypervisorConfig contains SSH configuration for hypervisor-based recovery operations
+	// If set, will run recovery tests that require the hypervisor-based recovery, such as
+	// the node replacement test in the two_node recovery suite.
+	HypervisorConfig *clusterdiscovery.HypervisorConfig
+
 	// ClusterConfig contains cluster-specific configuration for filtering tests
 	ClusterConfig *clusterdiscovery.ClusterConfiguration
 

pkg/test/filters/cluster_state.go

@@ -67,6 +67,10 @@ func NewClusterStateFilter(config *clusterdiscovery.ClusterConfiguration) *Clust
 		skips = append(skips, "[Skipped:NoOptionalCapabilities]")
 	}
 
+	if config.HypervisorConfig == nil {
+		skips = append(skips, "[Requires:HypervisorSSHConfig]")
+	}
+
 	logrus.WithField("skips", skips).Info("Generated skips for cluster state")
 
 	return &ClusterStateFilter{

pkg/test/ginkgo/cmd_runsuite.go

@@ -96,6 +96,9 @@ type GinkgoRunSuiteOptions struct {
 
 	// RetryStrategy controls retry behavior and final outcome decisions
 	RetryStrategy RetryStrategy
+
+	// WithHypervisorConfigJSON contains JSON configuration for hypervisor-based recovery operations
+	WithHypervisorConfigJSON string
 }
 
 func NewGinkgoRunSuiteOptions(streams genericclioptions.IOStreams) *GinkgoRunSuiteOptions {
@@ -133,6 +136,7 @@ func (o *GinkgoRunSuiteOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.ShardStrategy, "shard-strategy", o.ShardStrategy, "Which strategy to use for sharding (hash)")
 	availableStrategies := getAvailableRetryStrategies()
 	flags.Var(newRetryStrategyFlag(&o.RetryStrategy), "retry-strategy", fmt.Sprintf("Test retry strategy (available: %s, default: %s)", strings.Join(availableStrategies, ", "), defaultRetryStrategy))
+	flags.StringVar(&o.WithHypervisorConfigJSON, "with-hypervisor-json", os.Getenv("HYPERVISOR_CONFIG"), "JSON configuration for hypervisor-based recovery operations. Must contain hypervisorIP, sshUser, and privateKeyPath fields.")
 }
 
 func (o *GinkgoRunSuiteOptions) Validate() error {

test/extended/testdata/two_node/baremetalhost-template.yaml

@@ -0,0 +1,27 @@
+apiVersion: metal3.io/v1alpha1
+kind: BareMetalHost
+metadata:
+  finalizers:
+  - baremetalhost.metal3.io
+  labels:
+    installer.openshift.io/role: control-plane
+  name: {NAME}
+  namespace: openshift-machine-api
+spec:
+  architecture: x86_64
+  automatedCleaningMode: metadata
+  bmc:
+    address: redfish+https://{IP}:8000/redfish/v1/Systems/{UUID}
+    credentialsName: {CREDENTIALS_NAME}
+    disableCertificateVerification: true
+  bootMACAddress: {BOOT_MAC_ADDRESS}
+  bootMode: UEFI
+  customDeploy:
+    method: install_coreos
+  hardwareProfile: unknown
+  online: true
+  rootDeviceHints:
+    deviceName: /dev/sda
+  userData:
+    name: master-user-data-managed
+    namespace: openshift-machine-api

test/extended/testdata/two_node/machine-template.yaml

@@ -0,0 +1,30 @@
+apiVersion: machine.openshift.io/v1beta1
+kind: Machine
+metadata:
+  annotations:
+    metal3.io/BareMetalHost: openshift-machine-api/{NODE_NAME}
+  finalizers:
+  - machine.machine.openshift.io
+  labels:
+    machine.openshift.io/cluster-api-cluster: ostest-{MACHINE_HASH}
+    machine.openshift.io/cluster-api-machine-role: master
+    machine.openshift.io/cluster-api-machine-type: master
+  name: {MACHINE_NAME}
+  namespace: openshift-machine-api
+spec:
+  authoritativeAPI: MachineAPI
+  metadata: {}
+  providerSpec:
+    value:
+      apiVersion: baremetal.cluster.k8s.io/v1alpha1
+      customDeploy:
+        method: install_coreos
+      hostSelector: {}
+      image:
+        checksum: ""
+        url: ""
+      kind: BareMetalMachineProviderSpec
+      metadata:
+        creationTimestamp: null
+      userData:
+        name: master-user-data-managed

test/extended/two_node/arbiter_topology.go

@@ -37,7 +37,7 @@ var expectedPods = map[string]int{
 
 var _ = g.Describe("[sig-node][apigroup:config.openshift.io][OCPFeatureGate:HighlyAvailableArbiter] expected Master and Arbiter node counts", func() {
 	defer g.GinkgoRecover()
-	oc := exutil.NewCLIWithoutNamespace("")
+	oc := createCLI(nonAdmin)
 
 	g.BeforeEach(func() {
 		skipIfNotTopology(oc, v1.HighlyAvailableArbiterMode)
@@ -51,15 +51,11 @@ var _ = g.Describe("[sig-node][apigroup:config.openshift.io][OCPFeatureGate:High
 			expectedMasterNodes  = 2
 			expectedArbiterNodes = 1
 		)
-		masterNodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
-			LabelSelector: labelNodeRoleMaster,
-		})
+		masterNodes, err := getNodes(oc, labelNodeRoleControlPlane)
 		o.Expect(err).To(o.BeNil(), "Expected to retrieve Master nodes without error")
 		o.Expect(len(masterNodes.Items)).To(o.Equal(expectedMasterNodes))
 
-		arbiterNodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
-			LabelSelector: labelNodeRoleArbiter,
-		})
+		arbiterNodes, err := getNodes(oc, labelNodeRoleArbiter)
 		o.Expect(err).To(o.BeNil(), "Expected to retrieve Arbiter nodes without error")
 		o.Expect(len(arbiterNodes.Items)).To(o.Equal(expectedArbiterNodes))
 	})
@@ -68,16 +64,14 @@ var _ = g.Describe("[sig-node][apigroup:config.openshift.io][OCPFeatureGate:High
 var _ = g.Describe("[sig-node][apigroup:config.openshift.io][OCPFeatureGate:HighlyAvailableArbiter] required pods on the Arbiter node", func() {
 	defer g.GinkgoRecover()
 
-	oc := exutil.NewCLIWithoutNamespace("")
+	oc := createCLI(nonAdmin)
 
 	g.BeforeEach(func() {
 		skipIfNotTopology(oc, v1.HighlyAvailableArbiterMode)
 	})
 	g.It("Should verify that the correct number of pods are running on the Arbiter node", func() {
 		g.By("Retrieving the Arbiter node name")
-		nodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
-			LabelSelector: labelNodeRoleArbiter,
-		})
+		nodes, err := getNodes(oc, labelNodeRoleArbiter)
 		o.Expect(err).To(o.BeNil(), "Expected to retrieve nodes without error")
 		o.Expect(len(nodes.Items)).To(o.Equal(1))
 		g.By("by comparing pod counts")
@@ -154,7 +148,7 @@ var _ = g.Describe("[sig-apps][apigroup:apps.openshift.io][OCPFeatureGate:Highly
 		ctx := context.Background()
 		g.By("Retrieving Master nodes")
 		masterNodes, err := oc.AdminKubeClient().CoreV1().Nodes().List(ctx, metav1.ListOptions{
-			LabelSelector: labelNodeRoleMaster,
+			LabelSelector: labelNodeRoleControlPlane,
 		})
 		o.Expect(err).To(o.BeNil(), "Expected to retrieve Master nodes without error")
 		o.Expect(len(masterNodes.Items)).To(o.Equal(2), "Expected to find two Master nodes")

test/extended/two_node/common.go

@@ -1,18 +1,26 @@
 package two_node
 
 import (
+	"context"
 	"fmt"
 
 	v1 "github.com/openshift/api/config/v1"
 	exutil "github.com/openshift/origin/test/extended/util"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	e2eskipper "k8s.io/kubernetes/test/e2e/framework/skipper"
 )
 
 const (
-	labelNodeRoleMaster       = "node-role.kubernetes.io/master"
+	// Node filtering constants
+	allNodes                  = ""
 	labelNodeRoleControlPlane = "node-role.kubernetes.io/control-plane"
 	labelNodeRoleWorker       = "node-role.kubernetes.io/worker"
 	labelNodeRoleArbiter      = "node-role.kubernetes.io/arbiter"
+
+	// CLI privilege levels
+	nonAdmin = false
+	admin    = true
 )
 
 func skipIfNotTopology(oc *exutil.CLI, wanted v1.TopologyMode) {
@@ -42,3 +50,23 @@ func isClusterOperatorDegraded(operator *v1.ClusterOperator) bool {
 	}
 	return false
 }
+
+// createCLI creates a new CLI instance with optional admin privileges
+func createCLI(requireAdmin bool) *exutil.CLI {
+	if requireAdmin {
+		return exutil.NewCLIWithoutNamespace("").AsAdmin()
+	}
+	return exutil.NewCLIWithoutNamespace("")
+}
+
+// getNodes returns a list of nodes, optionally filtered by role label
+// When roleLabel is allNodes (""), returns all nodes
+// When roleLabel is specified, filters nodes by that label
+func getNodes(oc *exutil.CLI, roleLabel string) (*corev1.NodeList, error) {
+	if roleLabel == "" {
+		return oc.AdminKubeClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{})
+	}
+	return oc.AdminKubeClient().CoreV1().Nodes().List(context.Background(), metav1.ListOptions{
+		LabelSelector: roleLabel,
+	})
+}