WIP: Core libraries for two node disruptive tests.

pkg/clioptions/clusterdiscovery/cluster.go

@@ -30,6 +30,13 @@ import (
 	"github.com/openshift/origin/test/extended/util/azure"
 )
 
+// HypervisorConfig contains configuration for hypervisor-based recovery operations
+type HypervisorConfig struct {
+	HypervisorIP   string `json:"hypervisorIP"`
+	SSHUser        string `json:"sshUser"`
+	PrivateKeyPath string `json:"privateKeyPath"`
+}
+
 type ClusterConfiguration struct {
 	ProviderName string `json:"type"`
 
@@ -76,6 +83,9 @@ type ClusterConfiguration struct {
 	// IsNoOptionalCapabilities indicates the cluster has no optional capabilities enabled
 	HasNoOptionalCapabilities bool
 
+	// HypervisorConfig contains SSH configuration for hypervisor-based recovery operations
+	HypervisorConfig *HypervisorConfig
+
 	// APIGroups contains the set of API groups available in the cluster
 	APIGroups sets.Set[string] `json:"-"`
 	// EnabledFeatureGates contains the set of enabled feature gates in the cluster

pkg/cmd/openshift-tests/run/flags.go

@@ -1,6 +1,8 @@
 package run
 
 import (
+	"encoding/json"
+	"fmt"
 	"os"
 
 	"github.com/openshift-eng/openshift-tests-extension/pkg/extension"
@@ -28,9 +30,6 @@ type RunSuiteFlags struct {
 	ToImage      string
 	TestOptions  []string
 
-	// Shared by initialization code
-	config *clusterdiscovery.ClusterConfiguration
-
 	genericclioptions.IOStreams
 }
 
@@ -84,7 +83,7 @@ func (f *RunSuiteFlags) ToOptions(args []string, availableSuites []*testginkgo.T
 	// shallow copy to mutate
 	ginkgoOptions := f.GinkgoRunSuiteOptions
 
-	providerConfig, err := f.SuiteWithKubeTestInitializationPreSuite()
+	clusterConfig, err := f.SuiteWithKubeTestInitializationPreSuite()
 	if err != nil {
 		return nil, err
 	}
@@ -95,13 +94,39 @@ func (f *RunSuiteFlags) ToOptions(args []string, availableSuites []*testginkgo.T
 		return nil, err
 	}
 
+	// Parse hypervisor configuration if provided and set it in environment for test context
+	if f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON != "" {
+		// Validate the JSON format
+		var hypervisorConfig clusterdiscovery.HypervisorConfig
+		if err := json.Unmarshal([]byte(f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON), &hypervisorConfig); err != nil {
+			return nil, fmt.Errorf("failed to parse hypervisor configuration JSON: %v", err)
+		}
+
+		// Validate required fields
+		if hypervisorConfig.HypervisorIP == "" {
+			return nil, fmt.Errorf("hypervisorIP is required in hypervisor configuration")
+		}
+		if hypervisorConfig.SSHUser == "" {
+			return nil, fmt.Errorf("sshUser is required in hypervisor configuration")
+		}
+		if hypervisorConfig.PrivateKeyPath == "" {
+			return nil, fmt.Errorf("privateKey is required in hypervisor configuration")
+		}
+
+		// Set the hypervisor configuration in the cluster config
+		clusterConfig.HypervisorConfig = &hypervisorConfig
+
+		// Also set it in environment for test context access
+		os.Setenv("HYPERVISOR_CONFIG", f.GinkgoRunSuiteOptions.WithHypervisorConfigJSON)
+	}
+
 	o := &RunSuiteOptions{
 		GinkgoRunSuiteOptions: ginkgoOptions,
 		Suite:                 suite,
 		Extension:             internalExtension,
-		ClusterConfig:         providerConfig,
+		ClusterConfig:         clusterConfig,
 		FromRepository:        f.FromRepository,
-		CloudProviderJSON:     providerConfig.ToJSONString(),
+		CloudProviderJSON:     clusterConfig.ToJSONString(),
 		CloseFn:               closeFn,
 		IOStreams:             f.IOStreams,
 	}

pkg/cmd/openshift-tests/run/options.go

@@ -31,6 +31,11 @@ type RunSuiteOptions struct {
 	CloseFn iooptions.CloseFunc
 	genericclioptions.IOStreams
 
+	// HypervisorConfig contains SSH configuration for hypervisor-based recovery operations
+	// If set, will run recovery tests that require the hypervisor-based recovery, such as
+	// the node replacement test in the two_node recovery suite.
+	HypervisorConfig *clusterdiscovery.HypervisorConfig
+
 	// ClusterConfig contains cluster-specific configuration for filtering tests
 	ClusterConfig *clusterdiscovery.ClusterConfiguration
 

pkg/test/filters/cluster_state.go

@@ -67,6 +67,10 @@ func NewClusterStateFilter(config *clusterdiscovery.ClusterConfiguration) *Clust
 		skips = append(skips, "[Skipped:NoOptionalCapabilities]")
 	}
 
+	if config.HypervisorConfig == nil {
+		skips = append(skips, "[Requires:HypervisorSSHConfig]")
+	}
+
 	logrus.WithField("skips", skips).Info("Generated skips for cluster state")
 
 	return &ClusterStateFilter{

pkg/test/ginkgo/cmd_runsuite.go

@@ -96,6 +96,9 @@ type GinkgoRunSuiteOptions struct {
 
 	// RetryStrategy controls retry behavior and final outcome decisions
 	RetryStrategy RetryStrategy
+
+	// WithHypervisorConfigJSON contains JSON configuration for hypervisor-based recovery operations
+	WithHypervisorConfigJSON string
 }
 
 func NewGinkgoRunSuiteOptions(streams genericclioptions.IOStreams) *GinkgoRunSuiteOptions {
@@ -133,6 +136,7 @@ func (o *GinkgoRunSuiteOptions) BindFlags(flags *pflag.FlagSet) {
 	flags.StringVar(&o.ShardStrategy, "shard-strategy", o.ShardStrategy, "Which strategy to use for sharding (hash)")
 	availableStrategies := getAvailableRetryStrategies()
 	flags.Var(newRetryStrategyFlag(&o.RetryStrategy), "retry-strategy", fmt.Sprintf("Test retry strategy (available: %s, default: %s)", strings.Join(availableStrategies, ", "), defaultRetryStrategy))
+	flags.StringVar(&o.WithHypervisorConfigJSON, "with-hypervisor-json", os.Getenv("HYPERVISOR_CONFIG"), "JSON configuration for hypervisor-based recovery operations. Must contain hypervisorIP, sshUser, and privateKeyPath fields.")
 }
 
 func (o *GinkgoRunSuiteOptions) Validate() error {

test/extended/two_node/utils/hypervisor.go

@@ -0,0 +1,125 @@
+// Package utils provides hypervisor configuration and validation utilities for two-node cluster testing.
+//
+// Tests requiring hypervisor access should include the [Requires:HypervisorSSHConfig] annotation.
+//
+// Configuration can be provided via command-line flag or environment variable:
+//
+//	openshift-tests run openshift/two-node --with-hypervisor-json='{
+//	  "IP": "192.168.111.1",
+//	  "User": "root",
+//	  "privateKeyPath": "/path/to/private/key"
+//	}'
+//
+// Or:
+//
+//	export HYPERVISOR_CONFIG='{"IP":"192.168.111.1","User":"root","privateKeyPath":"/path/to/key"}'
+//	openshift-tests run openshift/two-node
+//
+// Usage example:
+//
+//	if !exutil.HasHypervisorConfig() {
+//	    utils.PrintHypervisorConfigUsage()
+//	    return
+//	}
+//	config := exutil.GetHypervisorConfig()
+//	utils.VerifyHypervisorConnectivity(&config, knownHostsPath)
+package utils
+
+import (
+	"fmt"
+	"strings"
+
+	g "github.com/onsi/ginkgo/v2"
+	"k8s.io/klog/v2"
+)
+
+// PrintHypervisorConfigUsage prints usage instructions for configuring hypervisor SSH access.
+// Call this when HasHypervisorConfig() returns false to provide configuration guidance.
+func PrintHypervisorConfigUsage() {
+	usageMessage := `
+================================================================================
+Two-Node Test Suite - Hypervisor Configuration Required
+================================================================================
+
+This test requires hypervisor SSH configuration to manage virtual machines
+and perform node operations. The [Requires:HypervisorSSHConfig] annotation
+indicates this requirement.
+
+CONFIGURATION METHODS:
+
+1. Command-Line Flag (recommended for interactive testing):
+
+   openshift-tests run openshift/two-node --with-hypervisor-json='{
+     "IP": "192.168.111.1",
+     "User": "root",
+     "privateKeyPath": "/path/to/private/key"
+   }'
+
+2. Environment Variable (recommended for CI/CD):
+
+   export HYPERVISOR_CONFIG='{"IP":"192.168.111.1","User":"root","privateKeyPath":"/path/to/key"}'
+   openshift-tests run openshift/two-node
+
+CONFIGURATION FIELDS:
+
+- IP: IP address or hostname of the hypervisor
+- User: SSH username (typically "root")
+- privateKeyPath: Absolute path to SSH private key file
+
+TROUBLESHOOTING:
+
+If configuration fails:
+1. Verify JSON syntax is valid
+2. Check that the private key file exists
+3. Test SSH connectivity: ssh -i <privateKeyPath> <User>@<IP>
+4. Verify virsh is available: ssh <User>@<IP> 'virsh version'
+
+================================================================================
+`
+	g.GinkgoT().Logf(usageMessage)
+}
+
+// VerifyHypervisorAvailability verifies SSH connectivity to the hypervisor and checks
+// that virsh and libvirt are available.
+func VerifyHypervisorAvailability(sshConfig *SSHConfig, knownHostsPath string) error {
+	klog.V(2).Infof("Verifying hypervisor connectivity to %s@%s", sshConfig.User, sshConfig.IP)
+
+	// Test basic SSH connectivity
+	output, _, err := VerifyConnectivity(sshConfig, knownHostsPath)
+	if err != nil {
+		klog.ErrorS(err, "Failed to establish SSH connection to hypervisor",
+			"user", sshConfig.User,
+			"host", sshConfig.IP,
+			"output", output)
+		klog.ErrorS(nil, "Ensure the hypervisor is accessible and SSH key is correct")
+		return fmt.Errorf("failed to establish SSH connection to hypervisor %s@%s: %w", sshConfig.User, sshConfig.IP, err)
+	}
+	klog.V(2).Infof("SSH connectivity verified: %s", strings.TrimSpace(output))
+
+	// Test virsh availability and basic functionality
+	output, err = VerifyVirsh(sshConfig, knownHostsPath)
+	if err != nil {
+		klog.ErrorS(err, "virsh is not available or not working on hypervisor",
+			"user", sshConfig.User,
+			"host", sshConfig.IP,
+			"output", output)
+		klog.ErrorS(nil, "Ensure libvirt and virsh are installed on the hypervisor")
+		return fmt.Errorf("virsh is not available or not working on hypervisor %s@%s: %w", sshConfig.User, sshConfig.IP, err)
+	}
+	klog.V(2).Infof("virsh availability verified: %s", strings.TrimSpace(output))
+
+	// Test libvirt connection by listing VMs
+	output, err = VirshListAllVMs(sshConfig, knownHostsPath)
+	if err != nil {
+		klog.ErrorS(err, "Failed to connect to libvirt on hypervisor",
+			"user", sshConfig.User,
+			"host", sshConfig.IP,
+			"output", output)
+		klog.ErrorS(nil, "Ensure libvirtd service is running and user has access")
+		return fmt.Errorf("failed to connect to libvirt on hypervisor %s@%s: %w", sshConfig.User, sshConfig.IP, err)
+	}
+	klog.V(2).Infof("libvirt connection verified, found VMs: %s", strings.TrimSpace(output))
+
+	klog.V(2).Infof("Hypervisor connectivity verification completed successfully")
+	return nil
+}