extended resource backed by DRA: test

Author: yliaog

test/compatibility_lifecycle/reference/versioned_feature_list.yaml

@@ -479,6 +479,12 @@
     lockToDefault: false
     preRelease: Alpha
     version: "1.33"
+- name: DRAExtendedResource
+  versionedSpecs:
+  - default: false
+    lockToDefault: false
+    preRelease: Alpha
+    version: "1.34"
 - name: DRAPartitionableDevices
   versionedSpecs:
   - default: false

test/e2e/dra/dra.go

@@ -1806,7 +1806,6 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
 			createdTaint := b.Create(ctx, taint)
 			taint = createdTaint[0].(*resourcealphaapi.DeviceTaintRule)
 			gomega.Expect(*taint).Should(gomega.HaveField("Spec.Taint.TimeAdded.Time", gomega.BeTemporally("~", time.Now(), time.Minute /* allow for some clock drift and delays */)))
-
 			framework.ExpectNoError(e2epod.WaitForPodTerminatingInNamespaceTimeout(ctx, f.ClientSet, pod.Name, f.Namespace.Name, f.Timeouts.PodStart))
 			pod, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod.Name, metav1.GetOptions{})
 			framework.ExpectNoError(err, "get pod")
@@ -1820,6 +1819,177 @@ var _ = framework.SIGDescribe("node")(framework.WithLabel("DRA"), func() {
 		})
 	})
 
+	framework.Context(f.WithFeatureGate(features.DRAExtendedResource), func() {
+		nodes := drautils.NewNodes(f, 1, 1)
+		driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(10, false))
+		b := drautils.NewBuilder(f, driver)
+		b.UseExtendedResourceName = true
+
+		ginkgo.It("must run a pod with extended resource with one container one resource", func(ctx context.Context) {
+			pod := b.Pod()
+			res := v1.ResourceList{}
+			res[v1.ResourceName(drautils.ExtendedResourceName(0))] = resource.MustParse("1")
+			pod.Spec.Containers[0].Resources.Requests = res
+			pod.Spec.Containers[0].Resources.Limits = res
+
+			b.Create(ctx, pod)
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err, "start pod")
+			containerEnv := []string{
+				"container_0_request_0", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
+		})
+
+		ginkgo.It("must run a pod with extended resource with one container three resources", func(ctx context.Context) {
+			pod := b.Pod()
+			res := v1.ResourceList{}
+			for i := range 3 {
+				res[v1.ResourceName(drautils.ExtendedResourceName(i))] = resource.MustParse("1")
+			}
+			pod.Spec.Containers[0].Resources.Requests = res
+			pod.Spec.Containers[0].Resources.Limits = res
+
+			b.Create(ctx, pod)
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err, "start pod")
+			containerEnv := []string{
+				"container_0_request_0", "true",
+				"container_0_request_1", "true",
+				"container_0_request_2", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
+		})
+		ginkgo.It("must run a pod with extended resource with three containers one resource each", func(ctx context.Context) {
+			pod := b.Pod()
+			pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
+			pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
+			pod.Spec.Containers[0].Name = "container0"
+			pod.Spec.Containers[1].Name = "container1"
+			pod.Spec.Containers[2].Name = "container2"
+
+			for i := range 3 {
+				res := v1.ResourceList{}
+				res[v1.ResourceName(drautils.ExtendedResourceName(i))] = resource.MustParse("1")
+				pod.Spec.Containers[i].Resources.Requests = res
+				pod.Spec.Containers[i].Resources.Limits = res
+			}
+
+			b.Create(ctx, pod)
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err, "start pod")
+			for i := range 3 {
+				containerEnv := []string{
+					fmt.Sprintf("container_%d_request_0", i), "true",
+				}
+				drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[i].Name, false, containerEnv...)
+			}
+		})
+		ginkgo.It("must run a pod with extended resource with three containers multiple resources each", func(ctx context.Context) {
+			pod := b.Pod()
+			pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
+			pod.Spec.Containers = append(pod.Spec.Containers, *pod.Spec.Containers[0].DeepCopy())
+			pod.Spec.Containers[0].Name = "container0"
+			pod.Spec.Containers[1].Name = "container1"
+			pod.Spec.Containers[2].Name = "container2"
+
+			res := v1.ResourceList{}
+			res[v1.ResourceName(drautils.ExtendedResourceName(0))] = resource.MustParse("1")
+			pod.Spec.Containers[0].Resources.Requests = res
+			pod.Spec.Containers[0].Resources.Limits = res
+			res = v1.ResourceList{}
+			res[v1.ResourceName(drautils.ExtendedResourceName(1))] = resource.MustParse("1")
+			res[v1.ResourceName(drautils.ExtendedResourceName(2))] = resource.MustParse("1")
+			pod.Spec.Containers[1].Resources.Requests = res
+			pod.Spec.Containers[1].Resources.Limits = res
+			res = v1.ResourceList{}
+			res[v1.ResourceName(drautils.ExtendedResourceName(3))] = resource.MustParse("1")
+			res[v1.ResourceName(drautils.ExtendedResourceName(4))] = resource.MustParse("1")
+			res[v1.ResourceName(drautils.ExtendedResourceName(5))] = resource.MustParse("1")
+			pod.Spec.Containers[2].Resources.Requests = res
+			pod.Spec.Containers[2].Resources.Limits = res
+
+			b.Create(ctx, pod)
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod)
+			framework.ExpectNoError(err, "start pod")
+			containerEnv := []string{
+				"container_0_request_0", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[0].Name, false, containerEnv...)
+			containerEnv = []string{
+				"container_1_request_0", "true",
+				"container_1_request_1", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[1].Name, false, containerEnv...)
+			containerEnv = []string{
+				"container_2_request_0", "true",
+				"container_2_request_1", "true",
+				"container_2_request_2", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, pod, pod.Spec.Containers[2].Name, false, containerEnv...)
+		})
+	})
+
+	framework.Context(f.WithFeatureGate(features.DRAExtendedResource), func() {
+		nodes := drautils.NewNodes(f, 2, 2)
+		nodes.NumReservedNodes = 1
+		driver := drautils.NewDriver(f, nodes, drautils.NetworkResources(2, false))
+		b := drautils.NewBuilder(f, driver)
+		b.UseExtendedResourceName = true
+
+		// This test needs the entire test cluster for itself, one node in the cluster
+		// is deployed device plugin for the test, therefore it is marked as serial.
+		// The test runs two pods, one pod request extended resource backed by DRA,
+		// the other pod requests extended resource by device plugin.
+		f.It("must run pods with extended resource on dra nodes and device plugin nodes", f.WithSerial(), func(ctx context.Context) {
+			extendedResourceName := deployDevicePlugin(ctx, f, nodes.ExtraNodeNames)
+			// drautils.ExtendedResourceName(-1) must be the same as the returned extendedResourceName
+			// drautils.ExtendedResourceName(-1) is used for DRA drivers
+			// extendedResourceName is used for device plugin.
+			gomega.Expect(string(extendedResourceName)).To(gomega.Equal(drautils.ExtendedResourceName(-1)))
+
+			pod1 := b.Pod()
+			res := v1.ResourceList{}
+			res[v1.ResourceName(drautils.ExtendedResourceName(-1))] = resource.MustParse("2")
+			pod1.Spec.Containers[0].Resources.Requests = res
+			pod1.Spec.Containers[0].Resources.Limits = res
+			b.Create(ctx, pod1)
+
+			pod2 := b.Pod()
+			pod2.Spec.Containers[0].Resources.Requests = res
+			pod2.Spec.Containers[0].Resources.Limits = res
+			b.Create(ctx, pod2)
+
+			err := e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod1)
+			framework.ExpectNoError(err, "start pod1")
+			err = e2epod.WaitForPodRunningInNamespace(ctx, f.ClientSet, pod2)
+			framework.ExpectNoError(err, "start pod2")
+
+			scheduledPod1, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod1.Name, metav1.GetOptions{})
+			gomega.Expect(scheduledPod1).ToNot(gomega.BeNil())
+			framework.ExpectNoError(err)
+
+			scheduledPod2, err := f.ClientSet.CoreV1().Pods(f.Namespace.Name).Get(ctx, pod2.Name, metav1.GetOptions{})
+			gomega.Expect(scheduledPod2).ToNot(gomega.BeNil())
+			framework.ExpectNoError(err)
+
+			draPod := scheduledPod1
+			devicePluginPod := scheduledPod2
+			if scheduledPod1.Spec.NodeName == nodes.ExtraNodeNames[0] {
+				draPod = scheduledPod2
+				devicePluginPod = scheduledPod1
+			}
+
+			gomega.Expect(devicePluginPod.Spec.NodeName).To(gomega.Equal(nodes.ExtraNodeNames[0]))
+			gomega.Expect(devicePluginPod.Status.ExtendedResourceClaimStatus).To(gomega.BeNil())
+			gomega.Expect(draPod.Spec.NodeName).To(gomega.Equal(nodes.NodeNames[0]))
+			containerEnv := []string{
+				"container_0_request_0", "true",
+			}
+			drautils.TestContainerEnv(ctx, f, draPod, draPod.Spec.Containers[0].Name, false, containerEnv...)
+		})
+	})
+
 	ginkgo.Context("ResourceSlice Controller", func() {
 		// This is a stress test for creating many large slices.
 		// Each slice is as large as API limits allow.

test/e2e/dra/utils/builder.go

@@ -22,6 +22,7 @@ import (
 	"fmt"
 	"regexp"
 	"sort"
+	"strconv"
 	"strings"
 	"time"
 
@@ -47,11 +48,27 @@ import (
 	"k8s.io/utils/ptr"
 )
 
+// ExtendedResourceName returns hard coded extended resource name with a variable
+// suffix from the input integer when it's greater than or equal to 0.
+// "example.com/resource" is not special, any valid extended resource name can be used
+// instead, except when using example device plugin in the test, which hard coded it,
+// see test/e2e/dra/deploy_device_plugin.go.
+// i == -1 is special, the extended resource name has no extra suffix, it is
+// used in the test where a cluster has both DRA driver and device plugin.
+func ExtendedResourceName(i int) string {
+	suffix := ""
+	if i >= 0 {
+		suffix = strconv.Itoa(i)
+	}
+	return "example.com/resource" + suffix
+}
+
 // Builder contains a running counter to make objects unique within thir
 // namespace.
 type Builder struct {
-	f      *framework.Framework
-	driver *Driver
+	f                       *framework.Framework
+	driver                  *Driver
+	UseExtendedResourceName bool
 
 	podCounter      int
 	claimCounter    int
@@ -65,12 +82,26 @@ func (b *Builder) ClassName() string {
 
 // Class returns the device Class that the builder's other objects
 // reference.
-func (b *Builder) Class() *resourceapi.DeviceClass {
+// The input i is used to pick the extended resource name whose suffix has the
+// same i for the device class.
+// i == -1 is special, the extended resource name has no extra suffix, it is
+// used in the test where a cluster has both DRA driver and device plugin.
+func (b *Builder) Class(i int) *resourceapi.DeviceClass {
+	ern := ExtendedResourceName(i)
+	name := b.ClassName()
+	if i >= 0 {
+		name = b.ClassName() + strconv.Itoa(i)
+	}
 	class := &resourceapi.DeviceClass{
 		ObjectMeta: metav1.ObjectMeta{
-			Name: b.ClassName(),
+			Name: name,
 		},
 	}
+	if b.UseExtendedResourceName {
+		class.Spec = resourceapi.DeviceClassSpec{
+			ExtendedResourceName: &ern,
+		}
+	}
 	class.Spec.Selectors = []resourceapi.DeviceSelector{{
 		CEL: &resourceapi.CELDeviceSelector{
 			Expression: fmt.Sprintf(`device.driver == "%s"`, b.driver.Name),
@@ -415,7 +446,7 @@ func TestContainerEnv(ctx context.Context, f *framework.Framework, pod *v1.Pod,
 		gomega.Expect(actualEnv).To(gomega.Equal(expectEnv), fmt.Sprintf("container %s env output:\n%s", containerName, stdout))
 	} else {
 		for i := 0; i < len(env); i += 2 {
-			envStr := fmt.Sprintf("\n%s=%s\n", env[i], env[i+1])
+			envStr := fmt.Sprintf("%s=%s\n", env[i], env[i+1])
 			gomega.Expect(stdout).To(gomega.ContainSubstring(envStr), fmt.Sprintf("container %s env variables", containerName))
 		}
 	}
@@ -436,7 +467,9 @@ func NewBuilderNow(ctx context.Context, f *framework.Framework, driver *Driver)
 func (b *Builder) setUp(ctx context.Context) {
 	b.podCounter = 0
 	b.claimCounter = 0
-	b.Create(ctx, b.Class())
+	for i := -1; i < 6; i++ {
+		b.Create(ctx, b.Class(i))
+	}
 	ginkgo.DeferCleanup(b.tearDown)
 }
 

test/e2e/dra/utils/deploy.go

@@ -81,8 +81,19 @@ import (
 )
 
 type Nodes struct {
+	// NodeNames has the main set of node names.
 	NodeNames []string
 	tempDir   string
+	// NumReservedNodes specifies the desired number of
+	// extra nodes that get set aside. That many node names
+	// will be stored in ExtraNodeNames.
+	//
+	// Must be <= the minimum number of requested nodes.
+	NumReservedNodes int
+	// ExtraNodeNames has exactly as many node names as
+	// requested via NumReservedNodes. Those nodes are
+	// different than the nodes listed in NodeNames.
+	ExtraNodeNames []string
 }
 
 // NewNodes selects nodes to run the test on.
@@ -117,10 +128,14 @@ func (nodes *Nodes) init(ctx context.Context, f *framework.Framework, minNodes,
 	framework.ExpectNoError(err, "get nodes")
 	numNodes := int32(len(nodeList.Items))
 	if int(numNodes) < minNodes {
-		e2eskipper.Skipf("%d ready nodes required, only have %d", minNodes, numNodes)
+		e2eskipper.Skipf("%d ready nodes required, only have %d", minNodes+nodes.NumReservedNodes, numNodes)
 	}
 	nodes.NodeNames = nil
-	for _, node := range nodeList.Items {
+	for i, node := range nodeList.Items {
+		if i < nodes.NumReservedNodes {
+			nodes.ExtraNodeNames = append(nodes.ExtraNodeNames, node.Name)
+			continue
+		}
 		nodes.NodeNames = append(nodes.NodeNames, node.Name)
 	}
 	sort.Strings(nodes.NodeNames)

test/e2e/node/pod_admission.go

@@ -99,23 +99,24 @@ var _ = SIGDescribe("PodRejectionStatus", func() {
 			// This detects if there are any new fields in Status that were dropped by the pod rejection.
 			// These new fields either should be kept by kubelet's admission or added explicitly in the list of fields that are having a different value or must be cleared.
 			gomega.Expect(gotPod.Status).To(gstruct.MatchAllFields(gstruct.Fields{
-				"ObservedGeneration":         gstruct.Ignore(),
-				"Phase":                      gstruct.Ignore(),
-				"Conditions":                 gstruct.Ignore(),
-				"Message":                    gstruct.Ignore(),
-				"Reason":                     gstruct.Ignore(),
-				"NominatedNodeName":          gstruct.Ignore(),
-				"HostIP":                     gstruct.Ignore(),
-				"HostIPs":                    gstruct.Ignore(),
-				"PodIP":                      gstruct.Ignore(),
-				"PodIPs":                     gstruct.Ignore(),
-				"StartTime":                  gstruct.Ignore(),
-				"InitContainerStatuses":      gstruct.Ignore(),
-				"ContainerStatuses":          gstruct.Ignore(),
-				"QOSClass":                   gomega.Equal(pod.Status.QOSClass), // QOSClass should be kept
-				"EphemeralContainerStatuses": gstruct.Ignore(),
-				"Resize":                     gstruct.Ignore(),
-				"ResourceClaimStatuses":      gstruct.Ignore(),
+				"ObservedGeneration":          gstruct.Ignore(),
+				"Phase":                       gstruct.Ignore(),
+				"Conditions":                  gstruct.Ignore(),
+				"Message":                     gstruct.Ignore(),
+				"Reason":                      gstruct.Ignore(),
+				"NominatedNodeName":           gstruct.Ignore(),
+				"HostIP":                      gstruct.Ignore(),
+				"HostIPs":                     gstruct.Ignore(),
+				"PodIP":                       gstruct.Ignore(),
+				"PodIPs":                      gstruct.Ignore(),
+				"StartTime":                   gstruct.Ignore(),
+				"InitContainerStatuses":       gstruct.Ignore(),
+				"ContainerStatuses":           gstruct.Ignore(),
+				"QOSClass":                    gomega.Equal(pod.Status.QOSClass), // QOSClass should be kept
+				"EphemeralContainerStatuses":  gstruct.Ignore(),
+				"Resize":                      gstruct.Ignore(),
+				"ResourceClaimStatuses":       gstruct.Ignore(),
+				"ExtendedResourceClaimStatus": gstruct.Ignore(),
 			}))
 		})
 	})