Add Config Knob to allow Pod to use different VPC subnet

Author: liwenwu-amazon

config/eniconfig-crd.yaml

@@ -0,0 +1,13 @@
+apiVersion: apiextensions.k8s.io/v1beta1
+kind: CustomResourceDefinition
+metadata:
+  name: eniconfigs.crd.k8s.amazonaws.com
+spec:
+  scope: Cluster
+  group: crd.k8s.amazonaws.com
+  version: v1alpha1
+  names:
+    scope: Cluster
+    plural: eniconfigs
+    singuar: eniconfig
+    kind: ENIConfig

ipamd/datastore/data_store.go

@@ -351,8 +351,12 @@ func (ds *DataStore) getDeletableENI() *ENIIPPool {
 }
 
 // GetENINeedsIP finds out the eni in datastore which failed to get secondary IP address
-func (ds *DataStore) GetENINeedsIP(maxIPperENI int64) *ENIIPPool {
+func (ds *DataStore) GetENINeedsIP(maxIPperENI int64, skipPrimary bool) *ENIIPPool {
 	for _, eni := range ds.eniIPPools {
+		if skipPrimary && eni.IsPrimary {
+			log.Debugf("Skip the primary ENI for need IP check")
+			continue
+		}
 		if int64(len(eni.IPv4Addresses)) < maxIPperENI {
 			log.Debugf("Found eni %s that have less IP address allocated: cur=%d, max=%d",
 				eni.ID, len(eni.IPv4Addresses), maxIPperENI)

ipamd/introspect.go

@@ -69,9 +69,11 @@ func (c *IPAMContext) SetupHTTP() {
 
 func (c *IPAMContext) setupServer() *http.Server {
 	serverFunctions := map[string]func(w http.ResponseWriter, r *http.Request){
-		"/v1/enis":         eniV1RequestHandler(c),
-		"/v1/pods":         podV1RequestHandler(c),
-		"/v1/env-settings": envV1RequestHandler(c),
+		"/v1/enis":                      eniV1RequestHandler(c),
+		"/v1/pods":                      podV1RequestHandler(c),
+		"/v1/networkutils-env-settings": networkEnvV1RequestHandler(c),
+		"/v1/ipamd-env-settings":        ipamdEnvV1RequestHandler(c),
+		"/v1/eni-configs":               eniConfigRequestHandler(c),
 	}
 	paths := make([]string, 0, len(serverFunctions))
 	for path := range serverFunctions {
@@ -134,7 +136,19 @@ func podV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
 	}
 }
 
-func envV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
+func eniConfigRequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		responseJSON, err := json.Marshal(ipam.eniConfig.Getter())
+		if err != nil {
+			log.Error("Failed to marshal pod data: %v", err)
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+		w.Write(responseJSON)
+	}
+}
+
+func networkEnvV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		responseJSON, err := json.Marshal(networkutils.GetConfigForDebug())
 		if err != nil {
@@ -146,6 +160,18 @@ func envV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Requ
 	}
 }
 
+func ipamdEnvV1RequestHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
+		responseJSON, err := json.Marshal(GetConfigForDebug())
+		if err != nil {
+			log.Error("Failed to marshal env var data: %v", err)
+			http.Error(w, http.StatusText(http.StatusInternalServerError), http.StatusInternalServerError)
+			return
+		}
+		w.Write(responseJSON)
+	}
+}
+
 func metricsHandler(ipam *IPAMContext) func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		promhttp.Handler()

ipamd/ipamd.go

@@ -14,6 +14,7 @@
 package ipamd
 
 import (
+	"fmt"
 	"net"
 	"os"
 	"strconv"
@@ -31,6 +32,7 @@ import (
 	"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/docker"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils"
 )
@@ -43,11 +45,40 @@ const (
 	ipPoolMonitorInterval       = 5 * time.Second
 	maxRetryCheckENI            = 5
 	eniAttachTime               = 10 * time.Second
-	defaultWarmENITarget        = 1
 	nodeIPPoolReconcileInterval = 60 * time.Second
 	maxK8SRetries               = 12
 	retryK8SInterval            = 5 * time.Second
 	noWarmIPTarget              = 0
+
+	// This environment is used to specify the desired number of free IPs always available in "warm pool"
+	// When it is not set, ipamD defaut to use the number IPs per ENI for that instance.
+	// For example, for a m4.4xlarge node,
+	//     if WARM-IP-TARGET is set to 1, and there are 9 pods running on the node, ipamD will try
+	//     to make "warm pool" to have 10 IP address with 9 being assigned to Pod and 1 free IP.
+	//
+	//     if "WARM-IP-TARGET is not set, it will be defaulted to 30 (which the number of IPs per ENI). If there are 9 pods
+	//     running on the node, ipamD will try to make "warm pool" to have 39 IPs with 9 being assigned to Pod and 30 free IPs.
+	envWarmIPTarget = "WARM_IP_TARGET"
+
+	// This environment is used to specify the desired number of free ENIs along with all of its IP addresses
+	// always available in "warm pool".
+	// When it is not set, it is default to 1.
+	//
+	// when "WARM-IP-TARGET" is defined, ipamD will use behavior defined for "WARM-IP-TARGET".
+	//
+	// For example, for a m4.4xlarget node
+	//     if WARM_ENI_TARGET is set to 2, and there are 9 pods running on the node, ipamD will try to
+	//     make "warm  pool" to have 2 extra ENIs and its IP addresses,  in another word, 90 IP addresses with 9 IPs assigne to Pod
+	//     and 81 free IPs.
+	//
+	//     if "WARM_ENI_TARGET" is not set, it is default to 1,  if there 9 pods running on the node, ipamD will try to
+	//     make "warm pool" to have 1 extra ENI, in aother word 60 IPs with 9 being assigned to Pod and 51 free IPs.
+	envWarmENITarget     = "WARM_ENI_TARGET"
+	defaultWarmENITarget = 1
+
+	// This environment is used to specify whether Pods need to use securitygroup and subnet defined in ENIConfig CRD
+	// When it is NOT set or set to false, ipamD will use primary interface security group and subnet for Pod network.
+	envCustomNetworkCfg = "AWS_VPC_K8S_CNI_CUSTOM_NETWORK_CFG"
 )
 
 var (
@@ -105,6 +136,7 @@ type IPAMContext struct {
 	awsClient     awsutils.APIs
 	dataStore     *datastore.DataStore
 	k8sClient     k8sapi.K8SAPIs
+	eniConfig     eniconfig.ENIConfig
 	dockerClient  docker.APIs
 	networkClient networkutils.NetworkAPIs
 
@@ -133,13 +165,14 @@ func prometheusRegister() {
 
 // New retrieves IP address usage information from Instance MetaData service and Kubelet
 // then initializes IP address pool data store
-func New(k8sapiClient k8sapi.K8SAPIs) (*IPAMContext, error) {
+func New(k8sapiClient k8sapi.K8SAPIs, eniConfig *eniconfig.ENIConfigController) (*IPAMContext, error) {
 	prometheusRegister()
 	c := &IPAMContext{}
 
 	c.k8sClient = k8sapiClient
 	c.networkClient = networkutils.New()
 	c.dockerClient = docker.New()
+	c.eniConfig = eniConfig
 
 	client, err := awsutils.New()
 	if err != nil {
@@ -314,7 +347,7 @@ func (c *IPAMContext) retryAllocENIIP() {
 		log.Infof("Failed to retrieve ENI IP limit: %v", err)
 		return
 	}
-	eni := c.dataStore.GetENINeedsIP(maxIPLimit)
+	eni := c.dataStore.GetENINeedsIP(maxIPLimit, useCustomNetworkCfg())
 	if eni != nil {
 		log.Debugf("Attempt again to allocate IP address for eni :%s", eni.ID)
 		var err error
@@ -392,7 +425,25 @@ func (c *IPAMContext) increaseIPPool() {
 		log.Debugf("Skipping increase IPPOOL due to max ENI already attached to the instance : %d", c.maxENI)
 		return
 	}
-	eni, err := c.awsClient.AllocENI()
+
+	var securityGroups []string
+	var subnet string
+	customNetworkCfg := useCustomNetworkCfg()
+
+	if customNetworkCfg {
+		eniCfg, err := c.eniConfig.MyENIConfig()
+
+		if err != nil {
+			log.Errorf("Failed to get pod ENI config")
+			return
+		}
+
+		log.Infof("ipamd: using custom network config: %v, %s", eniCfg.SecurityGroups, eniCfg.Subnet)
+		securityGroups = eniCfg.SecurityGroups
+		subnet = eniCfg.Subnet
+	}
+
+	eni, err := c.awsClient.AllocENI(customNetworkCfg, securityGroups, subnet)
 	if err != nil {
 		log.Errorf("Failed to increase pool size due to not able to allocate ENI %v", err)
 
@@ -546,7 +597,7 @@ func (c *IPAMContext) waitENIAttached(eni string) (awsutils.ENIMetadata, error)
 }
 
 func getWarmENITarget() int {
-	inputStr, found := os.LookupEnv("WARM_ENI_TARGET")
+	inputStr, found := os.LookupEnv(envWarmENITarget)
 
 	if !found {
 		return defaultWarmENITarget
@@ -722,8 +773,21 @@ func (c *IPAMContext) eniIPPoolReconcile(ipPool map[string]*datastore.AddressInf
 
 }
 
+func useCustomNetworkCfg() bool {
+	defaultValue := false
+	if strValue := os.Getenv(envCustomNetworkCfg); strValue != "" {
+		parsedValue, err := strconv.ParseBool(strValue)
+		if err != nil {
+			log.Error("Failed to parse "+envCustomNetworkCfg+"; using default: "+fmt.Sprint(defaultValue), err.Error())
+			return defaultValue
+		}
+		return parsedValue
+	}
+	return defaultValue
+}
+
 func getWarmIPTarget() int {
-	inputStr, found := os.LookupEnv("WARM_IP_TARGET")
+	inputStr, found := os.LookupEnv(envWarmIPTarget)
 
 	if !found {
 		return noWarmIPTarget
@@ -753,3 +817,12 @@ func (c *IPAMContext) getCurWarmIPTarget() (int, bool) {
 
 	return curTarget, true
 }
+
+// GetConfigForDebug returns the active values of the configuration env vars (for debugging purposes).
+func GetConfigForDebug() map[string]interface{} {
+	return map[string]interface{}{
+		envWarmIPTarget:     getWarmIPTarget(),
+		envWarmENITarget:    getWarmENITarget(),
+		envCustomNetworkCfg: useCustomNetworkCfg(),
+	}
+}

ipamd/ipamd_test.go

@@ -18,11 +18,15 @@ import (
 	"os"
 	"testing"
 
+	"github.com/pkg/errors"
+
 	"github.com/aws/amazon-vpc-cni-k8s/ipamd/datastore"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/apis/crd/v1alpha1"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/awsutils/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/docker"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/docker/mocks"
+	"github.com/aws/amazon-vpc-cni-k8s/pkg/eniconfig/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/k8sapi/mocks"
 	"github.com/aws/amazon-vpc-cni-k8s/pkg/networkutils/mocks"
@@ -57,17 +61,19 @@ func setup(t *testing.T) (*gomock.Controller,
 	*mock_awsutils.MockAPIs,
 	*mock_k8sapi.MockK8SAPIs,
 	*mock_docker.MockAPIs,
-	*mock_networkutils.MockNetworkAPIs) {
+	*mock_networkutils.MockNetworkAPIs,
+	*mock_eniconfig.MockENIConfig) {
 	ctrl := gomock.NewController(t)
 	return ctrl,
 		mock_awsutils.NewMockAPIs(ctrl),
 		mock_k8sapi.NewMockK8SAPIs(ctrl),
 		mock_docker.NewMockAPIs(ctrl),
-		mock_networkutils.NewMockNetworkAPIs(ctrl)
+		mock_networkutils.NewMockNetworkAPIs(ctrl),
+		mock_eniconfig.NewMockENIConfig(ctrl)
 }
 
 func TestNodeInit(t *testing.T) {
-	ctrl, mockAWS, mockK8S, mockDocker, mockNetwork := setup(t)
+	ctrl, mockAWS, mockK8S, mockDocker, mockNetwork, _ := setup(t)
 	defer ctrl.Finish()
 
 	mockContext := &IPAMContext{
@@ -143,14 +149,47 @@ func TestNodeInit(t *testing.T) {
 	assert.NoError(t, err)
 }
 
-func TestIncreaseIPPool(t *testing.T) {
-	ctrl, mockAWS, mockK8S, _, mockNetwork := setup(t)
+func TestIncreaseIPPoolDefault(t *testing.T) {
+	os.Unsetenv(envCustomNetworkCfg)
+	testIncreaseIPPool(t, false)
+}
+
+func TestIncreaseIPPoolCustomENI(t *testing.T) {
+	os.Setenv(envCustomNetworkCfg, "true")
+	testIncreaseIPPool(t, true)
+}
+
+func TestIncreaseIPPoolCustomENINoCfg(t *testing.T) {
+	os.Setenv(envCustomNetworkCfg, "true")
+	ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t)
+	defer ctrl.Finish()
+
+	mockContext := &IPAMContext{
+		awsClient:     mockAWS,
+		k8sClient:     mockK8S,
+		networkClient: mockNetwork,
+		eniConfig:     mockENIConfig,
+		primaryIP:     make(map[string]string),
+	}
+
+	mockContext.dataStore = datastore.NewDataStore()
+
+	mockAWS.EXPECT().GetENILimit().Return(4, nil)
+	mockENIConfig.EXPECT().MyENIConfig().Return(nil, errors.New("no POD eni config"))
+
+	mockContext.increaseIPPool()
+
+}
+
+func testIncreaseIPPool(t *testing.T, useENIConfig bool) {
+	ctrl, mockAWS, mockK8S, _, mockNetwork, mockENIConfig := setup(t)
 	defer ctrl.Finish()
 
 	mockContext := &IPAMContext{
 		awsClient:     mockAWS,
 		k8sClient:     mockK8S,
 		networkClient: mockNetwork,
+		eniConfig:     mockENIConfig,
 		primaryIP:     make(map[string]string),
 	}
 
@@ -159,7 +198,18 @@ func TestIncreaseIPPool(t *testing.T) {
 	eni2 := secENIid
 
 	mockAWS.EXPECT().GetENILimit().Return(4, nil)
-	mockAWS.EXPECT().AllocENI().Return(eni2, nil)
+
+	podENIConfig := &v1alpha1.ENIConfigSpec{
+		SecurityGroups: []string{"sg1-id", "sg2-id"},
+		Subnet:         "subnet1",
+	}
+
+	if useENIConfig {
+		mockENIConfig.EXPECT().MyENIConfig().Return(podENIConfig, nil)
+		mockAWS.EXPECT().AllocENI(true, podENIConfig.SecurityGroups, podENIConfig.Subnet).Return(eni2, nil)
+	} else {
+		mockAWS.EXPECT().AllocENI(false, nil, "").Return(eni2, nil)
+	}
 
 	mockAWS.EXPECT().GetENIipLimit().Return(int64(5), nil)
 
@@ -203,7 +253,7 @@ func TestIncreaseIPPool(t *testing.T) {
 }
 
 func TestNodeIPPoolReconcile(t *testing.T) {
-	ctrl, mockAWS, mockK8S, _, mockNetwork := setup(t)
+	ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t)
 	defer ctrl.Finish()
 
 	mockContext := &IPAMContext{
@@ -273,7 +323,7 @@ func TestNodeIPPoolReconcile(t *testing.T) {
 }
 
 func TestGetWarmENITarget(t *testing.T) {
-	ctrl, _, _, _, _ := setup(t)
+	ctrl, _, _, _, _, _ := setup(t)
 	defer ctrl.Finish()
 
 	os.Setenv("WARM_IP_TARGET", "5")
@@ -290,7 +340,7 @@ func TestGetWarmENITarget(t *testing.T) {
 }
 
 func TestGetCurWarmIPTarget(t *testing.T) {
-	ctrl, mockAWS, mockK8S, _, mockNetwork := setup(t)
+	ctrl, mockAWS, mockK8S, _, mockNetwork, _ := setup(t)
 	defer ctrl.Finish()
 
 	mockContext := &IPAMContext{