diff --git a/test/framework/resources/aws/utils/nodegroup.go b/test/framework/resources/aws/utils/nodegroup.go index 9847621e9a..51b51b17f2 100644 --- a/test/framework/resources/aws/utils/nodegroup.go +++ b/test/framework/resources/aws/utils/nodegroup.go @@ -218,12 +218,10 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node } func DeleteAndWaitTillSelfManagedNGStackDeleted(f *framework.Framework, properties NodeGroupProperties) error { - err := f.CloudServices.CloudFormation(). - WaitTillStackDeleted(properties.NodeGroupName) + err := f.CloudServices.CloudFormation().WaitTillStackDeleted(properties.NodeGroupName) if err != nil { return fmt.Errorf("failed to delete node group cfn stack: %v", err) } - return nil } diff --git a/test/framework/resources/k8s/manifest/eniconfig.go b/test/framework/resources/k8s/manifest/eniconfig.go index 1c3d7a12ad..1e9445a5cb 100644 --- a/test/framework/resources/k8s/manifest/eniconfig.go +++ b/test/framework/resources/k8s/manifest/eniconfig.go @@ -52,13 +52,24 @@ func (e *ENIConfigBuilder) Build() (*v1alpha1.ENIConfig, error) { return nil, fmt.Errorf("subnet id is a required field") } - return &v1alpha1.ENIConfig{ - ObjectMeta: v1.ObjectMeta{ - Name: e.name, - }, - Spec: v1alpha1.ENIConfigSpec{ - SecurityGroups: e.securityGroup, - Subnet: e.subnetID, - }, - }, nil + if e.securityGroup == nil { + return &v1alpha1.ENIConfig{ + ObjectMeta: v1.ObjectMeta{ + Name: e.name, + }, + Spec: v1alpha1.ENIConfigSpec{ + Subnet: e.subnetID, + }, + }, nil + } else { + return &v1alpha1.ENIConfig{ + ObjectMeta: v1.ObjectMeta{ + Name: e.name, + }, + Spec: v1alpha1.ENIConfigSpec{ + SecurityGroups: e.securityGroup, + Subnet: e.subnetID, + }, + }, nil + } } diff --git a/test/integration/custom-networking/custom_networking_suite_test.go b/test/integration/custom-networking/custom_networking_suite_test.go index bddd5cfa44..d79ecb1b86 100644 --- a/test/integration/custom-networking/custom_networking_suite_test.go +++ b/test/integration/custom-networking/custom_networking_suite_test.go @@ -55,7 +55,8 @@ var ( customNetworkingSGOpenPort = 8080 customNetworkingSubnetIDList []string // List of ENIConfig per Availability Zone - eniConfigList []*v1alpha1.ENIConfig + eniConfigList []*v1alpha1.ENIConfig + eniConfigBuilderList []*manifest.ENIConfigBuilder // Properties of the self managed node group created using CFN template nodeGroupProperties awsUtils.NodeGroupProperties err error @@ -73,8 +74,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) By("creating test namespace") - f.K8sResourceManagers.NamespaceManager(). - CreateNamespace(utils.DefaultTestNamespace) + f.K8sResourceManagers.NamespaceManager().CreateNamespace(utils.DefaultTestNamespace) By("getting the cluster VPC Config") clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f) @@ -91,10 +91,10 @@ var _ = BeforeSuite(func() { customNetworkingSGID = *createSecurityGroupOutput.GroupId By("authorizing egress and ingress on security group for single port") - f.CloudServices.EC2(). - AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") - f.CloudServices.EC2(). - AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") + f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP", + customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") + f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", + customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0") By("associating cidr range to the VPC") association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(f.Options.AWSVPCID, cidrRange.String()) @@ -114,19 +114,19 @@ var _ = BeforeSuite(func() { subnetID := *createSubnetOutput.Subnet.SubnetId By("associating the route table with the newly created subnet") - err = f.CloudServices.EC2(). - AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID) + err = f.CloudServices.EC2().AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID) Expect(err).ToNot(HaveOccurred()) - eniConfig, err := manifest.NewENIConfigBuilder(). + eniConfigBuilder := manifest.NewENIConfigBuilder(). Name(az). SubnetID(subnetID). - SecurityGroup([]string{customNetworkingSGID}). - Build() + SecurityGroup([]string{customNetworkingSGID}) + eniConfig, err := eniConfigBuilder.Build() Expect(err).ToNot(HaveOccurred()) - // For deleting later + // For updating/deleting later customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID) + eniConfigBuilderList = append(eniConfigBuilderList, eniConfigBuilder) eniConfigList = append(eniConfigList, eniConfig.DeepCopy()) By("creating the ENIConfig with az name") @@ -205,3 +205,22 @@ var _ = AfterSuite(func() { } Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred()) }) + +func TerminateInstances(f *framework.Framework) { + By("getting the list of nodes created") + nodeList, err := f.K8sResourceManagers.NodeManager(). + GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal) + Expect(err).ToNot(HaveOccurred()) + + var instanceIDs []string + for _, node := range nodeList.Items { + instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node)) + } + + By("terminating all the nodes") + err = f.CloudServices.EC2().TerminateInstance(instanceIDs) + Expect(err).ToNot(HaveOccurred()) + + By("waiting for nodes to be recycled") + time.Sleep(time.Second * 300) +} diff --git a/test/integration/custom-networking/custom_networking_test.go b/test/integration/custom-networking/custom_networking_test.go index e04335b916..9dcabe1532 100644 --- a/test/integration/custom-networking/custom_networking_test.go +++ b/test/integration/custom-networking/custom_networking_test.go @@ -17,10 +17,8 @@ import ( "fmt" "net" "strconv" - "time" "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest" - k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils" "github.com/aws/amazon-vpc-cni-k8s/test/framework/utils" . "github.com/onsi/ginkgo/v2" @@ -30,7 +28,6 @@ import ( ) var _ = Describe("Custom Networking Test", func() { - var ( deployment *v1.Deployment podList coreV1.PodList @@ -78,7 +75,7 @@ var _ = Describe("Custom Networking Test", func() { testContainer := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry). Command([]string{"nc"}). - Args([]string{"-v", "-w2", pod.Status.PodIP, strconv.Itoa(port)}). + Args([]string{"-v", "-w3", pod.Status.PodIP, strconv.Itoa(port)}). Build() testJob := manifest.NewDefaultJobBuilder(). @@ -87,8 +84,7 @@ var _ = Describe("Custom Networking Test", func() { Parallelism(1). Build() - _, err := f.K8sResourceManagers.JobManager(). - CreateAndWaitTillJobCompleted(testJob) + _, err := f.K8sResourceManagers.JobManager().CreateAndWaitTillJobCompleted(testJob) if shouldConnect { By("verifying connection to pod succeeds on port " + strconv.Itoa(port)) Expect(err).ToNot(HaveOccurred()) @@ -97,25 +93,22 @@ var _ = Describe("Custom Networking Test", func() { Expect(err).To(HaveOccurred()) } - err = f.K8sResourceManagers.JobManager(). - DeleteAndWaitTillJobIsDeleted(testJob) + err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(testJob) Expect(err).ToNot(HaveOccurred()) } }) JustAfterEach(func() { - err = f.K8sResourceManagers.DeploymentManager(). - DeleteAndWaitTillDeploymentIsDeleted(deployment) + err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment) Expect(err).ToNot(HaveOccurred()) }) Context("when connecting to reachable port", func() { BeforeEach(func() { port = customNetworkingSGOpenPort - replicaCount = 30 + replicaCount = 16 shouldConnect = true }) - It("should connect", func() {}) }) @@ -125,17 +118,15 @@ var _ = Describe("Custom Networking Test", func() { replicaCount = 1 shouldConnect = false }) - It("should fail to connect", func() {}) }) }) - Context("when creating deployment on nodes that don't have ENIConfig", func() { + Context("when creating deployment on nodes that do not have ENIConfig", func() { JustBeforeEach(func() { By("deleting ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { - err = f.K8sResourceManagers.CustomResourceManager(). - DeleteResource(eniConfig) + err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) Expect(err).ToNot(HaveOccurred()) } }) @@ -143,29 +134,13 @@ var _ = Describe("Custom Networking Test", func() { JustAfterEach(func() { By("re-creating ENIConfig for all availability zones") for _, eniConfig := range eniConfigList { - err = f.K8sResourceManagers.CustomResourceManager(). - CreateResource(eniConfig) + err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) Expect(err).ToNot(HaveOccurred()) } }) It("deployment should not become ready", func() { - By("getting the list of nodes created") - nodeList, err := f.K8sResourceManagers.NodeManager(). - GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal) - Expect(err).ToNot(HaveOccurred()) - - var instanceIDs []string - for _, node := range nodeList.Items { - instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node)) - } - - By("terminating all the nodes") - err = f.CloudServices.EC2().TerminateInstance(instanceIDs) - Expect(err).ToNot(HaveOccurred()) - - By("waiting for nodes to be removed") - time.Sleep(time.Second * 120) + TerminateInstances(f) // Nodes should be stuck in NotReady state since no ENIs could be attached and no pod // IP addresses are available. @@ -185,4 +160,42 @@ var _ = Describe("Custom Networking Test", func() { Expect(err).ToNot(HaveOccurred()) }) }) + + Context("when creating ENIConfigs without security groups", func() { + JustBeforeEach(func() { + By("deleting ENIConfig for each availability zone") + for _, eniConfig := range eniConfigList { + err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } + By("re-creating ENIConfigs with no security group") + eniConfigList = nil + for _, eniConfigBuilder := range eniConfigBuilderList { + eniConfigBuilder.SecurityGroup(nil) + eniConfig, err := eniConfigBuilder.Build() + eniConfigList = append(eniConfigList, eniConfig.DeepCopy()) + + err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig) + Expect(err).ToNot(HaveOccurred()) + } + }) + + It("deployment should become ready", func() { + TerminateInstances(f) + deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry). + Replicas(2). + NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal). + Build() + + By("verifying deployment succeeds") + deployment, err = f.K8sResourceManagers.DeploymentManager(). + CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout) + Expect(err).ToNot(HaveOccurred()) + + By("deleting the deployment") + err = f.K8sResourceManagers.DeploymentManager(). + DeleteAndWaitTillDeploymentIsDeleted(deployment) + Expect(err).ToNot(HaveOccurred()) + }) + }) })