Skip to content

Commit

Permalink
add custom-networking test covering ENIConfig objects with no security (
Browse files Browse the repository at this point in the history 
#2445)

groups
  • Loading branch information
@jdn5126
jdn5126 authored Jul 7, 2023
1 parent 438dcb3 commit 21ada86
Show file tree
Hide file tree
Showing 4 changed files with 100 additions and 59 deletions.
4 changes: 1 addition & 3 deletions test/framework/resources/aws/utils/nodegroup.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -218,12 +218,10 @@ func CreateAndWaitTillSelfManagedNGReady(f *framework.Framework, properties Node
}

func DeleteAndWaitTillSelfManagedNGStackDeleted(f *framework.Framework, properties NodeGroupProperties) error {
err := f.CloudServices.CloudFormation().
WaitTillStackDeleted(properties.NodeGroupName)
err := f.CloudServices.CloudFormation().WaitTillStackDeleted(properties.NodeGroupName)
if err != nil {
return fmt.Errorf("failed to delete node group cfn stack: %v", err)
}

return nil
}

Expand Down
29 changes: 20 additions & 9 deletions test/framework/resources/k8s/manifest/eniconfig.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -52,13 +52,24 @@ func (e *ENIConfigBuilder) Build() (*v1alpha1.ENIConfig, error) {
return nil, fmt.Errorf("subnet id is a required field")
}

return &v1alpha1.ENIConfig{
ObjectMeta: v1.ObjectMeta{
Name: e.name,
},
Spec: v1alpha1.ENIConfigSpec{
SecurityGroups: e.securityGroup,
Subnet: e.subnetID,
},
}, nil
if e.securityGroup == nil {
return &v1alpha1.ENIConfig{
ObjectMeta: v1.ObjectMeta{
Name: e.name,
},
Spec: v1alpha1.ENIConfigSpec{
Subnet: e.subnetID,
},
}, nil
} else {
return &v1alpha1.ENIConfig{
ObjectMeta: v1.ObjectMeta{
Name: e.name,
},
Spec: v1alpha1.ENIConfigSpec{
SecurityGroups: e.securityGroup,
Subnet: e.subnetID,
},
}, nil
}
}
45 changes: 32 additions & 13 deletions test/integration/custom-networking/custom_networking_suite_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -55,7 +55,8 @@ var (
customNetworkingSGOpenPort = 8080
customNetworkingSubnetIDList []string
// List of ENIConfig per Availability Zone
eniConfigList []*v1alpha1.ENIConfig
eniConfigList []*v1alpha1.ENIConfig
eniConfigBuilderList []*manifest.ENIConfigBuilder
// Properties of the self managed node group created using CFN template
nodeGroupProperties awsUtils.NodeGroupProperties
err error
Expand All @@ -73,8 +74,7 @@ var _ = BeforeSuite(func() {
Expect(err).ToNot(HaveOccurred())

By("creating test namespace")
f.K8sResourceManagers.NamespaceManager().
CreateNamespace(utils.DefaultTestNamespace)
f.K8sResourceManagers.NamespaceManager().CreateNamespace(utils.DefaultTestNamespace)

By("getting the cluster VPC Config")
clusterVPCConfig, err = awsUtils.GetClusterVPCConfig(f)
Expand All @@ -91,10 +91,10 @@ var _ = BeforeSuite(func() {
customNetworkingSGID = *createSecurityGroupOutput.GroupId

By("authorizing egress and ingress on security group for single port")
f.CloudServices.EC2().
AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")
f.CloudServices.EC2().
AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP", customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")
f.CloudServices.EC2().AuthorizeSecurityGroupEgress(customNetworkingSGID, "TCP",
customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")
f.CloudServices.EC2().AuthorizeSecurityGroupIngress(customNetworkingSGID, "TCP",
customNetworkingSGOpenPort, customNetworkingSGOpenPort, "0.0.0.0/0")

By("associating cidr range to the VPC")
association, err := f.CloudServices.EC2().AssociateVPCCIDRBlock(f.Options.AWSVPCID, cidrRange.String())
Expand All @@ -114,19 +114,19 @@ var _ = BeforeSuite(func() {
subnetID := *createSubnetOutput.Subnet.SubnetId

By("associating the route table with the newly created subnet")
err = f.CloudServices.EC2().
AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID)
err = f.CloudServices.EC2().AssociateRouteTableToSubnet(clusterVPCConfig.PublicRouteTableID, subnetID)
Expect(err).ToNot(HaveOccurred())

eniConfig, err := manifest.NewENIConfigBuilder().
eniConfigBuilder := manifest.NewENIConfigBuilder().
Name(az).
SubnetID(subnetID).
SecurityGroup([]string{customNetworkingSGID}).
Build()
SecurityGroup([]string{customNetworkingSGID})
eniConfig, err := eniConfigBuilder.Build()
Expect(err).ToNot(HaveOccurred())

// For deleting later
// For updating/deleting later
customNetworkingSubnetIDList = append(customNetworkingSubnetIDList, subnetID)
eniConfigBuilderList = append(eniConfigBuilderList, eniConfigBuilder)
eniConfigList = append(eniConfigList, eniConfig.DeepCopy())

By("creating the ENIConfig with az name")
Expand Down Expand Up @@ -205,3 +205,22 @@ var _ = AfterSuite(func() {
}
Expect(errs.MaybeUnwrap()).ToNot(HaveOccurred())
})

func TerminateInstances(f *framework.Framework) {
By("getting the list of nodes created")
nodeList, err := f.K8sResourceManagers.NodeManager().
GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal)
Expect(err).ToNot(HaveOccurred())

var instanceIDs []string
for _, node := range nodeList.Items {
instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node))
}

By("terminating all the nodes")
err = f.CloudServices.EC2().TerminateInstance(instanceIDs)
Expect(err).ToNot(HaveOccurred())

By("waiting for nodes to be recycled")
time.Sleep(time.Second * 300)
}
81 changes: 47 additions & 34 deletions test/integration/custom-networking/custom_networking_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -17,10 +17,8 @@ import (
"fmt"
"net"
"strconv"
"time"

"github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/manifest"
k8sUtils "github.com/aws/amazon-vpc-cni-k8s/test/framework/resources/k8s/utils"
"github.com/aws/amazon-vpc-cni-k8s/test/framework/utils"

. "github.com/onsi/ginkgo/v2"
Expand All @@ -30,7 +28,6 @@ import (
)

var _ = Describe("Custom Networking Test", func() {

var (
deployment *v1.Deployment
podList coreV1.PodList
Expand Down Expand Up @@ -78,7 +75,7 @@ var _ = Describe("Custom Networking Test", func() {

testContainer := manifest.NewNetCatAlpineContainer(f.Options.TestImageRegistry).
Command([]string{"nc"}).
Args([]string{"-v", "-w2", pod.Status.PodIP, strconv.Itoa(port)}).
Args([]string{"-v", "-w3", pod.Status.PodIP, strconv.Itoa(port)}).
Build()

testJob := manifest.NewDefaultJobBuilder().
Expand All @@ -87,8 +84,7 @@ var _ = Describe("Custom Networking Test", func() {
Parallelism(1).
Build()

_, err := f.K8sResourceManagers.JobManager().
CreateAndWaitTillJobCompleted(testJob)
_, err := f.K8sResourceManagers.JobManager().CreateAndWaitTillJobCompleted(testJob)
if shouldConnect {
By("verifying connection to pod succeeds on port " + strconv.Itoa(port))
Expect(err).ToNot(HaveOccurred())
Expand All @@ -97,25 +93,22 @@ var _ = Describe("Custom Networking Test", func() {
Expect(err).To(HaveOccurred())
}

err = f.K8sResourceManagers.JobManager().
DeleteAndWaitTillJobIsDeleted(testJob)
err = f.K8sResourceManagers.JobManager().DeleteAndWaitTillJobIsDeleted(testJob)
Expect(err).ToNot(HaveOccurred())
}
})

JustAfterEach(func() {
err = f.K8sResourceManagers.DeploymentManager().
DeleteAndWaitTillDeploymentIsDeleted(deployment)
err = f.K8sResourceManagers.DeploymentManager().DeleteAndWaitTillDeploymentIsDeleted(deployment)
Expect(err).ToNot(HaveOccurred())
})

Context("when connecting to reachable port", func() {
BeforeEach(func() {
port = customNetworkingSGOpenPort
replicaCount = 30
replicaCount = 16
shouldConnect = true
})

It("should connect", func() {})
})

Expand All @@ -125,47 +118,29 @@ var _ = Describe("Custom Networking Test", func() {
replicaCount = 1
shouldConnect = false
})

It("should fail to connect", func() {})
})
})

Context("when creating deployment on nodes that don't have ENIConfig", func() {
Context("when creating deployment on nodes that do not have ENIConfig", func() {
JustBeforeEach(func() {
By("deleting ENIConfig for all availability zones")
for _, eniConfig := range eniConfigList {
err = f.K8sResourceManagers.CustomResourceManager().
DeleteResource(eniConfig)
err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
Expect(err).ToNot(HaveOccurred())
}
})

JustAfterEach(func() {
By("re-creating ENIConfig for all availability zones")
for _, eniConfig := range eniConfigList {
err = f.K8sResourceManagers.CustomResourceManager().
CreateResource(eniConfig)
err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig)
Expect(err).ToNot(HaveOccurred())
}
})

It("deployment should not become ready", func() {
By("getting the list of nodes created")
nodeList, err := f.K8sResourceManagers.NodeManager().
GetNodes(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal)
Expect(err).ToNot(HaveOccurred())

var instanceIDs []string
for _, node := range nodeList.Items {
instanceIDs = append(instanceIDs, k8sUtils.GetInstanceIDFromNode(node))
}

By("terminating all the nodes")
err = f.CloudServices.EC2().TerminateInstance(instanceIDs)
Expect(err).ToNot(HaveOccurred())

By("waiting for nodes to be removed")
time.Sleep(time.Second * 120)
TerminateInstances(f)

// Nodes should be stuck in NotReady state since no ENIs could be attached and no pod
// IP addresses are available.
Expand All @@ -185,4 +160,42 @@ var _ = Describe("Custom Networking Test", func() {
Expect(err).ToNot(HaveOccurred())
})
})

Context("when creating ENIConfigs without security groups", func() {
JustBeforeEach(func() {
By("deleting ENIConfig for each availability zone")
for _, eniConfig := range eniConfigList {
err = f.K8sResourceManagers.CustomResourceManager().DeleteResource(eniConfig)
Expect(err).ToNot(HaveOccurred())
}
By("re-creating ENIConfigs with no security group")
eniConfigList = nil
for _, eniConfigBuilder := range eniConfigBuilderList {
eniConfigBuilder.SecurityGroup(nil)
eniConfig, err := eniConfigBuilder.Build()
eniConfigList = append(eniConfigList, eniConfig.DeepCopy())

err = f.K8sResourceManagers.CustomResourceManager().CreateResource(eniConfig)
Expect(err).ToNot(HaveOccurred())
}
})

It("deployment should become ready", func() {
TerminateInstances(f)
deployment := manifest.NewBusyBoxDeploymentBuilder(f.Options.TestImageRegistry).
Replicas(2).
NodeSelector(nodeGroupProperties.NgLabelKey, nodeGroupProperties.NgLabelVal).
Build()

By("verifying deployment succeeds")
deployment, err = f.K8sResourceManagers.DeploymentManager().
CreateAndWaitTillDeploymentIsReady(deployment, utils.DefaultDeploymentReadyTimeout)
Expect(err).ToNot(HaveOccurred())

By("deleting the deployment")
err = f.K8sResourceManagers.DeploymentManager().
DeleteAndWaitTillDeploymentIsDeleted(deployment)
Expect(err).ToNot(HaveOccurred())
})
})
})

0 comments on commit 21ada86

Please sign in to comment.