Support Local Testing Of Terraform (#411)

aws · Mar 28, 2022 · 45127bb · 45127bb
1 parent 8a25004
commit 45127bb
Show file tree

Hide file tree

Showing 6 changed files with 144 additions and 36 deletions.
diff --git a/.github/workflows/integrationTest.yml b/.github/workflows/integrationTest.yml
@@ -6,6 +6,7 @@ env:
   PRIVATE_KEY: ${{ secrets.AWS_PRIVATE_KEY  }}
   TERRAFORM_AWS_ACCESS_KEY_ID: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }}
   TERRAFORM_AWS_SECRET_ACCESS_KEY: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }}
+  S3_INTEGRATION_BUCKET: cloudwatch-agent-integration-bucket
 
 on:
   push:
@@ -68,7 +69,7 @@ jobs:
 
       - name: Upload to s3
         if: steps.cached_binaries.outputs.cache-hit != 'true'
-        run: aws s3 cp build/bin s3://cloudwatch-agent-integration-bucket/integration-test/binary/${{ github.sha }} --recursive
+        run: aws s3 cp build/bin s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ github.sha }} --recursive
 
   MakeMSIZip:
     name: 'MakeMSIZip'
@@ -99,7 +100,7 @@ jobs:
       - name: Copy binary
         if: steps.cached_win_zip.outputs.cache-hit != 'true'
         run: |
-          aws s3 cp s3://cloudwatch-agent-integration-bucket/integration-test/binary/${{ github.sha }} . --recursive
+          aws s3 cp s3://${S3_INTEGRATION_BUCKET}/integration-test/binary/${{ github.sha }} . --recursive
 
       - uses: montudor/action-zip@v1
         if: steps.cached_win_zip.outputs.cache-hit != 'true'
@@ -125,7 +126,7 @@ jobs:
 
       - name: Upload zip
         if: steps.cached_win_zip.outputs.cache-hit != 'true'
-        run: aws s3 cp buildMSI.zip s3://cloudwatch-agent-integration-bucket/integration-test/packaging/${{ github.sha }}/buildMSI.zip
+        run: aws s3 cp buildMSI.zip s3://${S3_INTEGRATION_BUCKET}/integration-test/packaging/${{ github.sha }}/buildMSI.zip
 
   BuildMSI:
     name: 'BuildMSI'
@@ -150,7 +151,7 @@ jobs:
 
       - name: Copy msi
         if: steps.cached_msi.outputs.cache-hit != 'true'
-        run: aws s3 cp s3://cloudwatch-agent-integration-bucket/integration-test/packaging/${{ github.sha }}/buildMSI.zip .
+        run: aws s3 cp s3://${S3_INTEGRATION_BUCKET}/integration-test/packaging/${{ github.sha }}/buildMSI.zip .
 
       - name: Create msi
         if: steps.cached_msi.outputs.cache-hit != 'true'
@@ -194,11 +195,12 @@ jobs:
           terraform apply --auto-approve
           -var="ssh_key=${PRIVATE_KEY}"
           -var="github_repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
-          -var="github_sha=${GITHUB_SHA}" &&
+          -var="github_sha=${GITHUB_SHA}" 
+          -var="s3_bucket=${S3_INTEGRATION_BUCKET}" &&
           LOCAL_STACK_HOST_NAME=$(terraform output -raw public_dns) &&
           echo $LOCAL_STACK_HOST_NAME &&
           echo "::set-output name=local_stack_host_name::$LOCAL_STACK_HOST_NAME" &&
-          aws s3 cp terraform.tfstate s3://cloudwatch-agent-integration-bucket/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate
+          aws s3 cp terraform.tfstate s3://${S3_INTEGRATION_BUCKET}/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate
 
   EC2LinuxIntegrationTest:
     needs: [MakeBinary, StartLocalStack]
@@ -211,47 +213,47 @@ jobs:
       fail-fast: false
       matrix:
         arrays: [
-          { os: "ubuntu", package: "package-deb",  username: "ubuntu",
+          { os: "ubuntu", username: "ubuntu",
             installAgentCommand: "dpkg -i -E ./amazon-cloudwatch-agent.deb",
             ami: "cloudwatch-agent-integration-test-ubuntu*", caCertPath: "/etc/ssl/certs/ca-certificates.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.deb" },
-          { os: "al2", package: "package-rpm",  username: "ec2-user",
+          { os: "al2", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-al2*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "rhel8", package: "package-rpm",  username: "ec2-user",
+          { os: "rhel8", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-rhel8-base*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "rhel8-1", package: "package-rpm",  username: "ec2-user",
+          { os: "rhel8-1", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-rhel8-1*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "rhel8-2", package: "package-rpm",  username: "ec2-user",
+          { os: "rhel8-2", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-rhel8-2*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "rhel8-3", package: "package-rpm",  username: "ec2-user",
+          { os: "rhel8-3", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-rhel8-3*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "rhel8-4", package: "package-rpm",  username: "ec2-user",
+          { os: "rhel8-4", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-rhel8-4*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "ol8-1", package: "package-rpm",  username: "ec2-user",
+          { os: "ol8-1", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-ol8-1*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "ol8-2", package: "package-rpm",  username: "ec2-user",
+          { os: "ol8-2", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-ol8-2*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "o8-3", package: "package-rpm",  username: "ec2-user",
+          { os: "o8-3", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-ol8-3*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" },
-          { os: "ol8-4", package: "package-rpm",  username: "ec2-user",
+          { os: "ol8-4", username: "ec2-user",
             installAgentCommand: "rpm -U ./amazon-cloudwatch-agent.rpm",
             ami: "cloudwatch-agent-integration-test-ol8-4*", caCertPath: "/etc/ssl/certs/ca-bundle.crt",
             arc: "amd64", binaryName: "amazon-cloudwatch-agent.rpm" }
@@ -291,20 +293,18 @@ jobs:
           -var="ssh_key=${PRIVATE_KEY}"
           -var="github_repo=${GITHUB_SERVER_URL}/${GITHUB_REPOSITORY}.git"
           -var="github_sha=${GITHUB_SHA}"
-          -var="package=${{ matrix.arrays.package }}"
           -var="install_agent=${{ matrix.arrays.installAgentCommand }}"
           -var="user=${{ matrix.arrays.username }}"
           -var="ami=${{ matrix.arrays.ami }}"
           -var="ca_cert_path=${{ matrix.arrays.caCertPath }}"
           -var="arc=${{ matrix.arrays.arc }}"
           -var="binary_name=${{ matrix.arrays.binaryName }}"
           -var="local_stack_host_name=${{ needs.StartLocalStack.outputs.local_stack_host_name }}"
+          -var="s3_bucket=${S3_INTEGRATION_BUCKET}"
 
       - name: Terraform destroy
         if: ${{ always() && steps.ec2-linux-integration-test.outputs.cache-hit != 'true' }}
-        run: >
-          terraform destroy --auto-approve
-          -var="ami=${{ matrix.arrays.ami }}"
+        run: terraform destroy --auto-approve
 
   EC2WinIntegrationTest:
     needs: [BuildMSI]
@@ -381,7 +381,7 @@ jobs:
           aws-region: us-west-2
 
       - name: Copy state
-        run: aws s3 cp s3://cloudwatch-agent-integration-bucket/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate .
+        run: aws s3 cp s3://${S3_INTEGRATION_BUCKET}/integration-test/local-stack-terraform-state/${GITHUB_SHA}/terraform.tfstate .
 
       - name: Verify Terraform version
         run: terraform --version

diff --git a/integration/terraform/ec2/linux/README.md b/integration/terraform/ec2/linux/README.md
@@ -1,8 +1,8 @@
 Instance assumptions
 
 1. docker
-    1. starts on start up
-    2. does not require sudo
+   1. starts on start up
+   2. does not require sudo
 2. docker-compose
 3. golang
 4. openssl
@@ -19,3 +19,107 @@ AMI builder pipeline builds the ami
 The pipeline installs required packages and updates ami software
 
 This process generates a new ami we can then use for testing
+
+**How to integration test in your aws account**
+1. Create resources and setup local
+   1. Install terraform
+   2. Set up aws terraform user credentials
+      1. User must include s3, ec2, and iam policy
+      2. Currently, only us-west-2 is support so please add that to your aws config file
+   3. Set up iam role for the ec2 instance
+      1. Role must include CloudWatchAgentServerPolicy and s3 policy that gives both read write access
+      2. Please refer to https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/create-iam-roles-for-cloudwatch-agent.html
+   4. Create s3 bucket
+   5. Create a key pair for ec2
+      1. Must be in the region ec2 instance
+   6. Hint make sure your security group allows ssh, http, and https from ipv4 all
+2. Upload build/bin directory to s3
+   1. make release
+      1. hints
+         1. You may want to do this on a linux ec2 instance installing the agent may fail if you build on Mac
+         2. If you want to build faster run make build && make ${package you want ex package-rpm}
+   2. aws s3 cp build/bin s3://${your bucket name}/integration-test/binary/${git commit sha} --recursive
+   3. This is the agent build packages ex rpm deb
+3. Start Local Stack
+   1. Go to Local Stack directory
+      1. cd ${path to agent dir}/integration/terraform/ec2/localstack
+   2. init terraform
+      1. terraform init
+   3. Apply terraform
+      1. ```
+         terraform apply --auto-approve \
+         -var="github_repo=${gh repo you want to use ex https://github.com/aws/amazon-cloudwatch-agent.git}" \
+         -var="github_sha=${commit sha you want to use ex fb9229b9eaabb42461a4c049d235567f9c0439f8}" \
+         -var='vpc_security_group_ids=["${name of your security group}"]' \
+         -var="key_name=${name of key pair your created}" \
+         -var="s3_bucket=${name of your s3 bucket created}" \
+         -var="iam_instance_profile=${name of your iam role created}" \
+         -var="ssh_key=${your key that you downloaded}"
+         ```
+      2. Write down the dns output that will be important for the next step
+      3. Expected output 
+         1. ```
+            aws_instance.integration-test: Creation complete after 1m47s [id=i-03e33419d42b90325]
+
+            Apply complete! Resources: 1 added, 0 changed, 0 destroyed.
+
+            Outputs:
+
+            public_dns = "ec2-35-87-254-148.us-west-2.compute.amazonaws.com"
+            ec2-35-87-254-148.us-west-2.compute.amazonaws.com
+            Completed 7.0 KiB/7.0 KiB (16.5 KiB/s) with 1 file(s) remaining
+            upload: ./terraform.tfstate to s3://***/integration-test/local-stack-terraform-state/1bc666bc04255402d4516a008bb1095c5d4d27b7/terraform.tfstate
+            ```
+   4. Go back to linux directory
+      2. cd ${path to agent dir}/integration/terraform/ec2/linux
+4. Start the test linux test
+   1. init terraform
+      1. terraform init
+   2. Apply terraform
+      1. ```
+         terraform apply --auto-approve \
+         -var="github_repo=${gh repo you want to use ex https://github.com/aws/amazon-cloudwatch-agent.git}" \
+         -var="github_sha=${commit sha you want to use ex fb9229b9eaabb42461a4c049d235567f9c0439f8}" \
+         -var='vpc_security_group_ids=["${name of your security group}"]' \
+         -var="s3_bucket=${name of your s3 bucket created}" \
+         -var="iam_instance_profile=${name of your iam role created}" \
+         -var="key_name=${name of key pair your created}" \
+         -var="ami=${ami for test you want to use ex cloudwatch-agent-integration-test-ubuntu*}" \
+         -var="user=${log in for the ec2 instance ex ubuntu}" \
+         -var="install_agent=${command to install agent ex dpkg -i -E ./amazon-cloudwatch-agent.deb}" \
+         -var="ca_cert_path=${where the default cert on the ec2 instance ex /etc/ssl/certs/ca-certificates.crt}" \
+         -var="arc=${what arc to use ex amd64}" \
+         -var="binary_name=${binary to install ex amazon-cloudwatch-agent.deb}" \
+         -var="local_stack_host_name=${dns value you got from the local stack terraform apply step}" \
+         -var="test_name=${what you want to call the ec2 instance name}" \
+         -var="ssh_key=${your key that you downloaded}"
+         ```
+   3. Expected Output
+      1. ```
+          aws_instance.integration-test (remote-exec): --- PASS: TestBundle (243.28s)
+          aws_instance.integration-test (remote-exec):     --- PASS: TestBundle/resource_file_location_resources/integration/ssl/with/combine/bundle_find_target_false (60.55s)
+          aws_instance.integration-test (remote-exec):     --- PASS: TestBundle/resource_file_location_resources/integration/ssl/without/bundle/http_find_target_false (60.55s)
+          aws_instance.integration-test (remote-exec):     --- PASS: TestBundle/resource_file_location_resources/integration/ssl/with/original/bundle_find_target_true (61.06s)
+          aws_instance.integration-test (remote-exec):     --- PASS: TestBundle/resource_file_location_resources/integration/ssl/without/bundle_find_target_true (61.13s)
+          aws_instance.integration-test (remote-exec): PASS
+          aws_instance.integration-test (remote-exec): ok  	github.com/aws/amazon-cloudwatch-agent/integration/test/ca_bundle	243.288s
+          aws_instance.integration-test (remote-exec): === RUN   TestEmpty
+          aws_instance.integration-test (remote-exec): --- PASS: TestEmpty (0.00s)
+          aws_instance.integration-test (remote-exec): PASS
+          aws_instance.integration-test (remote-exec): ok  	github.com/aws/amazon-cloudwatch-agent/integration/test/empty	0.002s
+          aws_instance.integration-test (remote-exec): === RUN   TestAgentStatus
+          aws_instance.integration-test: Still creating... [5m30s elapsed]
+          aws_instance.integration-test (remote-exec): --- PASS: TestAgentStatus (6.54s)
+          aws_instance.integration-test (remote-exec): PASS
+          aws_instance.integration-test (remote-exec): ok  	github.com/aws/amazon-cloudwatch-agent/integration/test/sanity	6.541s
+          aws_instance.integration-test: Creation complete after 5m35s [id=i-0f7f77a62c93df010]
+
+          Apply complete! Resources: 1 added, 0 changed, 0 destroyed.         
+         ```
+5. Tear down terraform state
+   1. Tear down test state
+      1. terraform destroy --auto-approve
+   2. Go to local stack directory
+      1. cd ${path to agent dir}/integration/terraform/ec2/localstack
+   3. Tear down localstack state
+      1. terraform destroy --auto-approve
diff --git a/integration/terraform/ec2/linux/main.tf b/integration/terraform/ec2/linux/main.tf
@@ -11,11 +11,11 @@ resource "aws_instance" "integration-test" {
       "git clone ${var.github_repo}",
       "cd amazon-cloudwatch-agent",
       "git reset --hard ${var.github_sha}",
-      "aws s3 cp s3://cloudwatch-agent-integration-bucket/integration-test/binary/${var.github_sha}/linux/${var.arc}/${var.binary_name} .",
+      "aws s3 cp s3://${var.s3_bucket}/integration-test/binary/${var.github_sha}/linux/${var.arc}/${var.binary_name} .",
       "sudo ${var.install_agent}",
       "echo get ssl pem for localstack and export local stack host name",
       "cd ~/amazon-cloudwatch-agent/integration/localstack/ls_tmp",
-      "aws s3 cp s3://cloudwatch-agent-integration-bucket/integration-test/ls_tmp/${var.github_sha} . --recursive",
+      "aws s3 cp s3://${var.s3_bucket}/integration-test/ls_tmp/${var.github_sha} . --recursive",
       "cat ${var.ca_cert_path} > original.pem",
       "cat original.pem snakeoil.pem > combine.pem",
       "sudo cp original.pem /opt/aws/amazon-cloudwatch-agent/original.pem",
@@ -36,7 +36,7 @@ resource "aws_instance" "integration-test" {
 
 data "aws_ami" "latest" {
   most_recent = true
-  owners      = ["self"]
+  owners      = ["self", "506463145083"]
 
   filter {
     name   = "name"

diff --git a/integration/terraform/ec2/linux/variables.tf b/integration/terraform/ec2/linux/variables.tf
@@ -25,7 +25,7 @@ variable "region" {
 
 variable "ami" {
   type = string
-  default = ""
+  default = "cloudwatch-agent-integration-test-ubuntu*"
 }
 
 variable "ssh_key" {
@@ -48,12 +48,6 @@ variable "github_repo" {
   default = ""
 }
 
-variable "package" {
-  description = "make command of package to build ex package-deb"
-  type = string
-  default = ""
-}
-
 variable "install_agent" {
   description = "command of package to install ex dpkg -i -E ./amazon-cloudwatch-agent.deb"
   type = string
@@ -78,4 +72,9 @@ variable "binary_name" {
 variable "local_stack_host_name" {
   type = string
   default = "localhost.localstack.cloud"
+}
+
+variable "s3_bucket" {
+  type = string
+  default = ""
 }
diff --git a/integration/terraform/ec2/localstack/main.tf b/integration/terraform/ec2/localstack/main.tf
@@ -19,7 +19,7 @@ resource "aws_instance" "integration-test" {
       "cat snakeoil.pem > server.test.pem.crt",
       "cd ~/amazon-cloudwatch-agent/integration/localstack",
       "docker-compose up -d --force-recreate",
-      "aws s3 cp ls_tmp s3://cloudwatch-agent-integration-bucket/integration-test/ls_tmp/${var.github_sha} --recursive"
+      "aws s3 cp ls_tmp s3://${var.s3_bucket}/integration-test/ls_tmp/${var.github_sha} --recursive"
     ]
     connection {
       type        = "ssh"
@@ -36,7 +36,7 @@ resource "aws_instance" "integration-test" {
 
 data "aws_ami" "latest" {
   most_recent = true
-  owners      = ["self"]
+  owners      = ["self", "506463145083"]
 
   filter {
     name   = "name"

diff --git a/integration/terraform/ec2/localstack/variables.tf b/integration/terraform/ec2/localstack/variables.tf
@@ -38,6 +38,11 @@ variable "github_repo" {
   default = ""
 }
 
+variable "s3_bucket" {
+  type = string
+  default = ""
+}
+
 output "public_dns" {
   description = "The public DNS name assigned to the instance. For EC2-VPC, this is only available if you've enabled DNS hostnames for your VPC"
   value       = aws_instance.integration-test.public_dns