From be2a6bd5ee17b9802caa3e0591b81198c0c3cecb Mon Sep 17 00:00:00 2001 From: Khanh Nguyen <91758108+khanhntd@users.noreply.github.com> Date: Fri, 6 May 2022 09:41:50 -0400 Subject: [PATCH] Add ECS Metadata Integration Test for detect changes in ECS Container Agent Metadata Endpoint (#458) * Initial commit for ecs * Add basic for fargate with terraform * Fix typo for CWAgent config * Add some tpl files for task def * Add new policy * Add basic components for CWAgent fargate on terraform * Finish formating terraform * Add basic testing * Add basic testing * fix some constraint * change to us west-2 * Add emf processor for config and sample app * Add redis sample app * Add redis sample app * Add basic testing for ecs fargate * reduce variables * Add readme doc and use make fmt * Add basic integration test on github workflow * delete xtool * Fix json config for default and ecs * Add ecs linux fargate and integration test * Change image repo again * Test integration test workflow * Delete branchs for testing * Test workflow * Fix tag for build docker image * Change back to format * Add back build constraints * Change to use test cwl * Change name to ecs fargate * Change to path go.mod * Fix variables and fix default extra app * Delete retry for testing * Add lock * Add unique for service role * Test and fixing readme * Add security group * Fix path testing * Test ecs * test for success path * Test again with go * Test again for constraints * Test again * Add basic testing * Change to cwagent integration test * Add back from source * Fix deprecated vpc and fix cloudwatch agent config issue * Fix cloudwatch agent issues * Add back check_secrets * Add back ECS fargate --- .github/pull_request_template.md | 2 +- .github/workflows/integrationTest.yml | 123 ++++++++++++++- go.mod | 4 +- go.sum | 8 +- ...matrix.json => ec2_linux_test_matrix.json} | 0 .../resources/ec2_mac_test_matrix.json | 2 + ...trix.json => ec2_windows_test_matrix.json} | 0 ...trix.json => ecs_fargate_test_matrix.json} | 0 integration/generator/test_case_generator.go | 16 +- integration/terraform/Makefile | 5 + integration/terraform/ecs/linux/README.md | 64 ++++++++ .../default_amazon_cloudwatch_agent.json | 31 ++++ .../default_ecs_prometheus.tpl | 8 + .../default_resources/default_ecs_taskdef.tpl | 29 ++++ .../default_resources/default_extra_apps.tpl | 57 +++++++ integration/terraform/ecs/linux/iam.tf | 94 ++++++++++++ integration/terraform/ecs/linux/main.tf | 142 ++++++++++++++++++ integration/terraform/ecs/linux/providers.tf | 3 + integration/terraform/ecs/linux/variables.tf | 19 +++ integration/terraform/ecs/linux/vpc.tf | 28 ++++ integration/test/agent_util.go | 5 +- integration/test/cwl_util.go | 29 +++- .../ecs/ecs_metadata/ecs_metadata_test.go | 49 ++++++ .../ecs/ecs_metadata/resources/config.json | 52 +++++++ .../ecs_metadata/resources/ecs_prometheus.tpl | 8 + .../ecs_metadata/resources/ecs_taskdef.tpl | 29 ++++ .../ecs/ecs_metadata/resources/extra_apps.tpl | 57 +++++++ 27 files changed, 848 insertions(+), 16 deletions(-) rename integration/generator/resources/{linux_test_matrix.json => ec2_linux_test_matrix.json} (100%) create mode 100644 integration/generator/resources/ec2_mac_test_matrix.json rename integration/generator/resources/{windows_test_matrix.json => ec2_windows_test_matrix.json} (100%) rename integration/generator/resources/{mac_test_matrix.json => ecs_fargate_test_matrix.json} (100%) create mode 100644 integration/terraform/Makefile create mode 100644 integration/terraform/ecs/linux/README.md create mode 100644 integration/terraform/ecs/linux/default_resources/default_amazon_cloudwatch_agent.json create mode 100644 integration/terraform/ecs/linux/default_resources/default_ecs_prometheus.tpl create mode 100644 integration/terraform/ecs/linux/default_resources/default_ecs_taskdef.tpl create mode 100644 integration/terraform/ecs/linux/default_resources/default_extra_apps.tpl create mode 100644 integration/terraform/ecs/linux/iam.tf create mode 100644 integration/terraform/ecs/linux/main.tf create mode 100644 integration/terraform/ecs/linux/providers.tf create mode 100644 integration/terraform/ecs/linux/variables.tf create mode 100644 integration/terraform/ecs/linux/vpc.tf create mode 100644 integration/test/ecs/ecs_metadata/ecs_metadata_test.go create mode 100644 integration/test/ecs/ecs_metadata/resources/config.json create mode 100644 integration/test/ecs/ecs_metadata/resources/ecs_prometheus.tpl create mode 100644 integration/test/ecs/ecs_metadata/resources/ecs_taskdef.tpl create mode 100644 integration/test/ecs/ecs_metadata/resources/extra_apps.tpl diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 22ba8e084d..6cd460eb18 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -10,7 +10,7 @@ By submitting this pull request, I confirm that you can use, modify, copy, and r # Tests _Describe what tests you have done._ -#Requirements +# Requirements _Before commit the code, please do the following steps._ 1. Run `make fmt` and `make fmt-sh` 2. Run `make linter` diff --git a/.github/workflows/integrationTest.yml b/.github/workflows/integrationTest.yml index 49e23f262f..11f8f41131 100644 --- a/.github/workflows/integrationTest.yml +++ b/.github/workflows/integrationTest.yml @@ -14,11 +14,13 @@ env: PASSPHRASE: ${{ secrets.PASSPHRASE }} GPG_KEY_NAME: ${{ secrets.GPG_KEY_NAME }} GPG_TTY: $(tty) + ECR_INTEGRATION_TEST_REPO: "cwagent-integration-test" on: push: branches: - master + workflow_dispatch: concurrency: @@ -26,6 +28,59 @@ concurrency: cancel-in-progress: true jobs: + MakeDockerImage: + name: 'MakeDockerImage' + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v3 + + - name: Cache if success + id: build-docker-image + uses: actions/cache@v2 + with: + path: | + RELEASE_NOTES + key: build-docker-image-${{ github.run_id }} + + - name: Configure AWS Credentials + if: steps.build-docker-image.outputs.cache-hit != 'true' + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + aws-region: us-west-2 + + - name: Login ECR + if: steps.build-docker-image.outputs.cache-hit != 'true' + id: login-ecr + uses: aws-actions/amazon-ecr-login@v1 + + - name: Set up Docker Buildx + if: steps.build-docker-image.outputs.cache-hit != 'true' + uses: docker/setup-buildx-action@v1 + + - name: Set up QEMU + if: steps.build-docker-image.outputs.cache-hit != 'true' + uses: docker/setup-qemu-action@v1 + + #Build the cloudwatch agent image for two primary reasons: + #-Using the cloudwatch agent image to do the integration test (can be used for internal) + #-Export it for the future use in CD release pipeline + #Documentation: https://github.com/docker/build-push-action + - name: Build Cloudwatch Agent Image + uses: docker/build-push-action@v2 + if: steps.build-docker-image.outputs.cache-hit != 'true' + with: + file: amazon-cloudwatch-container-insights/cloudwatch-agent-dockerfile/source/Dockerfile + context: . + push: true + tags: | + ${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}:${{ github.sha }} + build-args: BUILDMODE=copy + cache-from: type=registry + cache-to: type=inline + platforms: linux/amd64, linux/arm64 + MakeBinary: name: 'MakeBinary' runs-on: ubuntu-latest @@ -92,6 +147,7 @@ jobs: outputs: ec2_linux_matrix: ${{ steps.set-matrix.outputs.ec2_linux_matrix }} ec2_windows_matrix: ${{ steps.set-matrix.outputs.ec2_windows_matrix }} + ecs_fargate_matrix: ${{ steps.set-matrix.outputs.ecs_fargate_matrix }} steps: - uses: actions/checkout@v2 @@ -105,13 +161,15 @@ jobs: id: set-matrix run: | go run --tags=generator integration/generator/test_case_generator.go - echo "::set-output name=ec2_linux_matrix::$(echo $(cat integration/generator/resources/linux_complete_test_matrix.json))" - echo "::set-output name=ec2_windows_matrix::$(echo $(cat integration/generator/resources/windows_complete_test_matrix.json))" + echo "::set-output name=ec2_linux_matrix::$(echo $(cat integration/generator/resources/ec2_linux_complete_test_matrix.json))" + echo "::set-output name=ec2_windows_matrix::$(echo $(cat integration/generator/resources/ec2_windows_complete_test_matrix.json))" + echo "::set-output name=ecs_fargate_matrix::$(echo $(cat integration/generator/resources/ecs_fargate_complete_test_matrix.json))" - name: Echo test plan matrix run: | echo ${{ steps.set-matrix.outputs.ec2_linux_matrix }} echo ${{ steps.set-matrix.outputs.ec2_windows_matrix }} + echo ${{ steps.set-matrix.outputs.ecs_fargate_matrix }} MakeMSIZip: name: 'MakeMSIZip' @@ -499,4 +557,63 @@ jobs: run: terraform init - name: Terraform destroy - run: terraform destroy --auto-approve \ No newline at end of file + run: terraform destroy --auto-approve + + ECSFargateIntegrationTest: + name: 'ECSFargateIntegrationTest' + runs-on: ubuntu-latest + needs: [MakeDockerImage, GenerateTestMatrix] + strategy: + fail-fast: false + matrix: + arrays: ${{ fromJson(needs.GenerateTestMatrix.outputs.ecs_fargate_matrix) }} + steps: + - uses: actions/checkout@v2 + + - name: Configure AWS Credentials + uses: aws-actions/configure-aws-credentials@v1 + with: + aws-access-key-id: ${{ secrets.TERRAFORM_AWS_ACCESS_KEY_ID }} + aws-secret-access-key: ${{ secrets.TERRAFORM_AWS_SECRET_ACCESS_KEY }} + aws-region: us-east-2 + + - name: Cache if success + id: ecs-fargate-integration-test + uses: actions/cache@v2 + with: + path: go.mod + key: ecs-fargate-integration-test-${{ github.sha }}-${{ matrix.arrays.os }} + + - name: Login ECR + id: login-ecr + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + uses: aws-actions/amazon-ecr-login@v1 + + - name: Verify Terraform version + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + run: terraform --version + + - name: Terraform apply + if: steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + uses: nick-invision/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 15 + retry_wait_seconds: 5 + command: | + cd integration/terraform/ecs/linux + terraform init + if terraform apply --auto-approve -var="test_dir=${{ matrix.arrays.test_dir }}" -var="cwagent_image_repo=${{ steps.login-ecr.outputs.registry }}/${{ env.ECR_INTEGRATION_TEST_REPO }}" -var="cwagent_image_tag=${{ github.sha }}" ; then + terraform destroy -auto-approve + else + terraform destroy -auto-approve && exit 1 + fi + + - name: Terraform destroy + if: ${{ cancelled() && steps.ecs-fargate-integration-test.outputs.cache-hit != 'true' + uses: nick-invision/retry@v2 + with: + max_attempts: 3 + timeout_minutes: 8 + retry_wait_seconds: 5 + command: cd integration/terraform/ecs/linux && terraform destroy --auto-approve \ No newline at end of file diff --git a/go.mod b/go.mod index 86382b2f22..167b63eaa6 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/Jeffail/gabs v1.4.0 github.com/aws/aws-sdk-go v1.30.15 - github.com/aws/aws-sdk-go-v2 v1.16.2 + github.com/aws/aws-sdk-go-v2 v1.16.3 github.com/aws/aws-sdk-go-v2/config v1.15.3 github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.12.3 github.com/aws/aws-sdk-go-v2/service/cloudwatch v1.18.1 @@ -68,7 +68,7 @@ require ( github.com/prometheus/common v0.9.1 github.com/prometheus/prometheus v1.8.2-0.20200420081721-18254838fbe2 github.com/shirou/gopsutil v2.20.5+incompatible - github.com/stretchr/testify v1.5.1 + github.com/stretchr/testify v1.7.0 github.com/xeipuuv/gojsonschema v1.2.0 golang.org/x/net v0.0.0-20200301022130-244492dfa37a golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a diff --git a/go.sum b/go.sum index 34f7ed09ac..a2bd1e60e6 100644 --- a/go.sum +++ b/go.sum @@ -158,8 +158,9 @@ github.com/aws/aws-sdk-go v1.30.15 h1:Sd8QDVzzE8Sl+xNccmdj0HwMrFowv6uVUx9tGsCE1Z github.com/aws/aws-sdk-go v1.30.15/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU8YkpAk0= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aws/aws-sdk-go-v2 v1.13.0/go.mod h1:L6+ZpqHaLbAaxsqV0L4cvxZY7QupWJB4fhkf8LXvC7w= -github.com/aws/aws-sdk-go-v2 v1.16.2 h1:fqlCk6Iy3bnCumtrLz9r3mJ/2gUT0pJ0wLFVIdWh+JA= github.com/aws/aws-sdk-go-v2 v1.16.2/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= +github.com/aws/aws-sdk-go-v2 v1.16.3 h1:0W1TSJ7O6OzwuEvIXAtJGvOeQ0SGAhcpxPN2/NK5EhM= +github.com/aws/aws-sdk-go-v2 v1.16.3/go.mod h1:ytwTPBG6fXTZLxxeeCCWj2/EMYp/xDUgX+OET6TLNNU= github.com/aws/aws-sdk-go-v2/config v1.15.3 h1:5AlQD0jhVXlGzwo+VORKiUuogkG7pQcLJNzIzK7eodw= github.com/aws/aws-sdk-go-v2/config v1.15.3/go.mod h1:9YL3v07Xc/ohTsxFXzan9ZpFpdTOFl4X65BAKYaz8jg= github.com/aws/aws-sdk-go-v2/credentials v1.11.2 h1:RQQ5fzclAKJyY5TvF+fkjJEwzK4hnxQCLOu5JXzDmQo= @@ -1028,8 +1029,9 @@ github.com/stretchr/testify v0.0.0-20151208002404-e3a8ff8ce365/go.mod h1:a8OnRci github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= -github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0TYG7HtkIgExQo+2RdLuwRft63jn2HWj8= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= @@ -1471,6 +1473,8 @@ gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v3 v3.0.0-20200121175148-a6ecf24a6d71/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/integration/generator/resources/linux_test_matrix.json b/integration/generator/resources/ec2_linux_test_matrix.json similarity index 100% rename from integration/generator/resources/linux_test_matrix.json rename to integration/generator/resources/ec2_linux_test_matrix.json diff --git a/integration/generator/resources/ec2_mac_test_matrix.json b/integration/generator/resources/ec2_mac_test_matrix.json new file mode 100644 index 0000000000..32960f8ced --- /dev/null +++ b/integration/generator/resources/ec2_mac_test_matrix.json @@ -0,0 +1,2 @@ +[ +] \ No newline at end of file diff --git a/integration/generator/resources/windows_test_matrix.json b/integration/generator/resources/ec2_windows_test_matrix.json similarity index 100% rename from integration/generator/resources/windows_test_matrix.json rename to integration/generator/resources/ec2_windows_test_matrix.json diff --git a/integration/generator/resources/mac_test_matrix.json b/integration/generator/resources/ecs_fargate_test_matrix.json similarity index 100% rename from integration/generator/resources/mac_test_matrix.json rename to integration/generator/resources/ecs_fargate_test_matrix.json diff --git a/integration/generator/test_case_generator.go b/integration/generator/test_case_generator.go index 3abccd67ae..456c0e5f08 100644 --- a/integration/generator/test_case_generator.go +++ b/integration/generator/test_case_generator.go @@ -20,12 +20,17 @@ const ( //you can't have a const map in golang var osToTestDirMap = map[string][]string{ - linux: {"./integration/test/ca_bundle", + "ec2_linux": { + "./integration/test/ca_bundle", "./integration/test/cloudwatchlogs", - "./integration/test/metrics_number_dimension"}, + "./integration/test/metrics_number_dimension", + }, // @TODO add real tests - windows: {""}, - mac: {}, + "ec2_windows": {""}, + "ec2_mac": {}, + "ecs_fargate": { + "./integration/test/ecs/ecs_metadata", + }, } func main() { @@ -37,11 +42,14 @@ func main() { func genMatrix(targetOS string, testDirList []string) []map[string]string { openTestMatrix, err := os.Open(fmt.Sprintf("integration/generator/resources/%v_test_matrix.json", targetOS)) + if err != nil { log.Panicf("can't read file %v_test_matrix.json err %v", targetOS, err) } + byteValueTestMatrix, _ := ioutil.ReadAll(openTestMatrix) _ = openTestMatrix.Close() + var testMatrix []map[string]string err = json.Unmarshal(byteValueTestMatrix, &testMatrix) if err != nil { diff --git a/integration/terraform/Makefile b/integration/terraform/Makefile new file mode 100644 index 0000000000..4e765a6c3e --- /dev/null +++ b/integration/terraform/Makefile @@ -0,0 +1,5 @@ +fmt: + terraform fmt -recursive + +check-fmt: + terraform fmt -recursive -check diff --git a/integration/terraform/ecs/linux/README.md b/integration/terraform/ecs/linux/README.md new file mode 100644 index 0000000000..531c83b440 --- /dev/null +++ b/integration/terraform/ecs/linux/README.md @@ -0,0 +1,64 @@ +Running ECS Fargate Integration Tests +========================= + +## 1. How ECS Fargate are set up? +**Step 1:** Create a Fargate ECS Cluster with the default VPC Network. +**Step 2:** Create a security group to assign to the service in step 5 which allows all inbound +traffics and outbound traffics +**Step 3:** Create a IAM Role and IAM Execution Role for the containers to pull the image and +execute their purposes +**Step 4:** Create a [task definition](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/task_definitions.html) +to decide which containers serve a specific task and assign the IAM roles in step 3 to the containers +**Step 5:** Create a [service](https://docs.aws.amazon.com/AmazonECS/latest/developerguide/ecs_services.html) which configure +how many tasks are running in parallel and ensure availability of the task. + +## 2. Setup resources +By running `terraform apply -auto-approve -lock=false`, +you agree to setup the following resources: +* 1 IAM Task Role and 1 Execution Task Role (similar to [these IAM Roles](https://docs.aws.amazon.com/AmazonCloudWatch/latest/monitoring/deploy_servicelens_CloudWatch_agent_deploy_ECS.html)) +* 2 SSM Parameter Store +* 2 Task Definitions and 2 Services for those task definitions +* 1 Security group which allows all inbound and outbound traffics. + +To be more specifically, +* **IAM Task Role:** Contain the following policy + * **CloudWatchAgentPolicy:** CloudWatchAgent's related actions + * **service_discovery_police:** For describe ECS tasks and services +```json +{ + "Statement": [ + { + "Action": [ + "ecs:ListTasks", + "ecs:ListServices", + "ecs:DescribeTasks", + "ecs:DescribeTaskDefinition", + "ecs:DescribeServices", + "ecs:DescribeContainerInstances", + "ec2:DescribeInstances" + ], + "Effect": "Allow", + "Resource": "*", + "Sid": "" + } + ], + "Version": "2012-10-17" +} +``` + +* **IAM Execution Task Role:** Contain the following policy + * **AmazonECSTaskExecutionRolePolicy:** Pull CloudWatch Agent's image and extra app's image from ECR. + * **AmazonSSMReadOnlyAccess:** Pull Cloudwatch Agent's and Prometheus's config from SSM Parameter Store. +* **CloudWatchAgent Parameter Store:** Store CloudWatchAgent's configuration and CloudWatchAgent will pull the config from there. [Example configuration](default_resources/default_amazon_cloudwatch_agent.json) +* **Prometheus Parameter Store:** Store Prometheus's configuration and CloudWatchAgent will pull the config from there. [Example configuration](default_resources/default_ecs_prometheus.tpl) + +## 3. Run tests in your AWS account +```` +cd integration/terraform/ecs && terraform init && terraform apply -auto-approve \ + -var="test_dir={{your test case folder name}} \ +```` + +Don't forget to clean up your resources after integration test has passed: +```` +terraform destroy -auto-approve +```` \ No newline at end of file diff --git a/integration/terraform/ecs/linux/default_resources/default_amazon_cloudwatch_agent.json b/integration/terraform/ecs/linux/default_resources/default_amazon_cloudwatch_agent.json new file mode 100644 index 0000000000..c858540bf2 --- /dev/null +++ b/integration/terraform/ecs/linux/default_resources/default_amazon_cloudwatch_agent.json @@ -0,0 +1,31 @@ +{ + "logs": { + "metrics_collected": { + "prometheus": { + "prometheus_config_path": "env:PROMETHEUS_CONFIG_CONTENT", + "ecs_service_discovery": { + "sd_frequency": "1m", + "sd_result_file": "/tmp/cwagent_ecs_auto_sd.yaml", + "docker_label": {} + }, + "emf_processor": { + "metric_declaration": [ + { + "source_labels": ["container_name"], + "label_matcher": "^redis-exporter-.*$", + "dimensions": [["ClusterName","TaskDefinitionFamily"]], + "metric_selectors": [ + "^redis_net_(in|out)put_bytes_total$", + "^redis_(expired|evicted)_keys_total$", + "^redis_keyspace_(hits|misses)_total$", + "^redis_memory_used_bytes$", + "^redis_connected_clients$" + ] + } + ] + } + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/integration/terraform/ecs/linux/default_resources/default_ecs_prometheus.tpl b/integration/terraform/ecs/linux/default_resources/default_ecs_prometheus.tpl new file mode 100644 index 0000000000..a3149e8801 --- /dev/null +++ b/integration/terraform/ecs/linux/default_resources/default_ecs_prometheus.tpl @@ -0,0 +1,8 @@ +global: + scrape_interval: 1m + scrape_timeout: 10s +scrape_configs: + - job_name: cwagent-ecs-file-sd-config + sample_limit: 10000 + file_sd_configs: + - files: [ "/tmp/cwagent_ecs_auto_sd.yaml" ] \ No newline at end of file diff --git a/integration/terraform/ecs/linux/default_resources/default_ecs_taskdef.tpl b/integration/terraform/ecs/linux/default_resources/default_ecs_taskdef.tpl new file mode 100644 index 0000000000..32d22f5089 --- /dev/null +++ b/integration/terraform/ecs/linux/default_resources/default_ecs_taskdef.tpl @@ -0,0 +1,29 @@ +[ + { + "name": "cloudwatch_agent", + "image": "${cwagent_image}", + "essential": true, + "secrets": [ + { + "name": "CW_CONFIG_CONTENT", + "valueFrom": "${cwagent_ssm_parameter_arn}" + }, + { + "name": "PROMETHEUS_CONFIG_CONTENT", + "valueFrom": "${prometheus_ssm_parameter_arn}" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "mountPoints": [], + "memory": 2048, + "volumesFrom": [] + } +] diff --git a/integration/terraform/ecs/linux/default_resources/default_extra_apps.tpl b/integration/terraform/ecs/linux/default_resources/default_extra_apps.tpl new file mode 100644 index 0000000000..7535aed749 --- /dev/null +++ b/integration/terraform/ecs/linux/default_resources/default_extra_apps.tpl @@ -0,0 +1,57 @@ +[ + { + "name": "redis-0", + "image": "redis:6.0.8-alpine3.12", + "essential": true, + "portMappings": [ + { + "protocol": "tcp", + "containerPort": 6379 + } + ], + "dockerLabels": { + "app": "redis" + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "memcached-tutorial", + "awslogs-group": "${log_group}" + } + }, + "cpu": 128, + "mountPoints": [ ], + "memory": 512, + "volumesFrom": [ ] + }, + { + "name": "redis-exporter-0", + "image": "oliver006/redis_exporter:v1.11.1-alpine", + "essential": true, + "portMappings": [ + { + "protocol": "tcp", + "containerPort": 9121 + } + ], + "dockerLabels": { + "CWAgent-Usage-invalid-prometheus-label": "Prometheus-Monitoring-Workload-Demo", + "ECS_PROMETHEUS_EXPORTER_PORT": "9121", + "job": "prometheus-redis", + "app_x": "redis_exporter" + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "memcached-exporter-tutorial", + "awslogs-group": "${log_group}" + } + }, + "cpu": 128, + "mountPoints": [ ], + "memory": 512, + "volumesFrom": [ ] + } +] \ No newline at end of file diff --git a/integration/terraform/ecs/linux/iam.tf b/integration/terraform/ecs/linux/iam.tf new file mode 100644 index 0000000000..15c53af3f9 --- /dev/null +++ b/integration/terraform/ecs/linux/iam.tf @@ -0,0 +1,94 @@ +resource "aws_iam_role" "ecs_task_role" { + name = "cwagent-integ-test-task-role-${random_id.testing_id.hex}" + + assume_role_policy = < 0 { + return true + } + + return false +} + // getCloudWatchLogsClient returns a singleton SDK client for interfacing with CloudWatch Logs func getCloudWatchLogsClient() (*cloudwatchlogs.Client, *context.Context, error) { if cwl == nil { diff --git a/integration/test/ecs/ecs_metadata/ecs_metadata_test.go b/integration/test/ecs/ecs_metadata/ecs_metadata_test.go new file mode 100644 index 0000000000..a818c16ac2 --- /dev/null +++ b/integration/test/ecs/ecs_metadata/ecs_metadata_test.go @@ -0,0 +1,49 @@ +// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved. +// SPDX-License-Identifier: MIT + +//go:build integration +// +build integration + +package ecs_metadata + +import ( + "flag" + "fmt" + "log" + "testing" + "time" + "github.com/aws/amazon-cloudwatch-agent/integration/test" +) + +// Purpose: Detect the changes in metadata endpoint for ECS Container Agent https://github.com/aws/amazon-cloudwatch-agent/blob/master/translator/util/ecsutil/ecsutil.go#L67-L75 +// Implementation: Checking if a log group's the format(https://github.com/aws/amazon-cloudwatch-agent/blob/master/translator/translate/logs/metrics_collected/prometheus/ruleLogGroupName.go#L33) +// exists or not since the log group's format has the scrapping cluster name from metadata endpoint. + +const ( + RetryTime = 15 + // Log group format: https://github.com/aws/amazon-cloudwatch-agent/blob/master/translator/translate/logs/metrics_collected/prometheus/ruleLogGroupName.go#L33 + ECSLogGroupNameFormat = "/aws/ecs/containerinsights/%s" + // Log stream based on job name: https://github.com/khanhntd/amazon-cloudwatch-agent/blob/ecs_metadata/integration/test/ecs/ecs_metadata/resources/extra_apps.tpl#L41 + LogStreamName = "prometheus-redis" +) + +var clusterName = flag.String("clusterName", "", "Please provide the os preference, valid value: windows/linux.") + +func TestValidatingCloudWatchLogs(t *testing.T) { + logGroupName := fmt.Sprintf(ECSLogGroupNameFormat, *clusterName) + + for currentRetry := 1; ; currentRetry++ { + + if currentRetry == RetryTime { + t.Fatalf("Test metadata has exhausted %v retry time", RetryTime) + } + + if test.IsLogGroupExists(t,logGroupName) { + test.DeleteLogGroupAndStream(logGroupName,LogStreamName) + break + } + + log.Printf("Current retry: %v/%v and begin to sleep for 20s \n", currentRetry, RetryTime) + time.Sleep(20 * time.Second) + } +} diff --git a/integration/test/ecs/ecs_metadata/resources/config.json b/integration/test/ecs/ecs_metadata/resources/config.json new file mode 100644 index 0000000000..f75b339f16 --- /dev/null +++ b/integration/test/ecs/ecs_metadata/resources/config.json @@ -0,0 +1,52 @@ +{ + "agent": { + "metrics_collection_interval": 60, + "run_as_user": "root", + "debug": true + }, + "logs": { + "metrics_collected": { + "prometheus": { + "prometheus_config_path": "env:PROMETHEUS_CONFIG_CONTENT", + "ecs_service_discovery": { + "sd_frequency": "1m", + "sd_result_file": "/tmp/cwagent_ecs_auto_sd.yaml", + "docker_label": {} + }, + "emf_processor": { + "metric_declaration": [ + { + "source_labels": ["container_name"], + "label_matcher": "^redis-exporter-.*$", + "dimensions": [["ClusterName","TaskDefinitionFamily"]], + "metric_selectors": [ + "^redis_net_(in|out)put_bytes_total$", + "^redis_(expired|evicted)_keys_total$", + "^redis_keyspace_(hits|misses)_total$", + "^redis_memory_used_bytes$", + "^redis_connected_clients$" + ] + }, + { + "source_labels": ["container_name"], + "label_matcher": "^redis-exporter-.*$", + "dimensions": [["ClusterName","TaskDefinitionFamily","cmd"]], + "metric_selectors": [ + "^redis_commands_total$" + ] + }, + { + "source_labels": ["container_name"], + "label_matcher": "^redis-exporter-.*$", + "dimensions": [["ClusterName","TaskDefinitionFamily","db"]], + "metric_selectors": [ + "^redis_db_keys$" + ] + } + ] + } + } + }, + "force_flush_interval": 5 + } +} \ No newline at end of file diff --git a/integration/test/ecs/ecs_metadata/resources/ecs_prometheus.tpl b/integration/test/ecs/ecs_metadata/resources/ecs_prometheus.tpl new file mode 100644 index 0000000000..a3149e8801 --- /dev/null +++ b/integration/test/ecs/ecs_metadata/resources/ecs_prometheus.tpl @@ -0,0 +1,8 @@ +global: + scrape_interval: 1m + scrape_timeout: 10s +scrape_configs: + - job_name: cwagent-ecs-file-sd-config + sample_limit: 10000 + file_sd_configs: + - files: [ "/tmp/cwagent_ecs_auto_sd.yaml" ] \ No newline at end of file diff --git a/integration/test/ecs/ecs_metadata/resources/ecs_taskdef.tpl b/integration/test/ecs/ecs_metadata/resources/ecs_taskdef.tpl new file mode 100644 index 0000000000..fe93b1ec4b --- /dev/null +++ b/integration/test/ecs/ecs_metadata/resources/ecs_taskdef.tpl @@ -0,0 +1,29 @@ +[ + { + "name": "cloudwatch_agent", + "image": "${cwagent_image}", + "essential": true, + "secrets": [ + { + "name": "CW_CONFIG_CONTENT", + "valueFrom": "${cwagent_ssm_parameter_arn}" + }, + { + "name": "PROMETHEUS_CONFIG_CONTENT", + "valueFrom": "${prometheus_ssm_parameter_arn}" + } + ], + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "${testing_id}", + "awslogs-group": "${log_group}" + } + }, + "cpu": 1, + "mountPoints": [], + "memory": 1024, + "volumesFrom": [] + } +] diff --git a/integration/test/ecs/ecs_metadata/resources/extra_apps.tpl b/integration/test/ecs/ecs_metadata/resources/extra_apps.tpl new file mode 100644 index 0000000000..7535aed749 --- /dev/null +++ b/integration/test/ecs/ecs_metadata/resources/extra_apps.tpl @@ -0,0 +1,57 @@ +[ + { + "name": "redis-0", + "image": "redis:6.0.8-alpine3.12", + "essential": true, + "portMappings": [ + { + "protocol": "tcp", + "containerPort": 6379 + } + ], + "dockerLabels": { + "app": "redis" + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "memcached-tutorial", + "awslogs-group": "${log_group}" + } + }, + "cpu": 128, + "mountPoints": [ ], + "memory": 512, + "volumesFrom": [ ] + }, + { + "name": "redis-exporter-0", + "image": "oliver006/redis_exporter:v1.11.1-alpine", + "essential": true, + "portMappings": [ + { + "protocol": "tcp", + "containerPort": 9121 + } + ], + "dockerLabels": { + "CWAgent-Usage-invalid-prometheus-label": "Prometheus-Monitoring-Workload-Demo", + "ECS_PROMETHEUS_EXPORTER_PORT": "9121", + "job": "prometheus-redis", + "app_x": "redis_exporter" + }, + "logConfiguration": { + "logDriver": "awslogs", + "options": { + "awslogs-region": "${region}", + "awslogs-stream-prefix": "memcached-exporter-tutorial", + "awslogs-group": "${log_group}" + } + }, + "cpu": 128, + "mountPoints": [ ], + "memory": 512, + "volumesFrom": [ ] + } +] \ No newline at end of file