From f01a152ec1500b38db626774fa2eab51563c649b Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Mon, 17 May 2021 15:11:53 +0100 Subject: [PATCH 1/7] Adding additional files to support project --- .classpath | 37 ++++++++++++ .gitignore | 5 ++ .project | 18 ++++++ .settings/org.eclipse.jdt.core.prefs | 8 +++ Dockerfile | 14 +++++ build.sh | 4 ++ buildspec.yml | 53 ++++++++++++++++++ codeguru-java-app-deployment-with-issues.yaml | 20 +++++++ ...ru-java-app-deployment-without-issues.yaml | 20 +++++++ pom.xml | 56 ++++++++++++++++--- .../demoapplication/AwsCredsProvider.java | 29 ++++++++++ .../com/company/demoapplication/Main.java | 38 ++++++++----- 12 files changed, 280 insertions(+), 22 deletions(-) create mode 100644 .classpath create mode 100644 .project create mode 100644 .settings/org.eclipse.jdt.core.prefs create mode 100644 Dockerfile create mode 100644 build.sh create mode 100644 buildspec.yml create mode 100644 codeguru-java-app-deployment-with-issues.yaml create mode 100644 codeguru-java-app-deployment-without-issues.yaml create mode 100644 src/main/java/com/company/demoapplication/AwsCredsProvider.java diff --git a/.classpath b/.classpath new file mode 100644 index 0000000..b0a1bea --- /dev/null +++ b/.classpath @@ -0,0 +1,37 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/.gitignore b/.gitignore index 2f7896d..df00114 100644 --- a/.gitignore +++ b/.gitignore @@ -1 +1,6 @@ target/ +/bin/ +**/*.jar +*.jar +.DS_Store + diff --git a/.project b/.project new file mode 100644 index 0000000..1ad0e60 --- /dev/null +++ b/.project @@ -0,0 +1,18 @@ + + + DemoApplication + NO_M2ECLIPSE_SUPPORT: Project files created with the maven-eclipse-plugin are not supported in M2Eclipse. + + + + org.eclipse.jdt.core.javabuilder + + + org.eclipse.m2e.core.maven2Builder + + + + org.eclipse.m2e.core.maven2Nature + org.eclipse.jdt.core.javanature + + \ No newline at end of file diff --git a/.settings/org.eclipse.jdt.core.prefs b/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 0000000..2f5cc74 --- /dev/null +++ b/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,8 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..0a22282 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,14 @@ +# Dockerfile +FROM 338918620411.dkr.ecr.eu-west-1.amazonaws.com/openjdk8:latest +ARG ISSUE_FLAG +ARG IAM_ROLE +ARG SQS_URL +ARG S3_BUCKET +ENV ISSUE_FLAG_ENV=${ISSUE_FLAG} +ENV IAM_ROLE_ENV=${IAM_ROLE} +ENV SQS_URL_ENV=${SQS_URL} +ENV S3_BUCKET_ENV=${S3_BUCKET} +ENV DEMO_HOME=/usr/demo +COPY DemoApplication-1.0-jar-with-dependencies.jar $DEMO_HOME/ +WORKDIR $DEMO_HOME +CMD java -jar DemoApplication-1.0-jar-with-dependencies.jar ${ISSUE_FLAG_ENV} ${IAM_ROLE_ENV} ${SQS_URL_ENV} ${S3_BUCKET_ENV} \ No newline at end of file diff --git a/build.sh b/build.sh new file mode 100644 index 0000000..592092b --- /dev/null +++ b/build.sh @@ -0,0 +1,4 @@ +#!bin/bash +mvn clean package +cp target/DemoApplication-1.0-jar-with-dependencies.jar . +aws s3 cp target/DemoApplication-1.0-jar-with-dependencies.jar s3://338918620411-account-bucket/DemoApplication-1.0-jar-with-dependencies.jar \ No newline at end of file diff --git a/buildspec.yml b/buildspec.yml new file mode 100644 index 0000000..e172968 --- /dev/null +++ b/buildspec.yml @@ -0,0 +1,53 @@ +--- +version: 0.2 +phases: + install: + commands: + - curl -sS -o aws-iam-authenticator https://amazon-eks.s3-us-west-2.amazonaws.com/1.10.3/2018-07-26/bin/linux/amd64/aws-iam-authenticator + - curl -LO "https://dl.k8s.io/release/$(curl -L -s https://dl.k8s.io/release/stable.txt)/bin/linux/amd64/kubectl" + - chmod +x ./kubectl ./aws-iam-authenticator + - export PATH=$PWD/:$PATH + - yum -y install jq python3-pip python3-dev && pip3 install --upgrade awscli + pre_build: + commands: + - $(aws ecr get-login --no-include-email) + - export KUBECONFIG=$HOME/.kube/config + - aws s3 cp s3://338918620411-account-bucket/DemoApplication-1.0-jar-with-dependencies.jar . + + build: + commands: + - ls -xtra + - echo "Contents of Dockerfile" + - cat Dockerfile + - echo Got the following IAM_ROLE env $IAM_ROLE + - echo Got the following SQS_URL env $SQS_URL + - echo Got the following S3_BUCKET env $S3_BUCKET + + # building Docker image without-issues + - docker build -f Dockerfile -t codeguru-java-app-deployment-without-issues --build-arg ISSUE_FLAG="without-issues" --build-arg IAM_ROLE=$IAM_ROLE --build-arg SQS_URL=$SQS_URL --build-arg S3_BUCKET=$S3_BUCKET . + - docker tag codeguru-java-app-deployment-without-issues:latest 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-without-issues:latest + - docker push 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-without-issues:latest + # building Docker image with-issues + - docker build -f Dockerfile -t codeguru-java-app-deployment-with-issues --build-arg ISSUE_FLAG="with-issues" --build-arg IAM_ROLE=$IAM_ROLE --build-arg SQS_URL=$SQS_URL --build-arg S3_BUCKET=$S3_BUCKET . + - docker tag codeguru-java-app-deployment-with-issues:latest 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-with-issues:latest + - docker push 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-with-issues:latest + + post_build: + commands: + - EKS_KUBECTL_ROLE_ARN=arn:aws:iam::338918620411:role/eksctl-codeguru-cluster-ServiceRole-10S2R4QXTPDM5 + - EKS_CLUSTER_NAME=codeguru + - CREDENTIALS=$(aws sts assume-role --role-arn $EKS_KUBECTL_ROLE_ARN --role-session-name codebuild-kubectl --duration-seconds 900) + - export AWS_ACCESS_KEY_ID="$(echo ${CREDENTIALS} | jq -r '.Credentials.AccessKeyId')" + - export AWS_SECRET_ACCESS_KEY="$(echo ${CREDENTIALS} | jq -r '.Credentials.SecretAccessKey')" + - export AWS_SESSION_TOKEN="$(echo ${CREDENTIALS} | jq -r '.Credentials.SessionToken')" + - export AWS_EXPIRATION=$(echo ${CREDENTIALS} | jq -r '.Credentials.Expiration') + - echo $AWS_ACCESS_KEY_ID $AWS_EXPIRATION + - aws eks update-kubeconfig --name $EKS_CLUSTER_NAME + - kubectl config get-contexts + - kubectl get nodes + - kubectl apply -f codeguru-java-app-deployment-without-issues.yaml + - kubectl apply -f codeguru-java-app-deployment-with-issues.yaml + - kubectl get all -n default + - echo $EKS_KUBECTL_ROLE_ARN:$EKS_CLUSTER_NAME > build.json +artifacts: + files: build.json diff --git a/codeguru-java-app-deployment-with-issues.yaml b/codeguru-java-app-deployment-with-issues.yaml new file mode 100644 index 0000000..d8559dc --- /dev/null +++ b/codeguru-java-app-deployment-with-issues.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: codeguru-java-app-deployment-with-issues + labels: + app: codeguru-demo-app +spec: + replicas: 1 + selector: + matchLabels: + app: codeguru-demo-app + template: + metadata: + labels: + app: codeguru-demo-app + spec: + serviceAccountName: codeguru-profiler + containers: + - name: codeguru-java-app-deployment-with-issues + image: 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-with-issues:latest diff --git a/codeguru-java-app-deployment-without-issues.yaml b/codeguru-java-app-deployment-without-issues.yaml new file mode 100644 index 0000000..f9e50e4 --- /dev/null +++ b/codeguru-java-app-deployment-without-issues.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: codeguru-java-app-deployment-without-issues + labels: + app: codeguru-demo-app +spec: + replicas: 1 + selector: + matchLabels: + app: codeguru-demo-app + template: + metadata: + labels: + app: codeguru-demo-app + spec: + serviceAccountName: codeguru-profiler + containers: + - name: codeguru-java-app-deployment-without-issues + image: 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-without-issues:latest \ No newline at end of file diff --git a/pom.xml b/pom.xml index 8b5dc78..bca6d30 100644 --- a/pom.xml +++ b/pom.xml @@ -9,9 +9,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 - org.example + com.company.demoapplication DemoApplication 1.0 + + + codeguru-profiler + codeguru-profiler + https://d1osg35nybn3tt.cloudfront.net + + @@ -69,11 +76,6 @@ - - com.amazonaws - aws-java-sdk - 1.11.747 - org.imgscalr imgscalr-lib @@ -104,5 +106,45 @@ log4j-jul 2.13.0 + + com.amazonaws + aws-java-sdk-s3 + 1.11.1018 + + + com.amazonaws + aws-java-sdk-sqs + 1.11.1018 + + + software.amazon.awssdk + sts + 2.16.62 + + + software.amazon.awssdk + auth + 2.16.62 + + + com.amazonaws + codeguru-profiler-java-agent + 1.1.1 + + + software.amazon.awssdk + regions + 2.16.62 + + + com.amazonaws + aws-java-sdk + 1.4.0 + + + org.apache.httpcomponents + httpclient + 4.5.13 + - + \ No newline at end of file diff --git a/src/main/java/com/company/demoapplication/AwsCredsProvider.java b/src/main/java/com/company/demoapplication/AwsCredsProvider.java new file mode 100644 index 0000000..27700f8 --- /dev/null +++ b/src/main/java/com/company/demoapplication/AwsCredsProvider.java @@ -0,0 +1,29 @@ +package com.company.demoapplication; + +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.DefaultCredentialsProvider; +import software.amazon.awssdk.services.sts.StsClient; +import software.amazon.awssdk.services.sts.auth.StsAssumeRoleCredentialsProvider; +import software.amazon.awssdk.services.sts.model.AssumeRoleRequest; + + + +public class AwsCredsProvider { + + public static AwsCredentialsProvider getCredentials(final String roleArn, final String sessionName) { + + Main.logger().info("Building the STS request for roleArn: {} and session: {}", roleArn, sessionName); + + final AssumeRoleRequest assumeRoleRequest = AssumeRoleRequest.builder() + .roleArn(roleArn) + .roleSessionName(sessionName) + .build(); + + + return StsAssumeRoleCredentialsProvider.builder() + .stsClient(StsClient.builder().credentialsProvider(DefaultCredentialsProvider.create()).build()) + .refreshRequest(assumeRoleRequest) + .build(); + } + +} \ No newline at end of file diff --git a/src/main/java/com/company/demoapplication/Main.java b/src/main/java/com/company/demoapplication/Main.java index ca0ef3f..95f2210 100644 --- a/src/main/java/com/company/demoapplication/Main.java +++ b/src/main/java/com/company/demoapplication/Main.java @@ -5,15 +5,19 @@ package com.company.demoapplication; +import software.amazon.awssdk.regions.Region; +import software.amazon.codeguruprofilerjavaagent.Profiler; + import com.amazonaws.services.s3.AmazonS3; import com.amazonaws.services.s3.AmazonS3ClientBuilder; import com.amazonaws.services.sqs.AmazonSQS; import com.amazonaws.services.sqs.AmazonSQSClientBuilder; +import com.amazonaws.regions.Regions; + import com.fasterxml.jackson.databind.ObjectMapper; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import java.util.Objects; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; import java.util.concurrent.ScheduledExecutorService; @@ -21,25 +25,17 @@ public class Main { + static String roleArn; static String sqsQueueUrl; static String bucketName; + static String sessionName = "codeguru-session"; + static String profileGroupName; + static Region awsRegion = Region.EU_WEST_1; + static Regions awsRegions = Regions.EU_WEST_1; + static String sampleImagesFolder = "input-images/"; static boolean withIssues; - - private static String getEnvironmentVariable(String key, String exampleValue) { - String value = System.getenv(key); - if (value == null || value.isEmpty()) { - throw new IllegalStateException("Environment variable " + key + " must be set, e.g. " + exampleValue); - } - return value; - } - - static { - sqsQueueUrl = getEnvironmentVariable("DEMO_APP_SQS_URL", "https://sqs.eu-west-2.amazonaws.com/123456789000/ImageQueue"); - bucketName = getEnvironmentVariable("DEMO_APP_BUCKET_NAME", "test-images-for-my-demo-app"); - } - static boolean reuseMapper; static boolean reuseLogger; @@ -54,6 +50,11 @@ public static void main(String[] args) throws Exception { This demo application can be configured to demonstrate some common performance issues, for example expensive logging or forgetting to re-use serializers. */ + + Main.roleArn = args[1]; + Main.sqsQueueUrl = args[2]; + Main.bucketName = args[3]; + if (args.length > 0 && args[0].equals("with-issues")) { logger().info("Running with performance issues."); @@ -70,6 +71,13 @@ public static void main(String[] args) throws Exception { logger().error("Invalid arguments: '" + String.join(" ", args) + "'. Valid arguments are: with-issues or without-issues."); System.exit(-1); } + + Profiler.builder() + .profilingGroupName(profileGroupName) + .awsCredentialsProvider(AwsCredsProvider.getCredentials(roleArn, sessionName)) + .awsRegionToReportTo(awsRegion) + .build() + .start(); // Publisher ScheduledExecutorService publisherScheduler = Executors.newScheduledThreadPool(1); From 5e7e84caf04b75cd48f4404124d07347bfae7ac3 Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Mon, 17 May 2021 15:28:15 +0100 Subject: [PATCH 2/7] Adding CodeGuru profile group names --- src/main/java/com/company/demoapplication/Main.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/main/java/com/company/demoapplication/Main.java b/src/main/java/com/company/demoapplication/Main.java index 95f2210..12e6ed8 100644 --- a/src/main/java/com/company/demoapplication/Main.java +++ b/src/main/java/com/company/demoapplication/Main.java @@ -61,12 +61,15 @@ public static void main(String[] args) throws Exception { withIssues = true; reuseMapper = false; reuseLogger = false; + profileGroupName = "codeguru-java-app-with-problems"; } else if (args.length > 0 && args[0].equals("without-issues")) { logger().info("Running without performance issues."); withIssues = false; reuseMapper = true; reuseLogger = true; + profileGroupName = "codeguru-java-app-without-problems"; + } else { logger().error("Invalid arguments: '" + String.join(" ", args) + "'. Valid arguments are: with-issues or without-issues."); System.exit(-1); From b1184bbc10df5340a0d50edf39bfca2cefdddf65 Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Thu, 20 May 2021 10:33:00 +0100 Subject: [PATCH 3/7] Changing EKS role --- codeguru-java-app-deployment-without-issues.yaml | 2 +- src/main/java/com/company/demoapplication/Main.java | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/codeguru-java-app-deployment-without-issues.yaml b/codeguru-java-app-deployment-without-issues.yaml index f9e50e4..3d23bd4 100644 --- a/codeguru-java-app-deployment-without-issues.yaml +++ b/codeguru-java-app-deployment-without-issues.yaml @@ -17,4 +17,4 @@ spec: serviceAccountName: codeguru-profiler containers: - name: codeguru-java-app-deployment-without-issues - image: 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-without-issues:latest \ No newline at end of file + image: 338918620411.dkr.ecr.eu-west-1.amazonaws.com/codeguru-java-app-deployment-without-issues:latest diff --git a/src/main/java/com/company/demoapplication/Main.java b/src/main/java/com/company/demoapplication/Main.java index 12e6ed8..5fecb16 100644 --- a/src/main/java/com/company/demoapplication/Main.java +++ b/src/main/java/com/company/demoapplication/Main.java @@ -76,11 +76,11 @@ public static void main(String[] args) throws Exception { } Profiler.builder() - .profilingGroupName(profileGroupName) - .awsCredentialsProvider(AwsCredsProvider.getCredentials(roleArn, sessionName)) - .awsRegionToReportTo(awsRegion) - .build() - .start(); + .profilingGroupName(profileGroupName) + .awsCredentialsProvider(AwsCredsProvider.getCredentials(roleArn, sessionName)) + .awsRegionToReportTo(awsRegion) + .build() + .start(); // Publisher ScheduledExecutorService publisherScheduler = Executors.newScheduledThreadPool(1); From d53dc92bdba334c2751ea9dc6aff998397e47402 Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Thu, 20 May 2021 15:02:32 +0100 Subject: [PATCH 4/7] Updating buildspec to use Codebuild ENV --- buildspec.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index e172968..e06e05b 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -34,7 +34,6 @@ phases: post_build: commands: - - EKS_KUBECTL_ROLE_ARN=arn:aws:iam::338918620411:role/eksctl-codeguru-cluster-ServiceRole-10S2R4QXTPDM5 - EKS_CLUSTER_NAME=codeguru - CREDENTIALS=$(aws sts assume-role --role-arn $EKS_KUBECTL_ROLE_ARN --role-session-name codebuild-kubectl --duration-seconds 900) - export AWS_ACCESS_KEY_ID="$(echo ${CREDENTIALS} | jq -r '.Credentials.AccessKeyId')" From c3ec43a75227af8f7cb33541e8cbc9b9abaafb9b Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Thu, 20 May 2021 20:44:03 +0100 Subject: [PATCH 5/7] Updated buildspec.yml --- buildspec.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/buildspec.yml b/buildspec.yml index e06e05b..e894e82 100644 --- a/buildspec.yml +++ b/buildspec.yml @@ -34,7 +34,6 @@ phases: post_build: commands: - - EKS_CLUSTER_NAME=codeguru - CREDENTIALS=$(aws sts assume-role --role-arn $EKS_KUBECTL_ROLE_ARN --role-session-name codebuild-kubectl --duration-seconds 900) - export AWS_ACCESS_KEY_ID="$(echo ${CREDENTIALS} | jq -r '.Credentials.AccessKeyId')" - export AWS_SECRET_ACCESS_KEY="$(echo ${CREDENTIALS} | jq -r '.Credentials.SecretAccessKey')" From cea205724f5bfe55c1389544caac6e7226457bf5 Mon Sep 17 00:00:00 2001 From: Oli Leach Date: Thu, 20 May 2021 22:10:00 +0100 Subject: [PATCH 6/7] Updated moin.java class --- src/main/java/com/company/demoapplication/Main.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/main/java/com/company/demoapplication/Main.java b/src/main/java/com/company/demoapplication/Main.java index 5fecb16..5293ee1 100644 --- a/src/main/java/com/company/demoapplication/Main.java +++ b/src/main/java/com/company/demoapplication/Main.java @@ -98,12 +98,12 @@ static AmazonS3 s3Client() { if (sharedS3 != null) { return sharedS3; } else { - return AmazonS3ClientBuilder.defaultClient(); + return AmazonS3ClientBuilder.standard().withRegion(awsRegions).build(); } } static AmazonSQS sqsClient() { - return AmazonSQSClientBuilder.defaultClient(); + return AmazonSQSClientBuilder.standard().withRegion(awsRegions).build(); } static ObjectMapper objectMapper() { From 41f435f38100bb4d9c9f88ae5b51444be55cbb36 Mon Sep 17 00:00:00 2001 From: Rafael Ramos Date: Mon, 21 Jun 2021 10:40:50 +0200 Subject: [PATCH 7/7] Update Main.java Enabling heap summary --- src/main/java/com/company/demoapplication/Main.java | 1 + 1 file changed, 1 insertion(+) diff --git a/src/main/java/com/company/demoapplication/Main.java b/src/main/java/com/company/demoapplication/Main.java index 5293ee1..a7a81b3 100644 --- a/src/main/java/com/company/demoapplication/Main.java +++ b/src/main/java/com/company/demoapplication/Main.java @@ -79,6 +79,7 @@ public static void main(String[] args) throws Exception { .profilingGroupName(profileGroupName) .awsCredentialsProvider(AwsCredsProvider.getCredentials(roleArn, sessionName)) .awsRegionToReportTo(awsRegion) + .withHeapSummary(true) .build() .start();