Skip to content

Commit 8957ced

Browse files
martinschaefMartin Schaef
martinschaef
and
Martin Schaef
authored
Avoid packing build artifacts twice if they get included in the -s and -b option. (#31)
* Exclude build artifacts from source artifacts if folders overlap --------- Co-authored-by: Martin Schaef <schaef@amazon.com>
1 parent 8b005cc commit 8957ced

File tree

8 files changed

+313
-75
lines changed

8 files changed

+313
-75
lines changed

‎.github/workflows/cicd-demo.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ jobs:
4040
- name: Download CodeGuru Reviewer CLI
4141
shell: bash
4242
env:
43-
VERSION: 0.2.1
43+
VERSION: 0.2.4
4444
run: curl -OL "https://github.com/aws/aws-codeguru-cli/releases/download/$VERSION/aws-codeguru-cli.zip"
4545
- run: unzip aws-codeguru-cli.zip
4646

‎build.gradle

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ repositories {
2323

2424
defaultTasks 'clean', 'check', 'installDist'
2525

26-
version = '0.2.3'
26+
version = '0.2.4'
2727
jar.archiveName = "${jar.baseName}.${jar.extension}"
2828
distZip.archiveName = "${jar.baseName}.zip"
2929

‎src/main/java/com/amazonaws/gurureviewercli/adapter/ArtifactAdapter.java

Lines changed: 29 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,26 @@
11
package com.amazonaws.gurureviewercli.adapter;
22

3+
import com.amazonaws.gurureviewercli.exceptions.GuruCliException;
4+
import com.amazonaws.gurureviewercli.model.Configuration;
5+
import com.amazonaws.gurureviewercli.model.ErrorCodes;
6+
import com.amazonaws.gurureviewercli.model.ScanMetaData;
7+
import com.amazonaws.gurureviewercli.util.Log;
8+
import com.amazonaws.gurureviewercli.util.ZipUtils;
9+
import lombok.val;
10+
import software.amazon.awssdk.services.s3.S3Client;
11+
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
12+
313
import java.io.File;
414
import java.io.FileNotFoundException;
515
import java.io.IOException;
616
import java.nio.file.Files;
717
import java.nio.file.Path;
818
import java.util.ArrayList;
19+
import java.util.Collections;
920
import java.util.Comparator;
1021
import java.util.List;
1122
import java.util.UUID;
12-
13-
import lombok.val;
14-
import software.amazon.awssdk.services.s3.S3Client;
15-
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
16-
17-
import com.amazonaws.gurureviewercli.exceptions.GuruCliException;
18-
import com.amazonaws.gurureviewercli.model.Configuration;
19-
import com.amazonaws.gurureviewercli.model.ErrorCodes;
20-
import com.amazonaws.gurureviewercli.model.ScanMetaData;
21-
import com.amazonaws.gurureviewercli.util.Log;
22-
import com.amazonaws.gurureviewercli.util.ZipUtils;
23+
import java.util.stream.Collectors;
2324

2425
/**
2526
* Utility class class to Zip and upload source and build artifacts to S3.
@@ -70,16 +71,16 @@ public static ScanMetaData zipAndUpload(final Configuration config,
7071
Log.info("Adding %d out of %d files under version control in %s",
7172
versionedFiles, totalFiles, repositoryDir.toAbsolutePath());
7273
filesToScan.addAll(ZipUtils.getFilesInDirectory(repositoryDir.resolve(".git")));
73-
sourceKey = zipAndUploadFiles("analysis-src-" + UUID.randomUUID(), filesToScan, repositoryDir,
74-
bucketName, tempDir, config.getAccountId(), config.getS3Client());
74+
sourceKey = zipAndUploadFiles("analysis-src-" + UUID.randomUUID(), filesToScan, buildDirs,
75+
repositoryDir, bucketName, tempDir, config.getAccountId(), config.getS3Client());
7576
} else {
7677
val sourceDirsAndGit = new ArrayList<Path>(sourceDirs);
7778
if (config.getBeforeCommit() != null && config.getAfterCommit() != null) {
7879
// only add the git folder if a commit range is provided.
7980
sourceDirsAndGit.add(repositoryDir.resolve(".git"));
8081
}
81-
sourceKey = zipAndUploadDir("analysis-src-" + UUID.randomUUID(), sourceDirsAndGit, repositoryDir,
82-
bucketName, tempDir, config.getAccountId(), config.getS3Client());
82+
sourceKey = zipAndUploadDir("analysis-src-" + UUID.randomUUID(), sourceDirsAndGit,
83+
buildDirs, repositoryDir, bucketName, tempDir, config.getAccountId(), config.getS3Client());
8384
}
8485
final String buildKey;
8586
if (buildDirs != null && !buildDirs.isEmpty()) {
@@ -90,7 +91,7 @@ public static ScanMetaData zipAndUpload(final Configuration config,
9091
}
9192
buildKey =
9293
zipAndUploadDir("analysis-bin-" + UUID.randomUUID(), buildDirs,
93-
bucketName, tempDir, config.getAccountId(), config.getS3Client());
94+
Collections.emptyList(), bucketName, tempDir, config.getAccountId(), config.getS3Client());
9495
} else {
9596
buildKey = null;
9697
}
@@ -114,15 +115,17 @@ public static ScanMetaData zipAndUpload(final Configuration config,
114115

115116
private static String zipAndUploadDir(final String artifactName,
116117
final List<Path> dirNames,
118+
final List<Path> excludeList,
117119
final String bucketName,
118120
final Path tempDir,
119121
final String accountId,
120122
final S3Client s3Client) throws IOException {
121-
return zipAndUploadDir(artifactName, dirNames, null, bucketName, tempDir, accountId, s3Client);
123+
return zipAndUploadDir(artifactName, dirNames, excludeList, null, bucketName, tempDir, accountId, s3Client);
122124
}
123125

124126
private static String zipAndUploadDir(final String artifactName,
125127
final List<Path> dirNames,
128+
final List<Path> excludeList,
126129
final Path rootDir,
127130
final String bucketName,
128131
final Path tempDir,
@@ -134,9 +137,9 @@ private static String zipAndUploadDir(final String artifactName,
134137
val s3Key = zipFileName;
135138
if (!zipFile.toFile().isFile()) {
136139
if (rootDir != null) {
137-
ZipUtils.pack(dirNames, rootDir, zipFile.toString());
140+
ZipUtils.pack(dirNames, excludeList, rootDir, zipFile.toString());
138141
} else {
139-
ZipUtils.pack(dirNames, zipFile.toString());
142+
ZipUtils.pack(dirNames, excludeList, zipFile.toString());
140143
}
141144
}
142145
val putObjectRequest = PutObjectRequest.builder()
@@ -152,6 +155,7 @@ private static String zipAndUploadDir(final String artifactName,
152155

153156
private static String zipAndUploadFiles(final String artifactName,
154157
final List<Path> files,
158+
final List<Path> excludeDirs,
155159
final Path rootDir,
156160
final String bucketName,
157161
final Path tempDir,
@@ -162,7 +166,7 @@ private static String zipAndUploadFiles(final String artifactName,
162166
val zipFile = tempDir.resolve(zipFileName).toAbsolutePath();
163167
val s3Key = zipFileName;
164168
if (!zipFile.toFile().isFile()) {
165-
ZipUtils.packFiles(files, rootDir, zipFile);
169+
ZipUtils.packFiles(files, excludeDirs, rootDir, zipFile);
166170
}
167171
val putObjectRequest = PutObjectRequest.builder()
168172
.bucket(bucketName)
@@ -175,6 +179,11 @@ private static String zipAndUploadFiles(final String artifactName,
175179
return null;
176180
}
177181

182+
private static List<Path> filterAgainstExcludeDirs(final List<Path> original, final List<Path> exclude) {
183+
return original.stream().filter(path -> exclude.stream().anyMatch(ex -> path.startsWith(ex)))
184+
.collect(Collectors.toList());
185+
}
186+
178187
private ArtifactAdapter() {
179188
// do not instantiate
180189
}

‎src/main/java/com/amazonaws/gurureviewercli/util/ZipUtils.java

Lines changed: 45 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,8 @@
11
package com.amazonaws.gurureviewercli.util;
22

3+
import lombok.extern.log4j.Log4j2;
4+
import lombok.val;
5+
36
import java.io.IOException;
47
import java.nio.file.Files;
58
import java.nio.file.Path;
@@ -12,9 +15,6 @@
1215
import java.util.zip.ZipEntry;
1316
import java.util.zip.ZipOutputStream;
1417

15-
import lombok.extern.log4j.Log4j2;
16-
import lombok.val;
17-
1818
/**
1919
* Util class for ZipFile.
2020
*/
@@ -29,24 +29,31 @@ public final class ZipUtils {
2929
* @throws IOException io exception
3030
*/
3131
public static void pack(final List<Path> sourceDirPaths, final String zipFilePath) throws IOException {
32+
pack(sourceDirPaths, Collections.emptyList(), zipFilePath);
33+
}
34+
35+
public static void pack(final List<Path> sourceDirPaths,
36+
final List<Path> excludeDirs,
37+
final String zipFilePath) throws IOException {
3238
Path p = Files.createFile(Paths.get(zipFilePath).normalize().toAbsolutePath());
3339
try (ZipOutputStream zs = new ZipOutputStream(Files.newOutputStream(p))) {
3440
for (val sourceDirPath : sourceDirPaths) {
3541
Path pp = sourceDirPath.toRealPath();
3642
try (val walk = Files.walk(pp)) {
3743
walk.filter(path -> !Files.isDirectory(path))
38-
.forEach(path -> {
39-
val relativePath = pp.relativize(path.normalize().toAbsolutePath());
40-
// in case we run on Windows
41-
ZipEntry zipEntry = new ZipEntry(getUnixStylePathName(relativePath));
42-
try {
43-
zs.putNextEntry(zipEntry);
44-
zs.write(Files.readAllBytes(path));
45-
zs.closeEntry();
46-
} catch (Exception e) {
47-
log.error("Skipping file {} because of error: {}", path, e.getMessage());
48-
}
49-
});
44+
.filter(path -> isIncluded(path, excludeDirs))
45+
.forEach(path -> {
46+
val relativePath = pp.relativize(path.normalize().toAbsolutePath());
47+
// in case we run on Windows
48+
ZipEntry zipEntry = new ZipEntry(getUnixStylePathName(relativePath));
49+
try {
50+
zs.putNextEntry(zipEntry);
51+
zs.write(Files.readAllBytes(path));
52+
zs.closeEntry();
53+
} catch (Exception e) {
54+
log.error("Skipping file {} because of error: {}", path, e.getMessage());
55+
}
56+
});
5057
}
5158
}
5259
}
@@ -63,13 +70,19 @@ public static void pack(final List<Path> sourceDirPaths, final String zipFilePat
6370
public static void pack(final List<Path> sourceDirPaths,
6471
final Path relativeRoot,
6572
final String zipFilePath) throws IOException {
73+
pack(sourceDirPaths, Collections.emptyList(), relativeRoot, zipFilePath);
74+
}
6675

76+
public static void pack(final List<Path> sourceDirPaths,
77+
final List<Path> excludeDirs,
78+
final Path relativeRoot,
79+
final String zipFilePath) throws IOException {
6780
val files = getFilesInDirectories(sourceDirPaths);
6881
val codeGuruConfigFile = relativeRoot.resolve("aws-codeguru-reviewer.yml");
6982
if (codeGuruConfigFile != null && codeGuruConfigFile.toFile().isFile()) {
7083
files.add(codeGuruConfigFile);
7184
}
72-
packFiles(files, relativeRoot, Paths.get(zipFilePath));
85+
packFiles(files, excludeDirs, relativeRoot, Paths.get(zipFilePath));
7386
}
7487

7588
/**
@@ -81,13 +94,15 @@ public static void pack(final List<Path> sourceDirPaths,
8194
* @throws IOException io exception
8295
*/
8396
public static void packFiles(final Collection<Path> files,
97+
final List<Path> excludeDirs,
8498
final Path relativeRoot,
8599
final Path zipFilePath) throws IOException {
86100
val normalizedRoot = relativeRoot.toRealPath();
87101
val normalizedFiles = files.stream()
88-
.map(Path::toAbsolutePath)
89-
.map(Path::normalize)
90-
.collect(Collectors.toList());
102+
.map(Path::toAbsolutePath)
103+
.map(Path::normalize)
104+
.filter(p -> isIncluded(p, excludeDirs))
105+
.collect(Collectors.toList());
91106
normalizedFiles.forEach(file -> {
92107
if (!file.startsWith(normalizedRoot)) {
93108
val msg = String.format("%s is not a parent directory of %s", normalizedRoot, file);
@@ -144,7 +159,17 @@ public static List<Path> getFilesInDirectory(final Path directory) throws IOExce
144159
}
145160

146161
private static String getUnixStylePathName(final Path path) {
147-
return path.normalize().toString().replace('\\', '/' );
162+
return path.normalize().toString().replace('\\', '/');
163+
}
164+
165+
private static boolean isIncluded(final Path p, final Collection<Path> excludes) {
166+
if (excludes != null) {
167+
if (excludes.stream().anyMatch(ex -> p.startsWith(ex.toAbsolutePath().normalize()))) {
168+
log.warn("File excluded from source zip because it is part of the build zip already: {}", p);
169+
return false;
170+
}
171+
}
172+
return true;
148173
}
149174

150175
/**

0 commit comments

Comments
 (0)