Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add S3Client listObjects support in S3Template #831

Merged
merged 2 commits into from
Aug 14, 2023
Merged

Add S3Client listObjects support in S3Template #831

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
d4672ca
Add S3Client listObjects support in S3Template
alexisgra Jun 2, 2023
9dbb6db
Factorize listObjects() integration test
alexisgra Jun 19, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 12 additions & 1 deletion ...ud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Operations.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,7 @@
import java.io.InputStream;
import java.net.URL;
import java.time.Duration;
import java.util.List;
import org.springframework.lang.Nullable;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.CreateBucketResponse;
Expand Down Expand Up @@ -60,7 +61,17 @@ public interface S3Operations {
* @param s3Url - the S3 url s3://bucket/key
*/
void deleteObject(String s3Url);


/**
* Returns some or all (up to 1,000) of the objects in a bucket.
* Does not handle pagination. If you need pagination you should use {@link S3PathMatchingResourcePatternResolver} or {@link S3Client}
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Maybe we should include a paginator request since it will paginate if there are more than 1000 objects internally ->
https://sdk.amazonaws.com/java/api/latest/software/amazon/awssdk/services/s3/S3Client.html#listObjectsV2Paginator(software.amazon.awssdk.services.s3.model.ListObjectsV2Request)

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

According to the issue I used as a reference: #767 . It is specified that in the S3Template, pagination would not be managed on the list objects. If you need pagination, you should use the S3Client or S3PathMatchingResourcePatternResolver according to the issue.
I can manage the pagination if required. Thanks.

*
* @param bucketName - the bucket name
* @param prefix - objects prefix
* @return list of {@link S3Resource}
*/
List<S3Resource> listObjects(String bucketName, String prefix);

/**
* Stores a Java object in a S3 bucket. Uses {@link S3ObjectConverter} for serialization.
*
Expand Down
19 changes: 18 additions & 1 deletion ...loud-aws-s3-parent/spring-cloud-aws-s3/src/main/java/io/awspring/cloud/s3/S3Template.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,16 @@
import java.io.OutputStream;
import java.net.URL;
import java.time.Duration;
import java.util.List;
import java.util.stream.Collectors;

import org.springframework.lang.Nullable;
import org.springframework.util.Assert;
import org.springframework.util.StreamUtils;
import software.amazon.awssdk.services.s3.S3Client;
import software.amazon.awssdk.services.s3.model.GetObjectRequest;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Request;
import software.amazon.awssdk.services.s3.model.ListObjectsV2Response;
import software.amazon.awssdk.services.s3.model.PutObjectRequest;
import software.amazon.awssdk.services.s3.presigner.S3Presigner;
import software.amazon.awssdk.services.s3.presigner.model.GetObjectPresignRequest;
Expand Down Expand Up @@ -84,7 +89,19 @@ public void deleteObject(String s3Url) {
Location location = Location.of(s3Url);
this.deleteObject(location.getBucket(), location.getObject());
}


@Override
public List<S3Resource> listObjects(String bucketName, String prefix) {
Assert.notNull(bucketName, "bucketName is required");
Assert.notNull(prefix, "prefix is required");

final ListObjectsV2Request request = ListObjectsV2Request.builder().bucket(bucketName).prefix(prefix).build();
final ListObjectsV2Response response = s3Client.listObjectsV2(request);

return response.contents().stream()
.map(s3Object -> new S3Resource(bucketName, s3Object.key(), s3Client, s3OutputStreamProvider)).toList();
}

@Override
public S3Resource store(String bucketName, String key, Object object) {
Assert.notNull(bucketName, "bucketName is required");
Expand Down
16 changes: 16 additions & 0 deletions ...nt/spring-cloud-aws-s3/src/test/java/io/awspring/cloud/s3/S3TemplateIntegrationTests.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,8 @@
import java.net.URL;
import java.nio.charset.StandardCharsets;
import java.time.Duration;
import java.util.List;

import org.apache.http.HttpEntity;
import org.apache.http.HttpResponse;
import org.apache.http.client.methods.HttpGet;
Expand Down Expand Up @@ -147,6 +149,20 @@ void deletesObjectByS3Url() {
.isThrownBy(() -> client.headObject(r -> r.bucket(BUCKET_NAME).key("key.txt")));
}

@Test
void listObjects() throws IOException {
client.putObject(r -> r.bucket(BUCKET_NAME).key("hello-en.txt"), RequestBody.fromString("hello"));
client.putObject(r -> r.bucket(BUCKET_NAME).key("hello-fr.txt"), RequestBody.fromString("bonjour"));
client.putObject(r -> r.bucket(BUCKET_NAME).key("bye.txt"), RequestBody.fromString("bye"));

List<S3Resource> resources = s3Template.listObjects(BUCKET_NAME, "hello");
assertThat(resources.size()).isEqualTo(2);

// According to the S3Client doc : "Objects are returned sorted in an ascending order of the respective key names in the list."
assertThat(resources).extracting(S3Resource::getInputStream).map(is -> new String(is.readAllBytes(), StandardCharsets.UTF_8))
.containsExactly("hello", "bonjour");
}

@Test
void storesObject() throws IOException {
S3Resource storedObject = s3Template.store(BUCKET_NAME, "person.json", new Person("John", "Doe"));
Expand Down