Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

docs: Javadocs for TransferManager interface and ParallelUploadConfig #2094

Merged
merged 8 commits into from
Jun 26, 2023
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,11 @@
import java.util.Objects;
import org.checkerframework.checker.nullness.qual.NonNull;

/**
* Configuration for performing Parallel Downloads with {@link TransferManager}.
*
* @see Builder
*/
@BetaApi
public final class ParallelDownloadConfig {

Expand All @@ -48,7 +53,7 @@ private ParallelDownloadConfig(
}

/**
* A common prefix that is removed from downloaded object's name before written to the filesystem
* A common prefix that is removed from downloaded object's name before written to the filesystem.
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
*/
@BetaApi
public @NonNull String getStripPrefix() {
Expand All @@ -61,13 +66,16 @@ private ParallelDownloadConfig(
return downloadDirectory;
}

/** The bucket objects are being downloaded from */
/** The bucket objects are being downloaded from. */
@BetaApi
public @NonNull String getBucketName() {
return bucketName;
}

/** A list of common BlobSourceOptions that are used for each download request */
/**
* A list of common BlobSourceOptions that are used for each download request. Note this list of
* options will be applied to every single request.
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
*/
@BetaApi
public @NonNull List<BlobSourceOption> getOptionsPerRequest() {
return optionsPerRequest;
Expand Down Expand Up @@ -103,6 +111,11 @@ public String toString() {
.toString();
}

/**
* Builds an instance of ParallelDownloadConfig.
*
* @see ParallelDownloadConfig
*/
@BetaApi
public static Builder newBuilder() {
return new Builder();
Expand All @@ -123,30 +136,57 @@ private Builder() {
this.optionsPerRequest = ImmutableList.of();
}

/**
* Sets the value for stripPrefix. This string will be removed from the beginning of all object
* names before they are written to the filesystem.
*
* @return the builder instance with the value for stripPrefix modified.
*/
@BetaApi
public Builder setStripPrefix(String stripPrefix) {
this.stripPrefix = stripPrefix;
return this;
}

/**
* Sets the base directory on the filesystem that all objects will be written to.
*
* @return the builder instance with the value for downloadDirectory modified.
*/
@BetaApi
public Builder setDownloadDirectory(Path downloadDirectory) {
this.downloadDirectory = downloadDirectory;
return this;
}

/**
* Sets the bucketName that Transfer Manager will download from. This field is required.
*
* @return the builder instance with the value for bucketName modified.
*/
@BetaApi
public Builder setBucketName(String bucketName) {
this.bucketName = bucketName;
return this;
}

/**
* Sets the BlobSourceOptions that will be applied to each download request. Note these options
* will be applied to every single download request.
BenWhitehead marked this conversation as resolved.
Show resolved Hide resolved
*
* @return the builder instance with the value for OptionsPerRequest modified.
*/
@BetaApi
public Builder setOptionsPerRequest(List<BlobSourceOption> optionsPerRequest) {
this.optionsPerRequest = ImmutableList.copyOf(optionsPerRequest);
return this;
}

/**
* Creates a ParallelDownloadConfig object.
*
* @return {@link ParallelDownloadConfig}
*/
@BetaApi
public ParallelDownloadConfig build() {
checkNotNull(bucketName);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,38 +19,42 @@
import static com.google.common.base.Preconditions.checkNotNull;

import com.google.api.core.BetaApi;
import com.google.cloud.storage.Storage.BlobTargetOption;
import com.google.cloud.storage.Storage.BlobWriteOption;
import com.google.common.base.MoreObjects;
import com.google.common.collect.ImmutableList;
import java.util.List;
import java.util.Objects;
import org.checkerframework.checker.nullness.qual.NonNull;

/**
* Configuration for performing Parallel Uploads with {@link TransferManager}.
*
* @see Builder
*/
@BetaApi
public final class ParallelUploadConfig {

private final boolean skipIfExists;
@NonNull private final String prefix;
@NonNull private final String bucketName;
@NonNull private final List<BlobTargetOption> targetOptsPerRequest;

@NonNull private final List<BlobWriteOption> writeOptsPerRequest;

private ParallelUploadConfig(
boolean skipIfExists,
@NonNull String prefix,
@NonNull String bucketName,
@NonNull List<BlobTargetOption> targetOptsPerRequest,
@NonNull List<BlobWriteOption> writeOptsPerRequest) {
this.skipIfExists = skipIfExists;
this.prefix = prefix;
this.bucketName = bucketName;
this.targetOptsPerRequest = targetOptsPerRequest;
this.writeOptsPerRequest = applySkipIfExists(skipIfExists, writeOptsPerRequest);
}

/** If a corresponding object already exists skip uploading the object */
/**
* If set Transfer Manager will skip uploading an object if it already exists, equivalent to
* setting the precondition doesNotExist {@link BlobWriteOption}
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
*/
@BetaApi
public boolean isSkipIfExists() {
return skipIfExists;
Expand All @@ -68,13 +72,10 @@ public boolean isSkipIfExists() {
return bucketName;
}

/** A list of common BlobTargetOptions that are used for each upload request */
@BetaApi
public @NonNull List<BlobTargetOption> getTargetOptsPerRequest() {
return targetOptsPerRequest;
}

/** A list of common BlobWriteOptions that are used for each upload request */
/**
* A list of common BlobWriteOptions, note these options will be applied to every single upload
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
* request.
*/
@BetaApi
public @NonNull List<BlobWriteOption> getWriteOptsPerRequest() {
return writeOptsPerRequest;
Expand All @@ -92,12 +93,12 @@ public boolean equals(Object o) {
return skipIfExists == that.skipIfExists
&& prefix.equals(that.prefix)
&& bucketName.equals(that.bucketName)
&& targetOptsPerRequest.equals(that.targetOptsPerRequest);
&& writeOptsPerRequest.equals(that.writeOptsPerRequest);
}

@Override
public int hashCode() {
return Objects.hash(skipIfExists, prefix, bucketName, targetOptsPerRequest);
return Objects.hash(skipIfExists, prefix, bucketName, writeOptsPerRequest);
}

@Override
Expand All @@ -106,7 +107,7 @@ public String toString() {
.add("skipIfExists", skipIfExists)
.add("prefix", prefix)
.add("bucketName", bucketName)
.add("optionsPerRequest", targetOptsPerRequest)
.add("writeOptsPerRequest", writeOptsPerRequest)
.toString();
}

Expand All @@ -124,61 +125,82 @@ private static List<BlobWriteOption> applySkipIfExists(
return writeOptsPerRequest;
}

/**
* Builds an instance of ParallelUploadConfig.
*
* @see ParallelUploadConfig
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
*/
@BetaApi
public static final class Builder {

private boolean skipIfExists;
private @NonNull String prefix;
private @NonNull String bucketName;
private @NonNull List<BlobTargetOption> optionsPerRequest;

private @NonNull List<BlobWriteOption> writeOptsPerRequest;

private Builder() {
this.prefix = "";
this.bucketName = "";
this.optionsPerRequest = ImmutableList.of();
this.writeOptsPerRequest = ImmutableList.of();
}

/**
* Sets the parameter for skipIfExists. When set to true Transfer Manager will skip uploading an
* object if it already exists.
*
* @return the builder instance with the value for skipIfExists modified.
*/
@BetaApi
public Builder setSkipIfExists(boolean skipIfExists) {
this.skipIfExists = skipIfExists;
return this;
}

/**
* Sets a common prefix that will be applied to all object paths in the destination bucket.
*
* @return the builder instance with the value for prefix modified.
*/
@BetaApi
public Builder setPrefix(@NonNull String prefix) {
this.prefix = prefix;
return this;
}

/**
* Sets the bucketName that Transfer Manager will upload to. This field is required.
*
* @return the builder instance with the value for bucketName modified.
*/
@BetaApi
public Builder setBucketName(@NonNull String bucketName) {
this.bucketName = bucketName;
return this;
}

@BetaApi
public Builder setOptionsPerRequest(@NonNull List<BlobTargetOption> optionsPerRequest) {
this.optionsPerRequest = ImmutableList.copyOf(optionsPerRequest);
return this;
}

/**
* Sets the BlobWriteOptions that will be applied to each upload request. Note these options
* will be applied to every single upload request.
BenWhitehead marked this conversation as resolved.
Show resolved Hide resolved
*
* @return the builder instance with the value for WriteOptsPerRequest modified.
*/
@BetaApi
public Builder setWriteOptsPerRequest(@NonNull List<BlobWriteOption> writeOptsPerRequest) {
this.writeOptsPerRequest = writeOptsPerRequest;
return this;
}

/**
* Creates a ParallelUploadConfig object.
*
* @return {@link ParallelUploadConfig}
*/
@BetaApi
public ParallelUploadConfig build() {
checkNotNull(prefix);
checkNotNull(bucketName);
checkNotNull(optionsPerRequest);
checkNotNull(writeOptsPerRequest);
return new ParallelUploadConfig(
skipIfExists, prefix, bucketName, optionsPerRequest, writeOptsPerRequest);
return new ParallelUploadConfig(skipIfExists, prefix, bucketName, writeOptsPerRequest);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -18,19 +18,75 @@

import com.google.api.core.BetaApi;
import com.google.cloud.storage.BlobInfo;
import java.io.IOException;
import java.nio.file.Path;
import java.util.List;
import org.checkerframework.checker.nullness.qual.NonNull;

/**
* An interface for a Transfer Manager.
*
* <p>Transfer Manager handles Parallel Uploads and Parallel Downloads.
*/
@BetaApi
public interface TransferManager extends AutoCloseable {

/**
* Uploads a list of files in parallel.
*
* <p>Accepts a {@link ParallelUploadConfig} which defines the constraints of parallel uploads or
* predefined defaults.
*
* <p>Example of creating a parallel upload with Transfer Manager.
*
* <pre>{@code
* String bucketName = "my-unique-bucket";
* Path filePath = Paths.get("/path/to/my/file.txt");
* Path anotherFilePath = Paths.get("/path/to/another/file.txt");
* List<Path> files = List.of(filePath, anotherFilePath);
*
* ParallelUploadConfig parallelUploadConfig =
* ParallelUploadConfig.newBuilder()
* .setBucketName(bucketName)
* .build();
*
* UploadJob uploadedFiles = transferManager.uploadFiles(files, config);
*
* }</pre>
*
* @return an {@link UploadJob}
sydney-munro marked this conversation as resolved.
Show resolved Hide resolved
*/
@BetaApi
@NonNull
UploadJob uploadFiles(List<Path> files, ParallelUploadConfig opts) throws IOException;
UploadJob uploadFiles(List<Path> files, ParallelUploadConfig config);

/**
* Downloads a list of blobs in parallel.
*
* <p>Accepts a {@link ParallelDownloadConfig} which defines the constraints of parallel downloads
* or predefined defaults.
*
* <p>Example of creating a parallel download with Transfer Manager.
*
* <pre>{@code
* String bucketName = "my-unique-bucket";
* String blobName = "my-blob-name";
* BlobId blobId = BlobId.of(bucketName, blobName);
* BlobInfo blobInfo = BlobInfo.newBuilder(blobId).setContentType("text/plain").build();
* Path baseDir = Paths.get("/path/to/directory/");
*
* ParallelDownloadConfig parallelDownloadConfig =
* ParallelDownloadConfig.newBuilder()
* .setBucketName(bucketName)
* .setDownloadDirectory(baseDir)
* .build();
*
* DownloadJob downloadedBlobs = transferManager.downloadBlobs(files, config);
*
* }</pre>
*
* @return a {@link DownloadJob}
*/
@BetaApi
@NonNull
DownloadJob downloadBlobs(List<BlobInfo> blobs, ParallelDownloadConfig opts);
DownloadJob downloadBlobs(List<BlobInfo> blobs, ParallelDownloadConfig config);
}