Skip to content

Commit

Permalink
feat: [storagetransfer] add HDFS configuration (#5619)
Browse files Browse the repository at this point in the history 
* feat: add HDFS configuration
feat: add GCS Managed Folders
feat: add S3 Managed Private Network
feat: add S3 Cloudfront Domain

PiperOrigin-RevId: 662684810

Source-Link: googleapis/googleapis@83e5198

Source-Link: googleapis/googleapis-gen@d8e78bb
Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLXN0b3JhZ2V0cmFuc2Zlci8uT3dsQm90LnlhbWwiLCJoIjoiZDhlNzhiYjVjNDVlY2RhZDE2YjI2NDdjZmRiMDkxOTVkZDU1ZTk3NiJ9

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
@gcf-owl-bot
gcf-owl-bot[bot] and gcf-owl-bot[bot] committed Aug 15, 2024
1 parent d2ba736 commit 7500af4
Show file tree
Hide file tree
Showing 4 changed files with 570 additions and 46 deletions.
123 changes: 81 additions & 42 deletions packages/google-storagetransfer/protos/google/storagetransfer/v1/transfer_types.proto
View file
Original file line number Diff line number Diff line change
Expand Up @@ -205,6 +205,21 @@ message GcsData {
// The root path value must meet
// [Object Name Requirements](/storage/docs/naming#objectnames).
string path = 3;

// Preview. Enables the transfer of managed folders between Cloud Storage
// buckets. Set this option on the gcs_data_source.
//
// If set to true:
//
// - Managed folders in the source bucket are transferred to the
// destination bucket.
// - Managed folders in the destination bucket are overwritten. Other
// OVERWRITE options are not supported.
//
// See
// [Transfer Cloud Storage managed
// folders](/storage-transfer/docs/managed-folders).
bool managed_folder_transfer_enabled = 4;
}

// An AwsS3Data resource can be a data source, but not a data sink.
Expand Down Expand Up @@ -242,26 +257,46 @@ message AwsS3Data {
// this project.
string role_arn = 4;

// Optional. The CloudFront distribution domain name pointing to this bucket,
// to use when fetching.
//
// See
// [Transfer from S3 via
// CloudFront](https://cloud.google.com/storage-transfer/docs/s3-cloudfront)
// for more information.
//
// Format: `https://{id}.cloudfront.net` or any valid custom domain. Must
// begin with `https://`.
string cloudfront_domain = 6 [(google.api.field_behavior) = OPTIONAL];

// Optional. The Resource name of a secret in Secret Manager.
//
// The Azure SAS token must be stored in Secret Manager in JSON format:
// <pre>{
// "sas_token" : "<var>SAS_TOKEN</var>"
// }</pre>
// AWS credentials must be stored in Secret Manager in JSON format:
//
// {
// "access_key_id": "ACCESS_KEY_ID",
// "secret_access_key": "SECRET_ACCESS_KEY"
// }
//
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
// be granted `roles/secretmanager.secretAccessor` for the resource.
//
// See [Configure access to a source: Microsoft Azure Blob Storage]
// (https://cloud.google.com/storage-transfer/docs/source-microsoft-azure#secret_manager)
// See [Configure access to a source: Amazon S3]
// (https://cloud.google.com/storage-transfer/docs/source-amazon-s3#secret_manager)
// for more information.
//
// If `credentials_secret` is specified, do not specify [azure_credentials][].
//
// This feature is in
// [preview](https://cloud.google.com/terms/service-terms#1).
// If `credentials_secret` is specified, do not specify
// [role_arn][google.storagetransfer.v1.AwsS3Data.role_arn] or
// [aws_access_key][google.storagetransfer.v1.AwsS3Data.aws_access_key].
//
// Format: `projects/{project_number}/secrets/{secret_name}`
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];

oneof private_network {
// Egress bytes over a Google-managed private network.
// This network is shared between other users of Storage Transfer Service.
bool managed_private_network = 8;
}
}

// An AzureBlobStorageData resource can be a data source, but not a data sink.
Expand Down Expand Up @@ -298,9 +333,11 @@ message AzureBlobStorageData {
// Optional. The Resource name of a secret in Secret Manager.
//
// The Azure SAS token must be stored in Secret Manager in JSON format:
// <pre>{
// "sas_token" : "<var>SAS_TOKEN</var>"
// }</pre>
//
// {
// "sas_token" : "SAS_TOKEN"
// }
//
// [GoogleServiceAccount][google.storagetransfer.v1.GoogleServiceAccount] must
// be granted `roles/secretmanager.secretAccessor` for the resource.
//
Expand All @@ -311,9 +348,6 @@ message AzureBlobStorageData {
// If `credentials_secret` is specified, do not specify
// [azure_credentials][google.storagetransfer.v1.AzureBlobStorageData.azure_credentials].
//
// This feature is in
// [preview](https://cloud.google.com/terms/service-terms#1).
//
// Format: `projects/{project_number}/secrets/{secret_name}`
string credentials_secret = 7 [(google.api.field_behavior) = OPTIONAL];
}
Expand Down Expand Up @@ -371,6 +405,15 @@ message PosixFilesystem {
string root_directory = 1;
}

// An HdfsData resource specifies a path within an HDFS entity (e.g. a cluster).
// All cluster-specific settings, such as namenodes and ports, are configured on
// the transfer agents servicing requests, so HdfsData only contains the root
// path to the data in our transfer.
message HdfsData {
// Root path to transfer files.
string path = 1;
}

// An AwsS3CompatibleData resource.
message AwsS3CompatibleData {
// Required. Specifies the name of the bucket.
Expand Down Expand Up @@ -473,7 +516,7 @@ message S3CompatibleMetadata {
ListApi list_api = 4;
}

// Represents an On-Premises Agent pool.
// Represents an agent pool.
message AgentPool {
option (google.api.resource) = {
type: "storagetransfer.googleapis.com/agentPools"
Expand All @@ -485,8 +528,8 @@ message AgentPool {
// Default value. This value is unused.
STATE_UNSPECIFIED = 0;

// This is an initialization state. During this stage, the resources such as
// Pub/Sub topics are allocated for the AgentPool.
// This is an initialization state. During this stage, resources are
// allocated for the AgentPool.
CREATING = 1;

// Determines that the AgentPool is created for use. At this state, Agents
Expand Down Expand Up @@ -602,6 +645,9 @@ message TransferSpec {

// An AWS S3 compatible data source.
AwsS3CompatibleData aws_s3_compatible_data_source = 19;

// An HDFS cluster data source.
HdfsData hdfs_data_source = 20;
}

oneof intermediate_data_location {
Expand Down Expand Up @@ -768,10 +814,10 @@ message MetadataOptions {
// Do not preserve the `timeCreated` metadata from the source object.
TIME_CREATED_SKIP = 1;

// Preserves the source object's `timeCreated` metadata in the `customTime`
// field in the destination object. Note that any value stored in the
// source object's `customTime` field will not be propagated to the
// destination object.
// Preserves the source object's `timeCreated` or `lastModified` metadata in
// the `customTime` field in the destination object. Note that any value
// stored in the source object's `customTime` field will not be propagated
// to the destination object.
TIME_CREATED_PRESERVE_AS_CUSTOM_TIME = 2;
}

Expand Down Expand Up @@ -818,9 +864,10 @@ message MetadataOptions {
KmsKey kms_key = 8;

// Specifies how each object's `timeCreated` metadata is preserved for
// transfers between Google Cloud Storage buckets. If unspecified, the
// default behavior is the same as
// transfers. If unspecified, the default behavior is the same as
// [TIME_CREATED_SKIP][google.storagetransfer.v1.MetadataOptions.TimeCreated.TIME_CREATED_SKIP].
// This behavior is supported for transfers to Cloud Storage buckets from
// Cloud Storage, Amazon S3, S3-compatible storage, and Azure sources.
TimeCreated time_created = 9;
}

Expand Down Expand Up @@ -986,8 +1033,7 @@ message TransferJob {
// Transfer specification.
TransferSpec transfer_spec = 4;

// Notification configuration. This is not supported for transfers involving
// PosixFilesystem.
// Notification configuration.
NotificationConfig notification_config = 11;

// Logging configuration.
Expand Down Expand Up @@ -1219,16 +1265,10 @@ message NotificationConfig {

// Specifies the logging behavior for transfer operations.
//
// For cloud-to-cloud transfers, logs are sent to Cloud Logging. See
// Logs can be sent to Cloud Logging for all transfer types. See
// [Read transfer
// logs](https://cloud.google.com/storage-transfer/docs/read-transfer-logs) for
// details.
//
// For transfers to or from a POSIX file system, logs are stored in the
// Cloud Storage bucket that is the source or sink of the transfer.
// See [Managing Transfer for on-premises jobs]
// (https://cloud.google.com/storage-transfer/docs/managing-on-prem-jobs#viewing-logs)
// for details.
message LoggingConfig {
// Loggable actions.
enum LoggableAction {
Expand Down Expand Up @@ -1260,19 +1300,18 @@ message LoggingConfig {
}

// Specifies the actions to be logged. If empty, no logs are generated.
// Not supported for transfers with PosixFilesystem data sources; use
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
// instead.
repeated LoggableAction log_actions = 1;

// States in which `log_actions` are logged. If empty, no logs are generated.
// Not supported for transfers with PosixFilesystem data sources; use
// [enable_onprem_gcs_transfer_logs][google.storagetransfer.v1.LoggingConfig.enable_onprem_gcs_transfer_logs]
// instead.
repeated LoggableActionState log_action_states = 2;

// For transfers with a PosixFilesystem source, this option enables the Cloud
// Storage transfer logs for this transfer.
// For PosixFilesystem transfers, enables
// [file system transfer
// logs](https://cloud.google.com/storage-transfer/docs/on-prem-transfer-log-format)
// instead of, or in addition to, Cloud Logging.
//
// This option ignores [LoggableAction] and [LoggableActionState]. If these
// are set, Cloud Logging will also be enabled for this transfer.
bool enable_onprem_gcs_transfer_logs = 3;
}

Expand Down
Loading

0 comments on commit 7500af4

Please sign in to comment.