Skip to content

Commit

Permalink
chore(samples): Major update according review comments. (#369)
Browse files Browse the repository at this point in the history
* Events crud changes.

* Create/remove resources changes.

* Search changes.

* Import user events changes.

* Product changes.

* please revert this commit before merging

* Product changes according to review comments.

* Fix createTable NPE.

* Resources logic, minor changes.

* Remove init resources tests.

* Add test resources creation.

* Update config.

* Change bucket name read logic.

* Change bucket name read logic.

* Change bucket name read logic.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Code style changes.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* Minor code changes.

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

* 🦉 Updates from OwlBot post-processor

See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

Co-authored-by: Neenu Shaji <Neenu1995@users.noreply.github.com>
Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
3 people authored and Shabirmean committed Nov 15, 2022
1 parent c5b2e61 commit 0ed9463
Show file tree
Hide file tree
Showing 55 changed files with 965 additions and 934 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import com.google.cloud.retail.v2.UserEventServiceClient;
import com.google.longrunning.Operation;
import com.google.longrunning.OperationsClient;
import events.setup.EventsCreateGcsBucket;
import java.io.IOException;

public class ImportUserEventsGcs {
Expand All @@ -55,7 +56,7 @@ public static void main(String[] args) throws IOException, InterruptedException
public static void importUserEventsFromGcs(String gcsEventsObject, String defaultCatalog)
throws IOException, InterruptedException {
try {
String gcsBucket = String.format("gs://%s", System.getenv("EVENTS_BUCKET_NAME"));
String gcsBucket = String.format("gs://%s", EventsCreateGcsBucket.getBucketName());
String gcsErrorsBucket = String.format("%s/error", gcsBucket);

GcsSource gcsSource =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@
import static setup.SetupCleanup.writeUserEvent;

import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.PurgeMetadata;
import com.google.cloud.retail.v2.PurgeUserEventsRequest;
import com.google.cloud.retail.v2.PurgeUserEventsResponse;
Expand All @@ -38,7 +39,7 @@ public class PurgeUserEvent {
public static void main(String[] args)
throws IOException, ExecutionException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = System.getenv("PROJECT_ID");
String projectId = ServiceOptions.getDefaultProjectId();
String defaultCatalog =
String.format("projects/%s/locations/global/catalogs/default_catalog", projectId);
// visitorId generated randomly.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,6 +26,7 @@
import static setup.SetupCleanup.writeUserEvent;

import com.google.api.gax.longrunning.OperationFuture;
import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.RejoinUserEventsMetadata;
import com.google.cloud.retail.v2.RejoinUserEventsRequest;
import com.google.cloud.retail.v2.RejoinUserEventsRequest.UserEventRejoinScope;
Expand All @@ -40,7 +41,7 @@ public class RejoinUserEvent {
public static void main(String[] args)
throws IOException, ExecutionException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = System.getenv("PROJECT_ID");
String projectId = ServiceOptions.getDefaultProjectId();
String defaultCatalog =
String.format("projects/%s/locations/global/catalogs/default_catalog", projectId);
// visitorId generated randomly.
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@

import static setup.SetupCleanup.purgeUserEvent;

import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.UserEvent;
import com.google.cloud.retail.v2.UserEventServiceClient;
import com.google.cloud.retail.v2.WriteUserEventRequest;
Expand All @@ -38,17 +39,17 @@ public class WriteUserEvent {
public static void main(String[] args)
throws IOException, ExecutionException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = System.getenv("PROJECT_ID");
String projectId = ServiceOptions.getDefaultProjectId();
String defaultCatalog =
String.format("projects/%s/locations/global/catalogs/default_catalog", projectId);
// visitorId generated randomly.
String visitorId = UUID.randomUUID().toString();

writeUserEvent(defaultCatalog, visitorId);
purgeUserEvent(visitorId);
}

public static void writeUserEvent(String defaultCatalog, String visitorId) throws IOException {
public static void writeUserEvent(String defaultCatalog, String visitorId)
throws IOException, ExecutionException, InterruptedException {
// Initialize client that will be used to send requests. This client only needs to be created
// once, and can be reused for multiple requests. After completing all of your requests, call
// the "close" method on the client to safely clean up any remaining background resources.
Expand All @@ -74,6 +75,8 @@ public static void writeUserEvent(String defaultCatalog, String visitorId) throw
userEventServiceClient.writeUserEvent(writeUserEventRequest);
System.out.printf("Written user event: %s%n", userEvent);
}

purgeUserEvent(visitorId);
}
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,6 @@

package events.setup;

import static events.setup.EventsCreateGcsBucket.eventsCreateGcsBucketAndUploadJsonFiles;
import static setup.SetupCleanup.createBqDataset;
import static setup.SetupCleanup.createBqTable;
import static setup.SetupCleanup.getGson;
Expand All @@ -31,9 +30,7 @@

public class EventsCreateBigQueryTable {

public static void createBqTableWithEvents() throws IOException {
eventsCreateGcsBucketAndUploadJsonFiles();

public static void main(String... args) throws IOException {
String dataset = "user_events";
String validEventsTable = "events";
String invalidEventsTable = "events_some_invalid";
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -19,22 +19,40 @@
import static setup.SetupCleanup.createBucket;
import static setup.SetupCleanup.uploadObject;

import com.google.cloud.ServiceOptions;
import com.google.protobuf.Timestamp;
import java.io.IOException;
import java.time.Instant;

public class EventsCreateGcsBucket {

private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
private static final String PROJECT_ID = ServiceOptions.getDefaultProjectId();

public static String getBucketName() {
return BUCKET_NAME;
}
private static final Timestamp CURRENT_DATE =
Timestamp.newBuilder()
.setSeconds(Instant.now().getEpochSecond())
.setNanos(Instant.now().getNano())
.build();

public static void eventsCreateGcsBucketAndUploadJsonFiles() throws IOException {
private static final String BUCKET_NAME =
String.format("%s_events_%s", PROJECT_ID, CURRENT_DATE.getSeconds());

public static void main(String... args) throws IOException {
createBucket(BUCKET_NAME);
System.out.printf("Events gcs bucket %s was created.", BUCKET_NAME);

uploadObject(BUCKET_NAME, "user_events.json", "src/main/resources/user_events.json");
System.out.printf("File 'user_events.json' was uploaded into bucket '%s'.", BUCKET_NAME);

uploadObject(
BUCKET_NAME,
"user_events_some_invalid.json",
"src/main/resources/user_events_some_invalid.json");
System.out.printf(
"File 'user_events_some_invalid.json' was uploaded into bucket '%s'.", BUCKET_NAME);
}

public static String getBucketName() {
return BUCKET_NAME;
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -16,11 +16,7 @@

package init;

import static events.setup.EventsCreateBigQueryTable.createBqTableWithEvents;
import static events.setup.EventsCreateGcsBucket.eventsCreateGcsBucketAndUploadJsonFiles;
import static product.setup.ProductsCreateBigqueryTable.createBqTableWithProducts;
import static product.setup.ProductsCreateGcsBucket.productsCreateGcsBucketAndUploadJsonFiles;

import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.GcsSource;
import com.google.cloud.retail.v2.ImportErrorsConfig;
import com.google.cloud.retail.v2.ImportMetadata;
Expand All @@ -31,56 +27,57 @@
import com.google.cloud.retail.v2.ProductServiceClient;
import com.google.longrunning.Operation;
import com.google.longrunning.OperationsClient;
import events.setup.EventsCreateBigQueryTable;
import events.setup.EventsCreateGcsBucket;
import java.io.IOException;
import java.util.Collections;
import product.setup.ProductsCreateBigqueryTable;
import product.setup.ProductsCreateGcsBucket;

public class CreateTestResources {
private static final String PROJECT_ID = System.getenv("PROJECT_ID");
private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
private static final String GCS_BUCKET = String.format("gs://%s", System.getenv("BUCKET_NAME"));
private static final String GCS_ERROR_BUCKET = String.format("%s/errors", GCS_BUCKET);
private static final String DEFAULT_CATALOG =
String.format(
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/0", PROJECT_ID);

public static void main(String[] args) throws IOException, InterruptedException {
productsCreateGcsBucketAndUploadJsonFiles();
eventsCreateGcsBucketAndUploadJsonFiles();
importProductsFromGcs();
createBqTableWithProducts();
createBqTableWithEvents();
public static void main(String... args) throws IOException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = ServiceOptions.getDefaultProjectId();
String bucketName = System.getenv("BUCKET_NAME");
String gcsBucket = String.format("gs://%s", System.getenv("BUCKET_NAME"));
String gcsErrorBucket = String.format("%s/errors", gcsBucket);
String branchName =
String.format(
"projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId);

ProductsCreateGcsBucket.main();
EventsCreateGcsBucket.main();
importProductsFromGcs(bucketName, gcsErrorBucket, branchName);
ProductsCreateBigqueryTable.main();
EventsCreateBigQueryTable.main();
}

public static ImportProductsRequest getImportProductsGcsRequest(String gcsObjectName) {
public static void importProductsFromGcs(
String bucketName, String gcsErrorBucket, String branchName)
throws IOException, InterruptedException {
GcsSource gcsSource =
GcsSource.newBuilder()
.addAllInputUris(
Collections.singleton(String.format("gs://%s/%s", BUCKET_NAME, gcsObjectName)))
Collections.singleton(String.format("gs://%s/%s", bucketName, "products.json")))
.build();
ProductInputConfig inputConfig =
ProductInputConfig.newBuilder().setGcsSource(gcsSource).build();
System.out.println("GRS source: " + gcsSource.getInputUrisList());

ImportErrorsConfig errorsConfig =
ImportErrorsConfig.newBuilder().setGcsPrefix(GCS_ERROR_BUCKET).build();
ImportErrorsConfig.newBuilder().setGcsPrefix(gcsErrorBucket).build();
ImportProductsRequest importRequest =
ImportProductsRequest.newBuilder()
.setParent(DEFAULT_CATALOG)
.setParent(branchName)
.setReconciliationMode(ReconciliationMode.INCREMENTAL)
.setInputConfig(inputConfig)
.setErrorsConfig(errorsConfig)
.build();
System.out.println("Import products from google cloud source request: " + importRequest);

return importRequest;
}

public static void importProductsFromGcs() throws IOException, InterruptedException {
ImportProductsRequest importGcsRequest = getImportProductsGcsRequest("products.json");

try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
String operationName =
serviceClient.importProductsCallable().call(importGcsRequest).getName();
String operationName = serviceClient.importProductsCallable().call(importRequest).getName();
System.out.printf("OperationName = %s\n", operationName);

OperationsClient operationsClient = serviceClient.getOperationsClient();
Expand All @@ -89,8 +86,7 @@ public static void importProductsFromGcs() throws IOException, InterruptedExcept
while (!operation.getDone()) {
System.out.println("Please wait till operation is completed.");
// Keep polling the operation periodically until the import task is done.
int awaitDuration = 30000;
Thread.sleep(awaitDuration);
Thread.sleep(30_000);
operation = operationsClient.getOperation(operationName);
}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@
import static setup.SetupCleanup.deleteDataset;

import com.google.api.gax.rpc.PermissionDeniedException;
import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.DeleteProductRequest;
import com.google.cloud.retail.v2.ListProductsRequest;
import com.google.cloud.retail.v2.Product;
Expand All @@ -29,25 +30,26 @@

public class RemoveTestResources {

private static final String PROJECT_ID = System.getenv("PROJECT_ID");
private static final String BUCKET_NAME = System.getenv("BUCKET_NAME");
private static final String DEFAULT_CATALOG =
String.format(
"projects/%s/locations/global/catalogs/default_catalog/" + "branches/0", PROJECT_ID);
public static void main(String... args) throws IOException {
// TODO(developer): Replace these variables before running the sample.
String projectId = ServiceOptions.getDefaultProjectId();
String bucketName = System.getenv("BUCKET_NAME");
String branchName =
String.format(
"projects/%s/locations/global/catalogs/default_catalog/branches/0", projectId);

public static void main(String[] args) throws IOException {
deleteBucket(BUCKET_NAME);
deleteAllProducts();
deleteDataset(PROJECT_ID, "products");
deleteDataset(PROJECT_ID, "user_events");
deleteBucket(bucketName);
deleteAllProducts(branchName);
deleteDataset(projectId, "products");
deleteDataset(projectId, "user_events");
}

public static void deleteAllProducts() throws IOException {
public static void deleteAllProducts(String branchName) throws IOException {
System.out.println("Deleting products in process, please wait...");

try (ProductServiceClient productServiceClient = ProductServiceClient.create()) {
ListProductsRequest listRequest =
ListProductsRequest.newBuilder().setParent(DEFAULT_CATALOG).build();
ListProductsRequest.newBuilder().setParent(branchName).build();
ListProductsPagedResponse products = productServiceClient.listProducts(listRequest);

int deleteCount = 0;
Expand All @@ -66,7 +68,7 @@ public static void deleteAllProducts() throws IOException {
}
}

System.out.printf("%s products were deleted from %s%n", deleteCount, DEFAULT_CATALOG);
System.out.printf("%s products were deleted from %s%n", deleteCount, branchName);
}
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@
import static setup.SetupCleanup.deleteProduct;
import static setup.SetupCleanup.getProduct;

import com.google.cloud.ServiceOptions;
import com.google.cloud.retail.v2.AddFulfillmentPlacesRequest;
import com.google.cloud.retail.v2.ProductServiceClient;
import com.google.protobuf.Timestamp;
Expand All @@ -35,11 +36,11 @@ public class AddFulfillmentPlaces {

public static void main(String[] args) throws IOException, InterruptedException {
// TODO(developer): Replace these variables before running the sample.
String projectId = System.getenv("PROJECT_ID");
String projectId = ServiceOptions.getDefaultProjectId();
String generatedProductId = UUID.randomUUID().toString();
String productName =
String.format(
"projects/%s/locations/global/catalogs/default_catalog/branches/" + "0/products/%s",
"projects/%s/locations/global/catalogs/default_catalog/branches/0/products/%s",
projectId, generatedProductId);
Timestamp currentDate =
Timestamp.newBuilder()
Expand All @@ -57,13 +58,15 @@ public static void addFulfillmentPlaces(String productName, Timestamp timestamp,
throws IOException, InterruptedException {
AddFulfillmentPlacesRequest addFulfillmentRequest =
getAddFulfillmentRequest(productName, timestamp, placeId);
ProductServiceClient.create().addFulfillmentPlacesAsync(addFulfillmentRequest);
/*
This is a long-running operation and its result is not immediately
present with get operations,thus we simulate wait with sleep method.
*/
System.out.println("Add fulfillment places, wait 30 seconds: ");
Thread.sleep(30_000);
try (ProductServiceClient serviceClient = ProductServiceClient.create()) {
serviceClient.addFulfillmentPlacesAsync(addFulfillmentRequest);
/*
This is a long-running operation and its result is not immediately
present with get operations,thus we simulate wait with sleep method.
*/
System.out.println("Add fulfillment places, wait 30 seconds: ");
Thread.sleep(30_000);
}
}

public static AddFulfillmentPlacesRequest getAddFulfillmentRequest(
Expand Down
Loading

0 comments on commit 0ed9463

Please sign in to comment.