feat: [discoveryengine] support natural language understanding in sea…

…rch (#5596) * feat: support natural language understanding in search feat: support search as you type feature feat: allow set relevance threshold on search feat: support import and purge autocomplete data feat: populate more metadata in answer services feat: allow providing language info in data stores feat: add requirement service feat: support import data from AlloyDB feat: support add metadata for website uris docs: keep the API doc up-to-date with recent changes PiperOrigin-RevId: 658428107 Source-Link: googleapis/googleapis@98e5657 Source-Link: googleapis/googleapis-gen@a579c69 Copy-Tag: eyJwIjoicGFja2FnZXMvZ29vZ2xlLWNsb3VkLWRpc2NvdmVyeWVuZ2luZS8uT3dsQm90LnlhbWwiLCJoIjoiYTU3OWM2OTA4YWNkNjcwMWQ3ZjliYzY0YTVmMTc2YjIxNWJiYmZhYyJ9 * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * Update package.json --------- Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com> Co-authored-by: Daniel Bankhead <danielbankhead@google.com> Co-authored-by: sofisl <55454395+sofisl@users.noreply.github.com>
googleapis · Aug 15, 2024 · 5cf8d9e · 5cf8d9e
1 parent ea50493
commit 5cf8d9e
Show file tree

Hide file tree

Showing 54 changed files with 43,483 additions and 27,009 deletions.
diff --git a/packages/google-cloud-discoveryengine/README.md b/packages/google-cloud-discoveryengine/README.md
@@ -195,7 +195,9 @@ Samples are in the [`samples/`](https://github.com/googleapis/google-cloud-node/
 | Chunk_service.get_chunk | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/chunk_service.get_chunk.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/chunk_service.get_chunk.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Chunk_service.list_chunks | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/chunk_service.list_chunks.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/chunk_service.list_chunks.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Completion_service.complete_query | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.complete_query.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.complete_query.js,packages/google-cloud-discoveryengine/samples/README.md) |
+| Completion_service.import_completion_suggestions | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.import_completion_suggestions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.import_completion_suggestions.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Completion_service.import_suggestion_deny_list_entries | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.import_suggestion_deny_list_entries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.import_suggestion_deny_list_entries.js,packages/google-cloud-discoveryengine/samples/README.md) |
+| Completion_service.purge_completion_suggestions | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.purge_completion_suggestions.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.purge_completion_suggestions.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Completion_service.purge_suggestion_deny_list_entries | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.purge_suggestion_deny_list_entries.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/completion_service.purge_suggestion_deny_list_entries.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Control_service.create_control | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/control_service.create_control.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/control_service.create_control.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Control_service.delete_control | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/control_service.delete_control.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/control_service.delete_control.js,packages/google-cloud-discoveryengine/samples/README.md) |
@@ -280,8 +282,10 @@ Samples are in the [`samples/`](https://github.com/googleapis/google-cloud-node/
 | Site_search_engine_service.fetch_domain_verification_status | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.fetch_domain_verification_status.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.fetch_domain_verification_status.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Site_search_engine_service.get_site_search_engine | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_site_search_engine.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_site_search_engine.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Site_search_engine_service.get_target_site | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_target_site.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_target_site.js,packages/google-cloud-discoveryengine/samples/README.md) |
+| Site_search_engine_service.get_uri_pattern_document_data | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_uri_pattern_document_data.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.get_uri_pattern_document_data.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Site_search_engine_service.list_target_sites | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.list_target_sites.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.list_target_sites.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Site_search_engine_service.recrawl_uris | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.recrawl_uris.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.recrawl_uris.js,packages/google-cloud-discoveryengine/samples/README.md) |
+| Site_search_engine_service.set_uri_pattern_document_data | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.set_uri_pattern_document_data.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.set_uri_pattern_document_data.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | Site_search_engine_service.update_target_site | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.update_target_site.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/site_search_engine_service.update_target_site.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | User_event_service.collect_user_event | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/user_event_service.collect_user_event.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/user_event_service.collect_user_event.js,packages/google-cloud-discoveryengine/samples/README.md) |
 | User_event_service.import_user_events | [source code](https://github.com/googleapis/google-cloud-node/blob/main/packages/google-cloud-discoveryengine/samples/generated/v1alpha/user_event_service.import_user_events.js) | [![Open in Cloud Shell][shell_img]](https://console.cloud.google.com/cloudshell/open?git_repo=https://github.com/googleapis/google-cloud-node&page=editor&open_in_editor=packages/google-cloud-discoveryengine/samples/generated/v1alpha/user_event_service.import_user_events.js,packages/google-cloud-discoveryengine/samples/README.md) |

diff --git a/packages/google-cloud-discoveryengine/package.json b/packages/google-cloud-discoveryengine/package.json
@@ -43,7 +43,7 @@
     "postpack": "minifyProtoJson",
     "samples-test": "cd samples/ && npm link ../ && npm i && npm test",
     "system-test": "c8 mocha build/system-test",
-    "test": "c8 mocha build/test"
+    "test": "NODE_OPTIONS=--max-old-space-size=8192 c8 mocha build/test"
   },
   "dependencies": {
     "google-gax": "^4.0.3"
@@ -68,4 +68,4 @@
   "engines": {
     "node": ">=14.0.0"
   }
-}
+}
diff --git a/...ges/google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/answer.proto b/...ges/google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/answer.proto
@@ -69,6 +69,13 @@ message Answer {
 
         // Page identifier.
         string page_identifier = 2;
+
+        // The relevance of the chunk for a given query. Values range from 0.0
+        // (completely irrelevant) to 1.0 (completely relevant).
+        // This value is for informational purpose only. It may change for
+        // the same query and chunk at any time due to a model retraining or
+        // change in implementation.
+        optional float relevance_score = 3;
       }
 
       // Document resource name.
@@ -121,20 +128,38 @@ message Answer {
       // Chunk textual content.
       string content = 2;
 
-      // Relevance score.
+      // The relevance of the chunk for a given query. Values range from 0.0
+      // (completely irrelevant) to 1.0 (completely relevant).
+      // This value is for informational purpose only. It may change for
+      // the same query and chunk at any time due to a model retraining or
+      // change in implementation.
       optional float relevance_score = 3;
 
       // Document metadata.
       DocumentMetadata document_metadata = 4;
     }
 
+    // Structured search information.
+    message StructuredDocumentInfo {
+      // Document resource name.
+      string document = 1 [(google.api.resource_reference) = {
+        type: "discoveryengine.googleapis.com/Document"
+      }];
+
+      // Structured search data.
+      google.protobuf.Struct struct_data = 2;
+    }
+
     // Search result content.
     oneof content {
       // Unstructured document information.
       UnstructuredDocumentInfo unstructured_document_info = 1;
 
       // Chunk information.
       ChunkInfo chunk_info = 2;
+
+      // Structured document information.
+      StructuredDocumentInfo structured_document_info = 3;
     }
   }
 
@@ -168,7 +193,11 @@ message Answer {
             // Chunk textual content.
             string content = 2;
 
-            // Relevance score.
+            // The relevance of the chunk for a given query. Values range from
+            // 0.0 (completely irrelevant) to 1.0 (completely relevant).
+            // This value is for informational purpose only. It may change for
+            // the same query and chunk at any time due to a model retraining or
+            // change in implementation.
             optional float relevance_score = 3;
           }
 
@@ -188,6 +217,14 @@ message Answer {
           // If citation_type is CHUNK_LEVEL_CITATION and chunk mode is on,
           // populate chunk info.
           repeated ChunkInfo chunk_info = 5;
+
+          // Data representation.
+          // The structured JSON data for the document.
+          // It's populated from the struct data from the Document
+          // , or the Chunk in
+          // search result
+          // .
+          google.protobuf.Struct struct_data = 6;
         }
 
         // Search results observed by the search action, it can be snippets info
@@ -247,6 +284,9 @@ message Answer {
 
         // Non-answer-seeking query classification type.
         NON_ANSWER_SEEKING_QUERY = 2;
+
+        // Jail-breaking query classification type.
+        JAIL_BREAKING_QUERY = 3;
       }
 
       // Query classification type.
@@ -296,6 +336,19 @@ message Answer {
     // Google skips the answer if there is a potential policy violation
     // detected. This includes content that may be violent or toxic.
     POTENTIAL_POLICY_VIOLATION = 4;
+
+    // The no relevant content case.
+    //
+    // Google skips the answer if there is no relevant content in the
+    // retrieved search results.
+    NO_RELEVANT_CONTENT = 5;
+
+    // The jail-breaking query ignored case.
+    //
+    // For example, "Reply in the tone of a competing company's CEO".
+    // Google skips the answer if the query is classified as a jail-breaking
+    // query.
+    JAIL_BREAKING_QUERY_IGNORED = 6;
   }
 
   // Immutable. Fully qualified name

diff --git a/...ages/google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/chunk.proto b/...ages/google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/chunk.proto
@@ -96,7 +96,7 @@ message Chunk {
   // Content is a string from a document (parsed content).
   string content = 3;
 
-  // The relevance score based on similarity.
+  // Output only. Represents the relevance score based on similarity.
   // Higher score indicates higher chunk relevance.
   // The score is in range [-1.0, 1.0].
   // Only populated on [SearchService.SearchResponse][].

diff --git a/...google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/completion.proto b/...google-cloud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/completion.proto
@@ -50,3 +50,34 @@ message SuggestionDenyListEntry {
   // exact phrase, or block any suggestions containing this phrase.
   MatchOperator match_operator = 2 [(google.api.field_behavior) = REQUIRED];
 }
+
+// Autocomplete suggestions that are imported from Customer.
+message CompletionSuggestion {
+  // Ranking metrics of this suggestion.
+  oneof ranking_info {
+    // Global score of this suggestion. Control how this suggestion would be
+    // scored / ranked.
+    double global_score = 2;
+
+    // Frequency of this suggestion. Will be used to rank suggestions when score
+    // is not available.
+    int64 frequency = 3;
+  }
+
+  // Required. The suggestion text.
+  string suggestion = 1 [(google.api.field_behavior) = REQUIRED];
+
+  // BCP-47 language code of this suggestion.
+  string language_code = 4;
+
+  // If two suggestions have the same groupId, they will not be
+  // returned together. Instead the one ranked higher will be returned. This can
+  // be used to deduplicate semantically identical suggestions.
+  string group_id = 5;
+
+  // The score of this suggestion within its group.
+  double group_score = 6;
+
+  // Alternative matching phrases for this suggestion.
+  repeated string alternative_phrases = 7;
+}
diff --git a/...loud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/completion_service.proto b/...loud-discoveryengine/protos/google/cloud/discoveryengine/v1alpha/completion_service.proto
@@ -86,6 +86,44 @@ service CompletionService {
       metadata_type: "google.cloud.discoveryengine.v1alpha.PurgeSuggestionDenyListEntriesMetadata"
     };
   }
+
+  // Imports
+  // [CompletionSuggestion][google.cloud.discoveryengine.v1alpha.CompletionSuggestion]s
+  // for a DataStore.
+  rpc ImportCompletionSuggestions(ImportCompletionSuggestionsRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1alpha/{parent=projects/*/locations/*/collections/*/dataStores/*}/completionSuggestions:import"
+      body: "*"
+      additional_bindings {
+        post: "/v1alpha/{parent=projects/*/locations/*/dataStores/*}/completionSuggestions:import"
+        body: "*"
+      }
+    };
+    option (google.longrunning.operation_info) = {
+      response_type: "google.cloud.discoveryengine.v1alpha.ImportCompletionSuggestionsResponse"
+      metadata_type: "google.cloud.discoveryengine.v1alpha.ImportCompletionSuggestionsMetadata"
+    };
+  }
+
+  // Permanently deletes all
+  // [CompletionSuggestion][google.cloud.discoveryengine.v1alpha.CompletionSuggestion]s
+  // for a DataStore.
+  rpc PurgeCompletionSuggestions(PurgeCompletionSuggestionsRequest)
+      returns (google.longrunning.Operation) {
+    option (google.api.http) = {
+      post: "/v1alpha/{parent=projects/*/locations/*/collections/*/dataStores/*}/completionSuggestions:purge"
+      body: "*"
+      additional_bindings {
+        post: "/v1alpha/{parent=projects/*/locations/*/dataStores/*}/completionSuggestions:purge"
+        body: "*"
+      }
+    };
+    option (google.longrunning.operation_info) = {
+      response_type: "google.cloud.discoveryengine.v1alpha.PurgeCompletionSuggestionsResponse"
+      metadata_type: "google.cloud.discoveryengine.v1alpha.PurgeCompletionSuggestionsMetadata"
+    };
+  }
 }
 
 // Request message for