Added support for Amazon OpenSearch Serverless. (#96)

Signed-off-by: dblock <dblock@amazon.com>

CHANGELOG.md

@@ -6,6 +6,7 @@ Inspired from [Keep a Changelog](https://keepachangelog.com/en/1.0.0/)
 ### Added
 - Adds Github workflow for changelog verification ([#89](https://github.com/opensearch-project/opensearch-rs/pull/89))
 - Adds Github workflow for unit tests ([#112](https://github.com/opensearch-project/opensearch-rs/pull/112))
+- Adds support for OpenSearch Serverless ([#96](https://github.com/opensearch-project/opensearch-rs/pull/96))
 
 ### Dependencies
 - Bumps `simple_logger` from 2.3.0 to 4.0.0

USER_GUIDE.md

@@ -6,7 +6,7 @@
     - [Add a Document to the Index](#add-a-document-to-the-index)
     - [Search for a Document](#search-for-a-document)
     - [Delete the Index](#delete-the-index)
-  - [Amazon OpenSearch Service](#amazon-opensearch-service)
+  - [Amazon OpenSearch and OpenSearch Serverless](#amazon-opensearch-and-opensearch-serverless)
     - [Create a Client](#create-a-client-1)
 
 # User Guide
@@ -103,21 +103,23 @@ client
     .await?;
 ```
 
-## Amazon OpenSearch Service
+## Amazon OpenSearch and OpenSearch Serverless
 
-This library supports [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/).
+This library supports [Amazon OpenSearch Service](https://aws.amazon.com/opensearch-service/) and [OpenSearch Serverless](https://docs.aws.amazon.com/opensearch-service/latest/developerguide/serverless.html).
 
 ### Create a Client
 
 Create a client with AWS credentials as follows. Make sure to specify the correct service name and signing region.
 
 ```rust
 let url = Url::parse("https://...");
+let service_name = "es"; // use "aoss" for OpenSearch Serverless
 let conn_pool = SingleNodeConnectionPool::new(url?);
 let region_provider = RegionProviderChain::default_provider().or_else("us-east-1");
 let aws_config = aws_config::from_env().region(region_provider).load().await.clone();
 let transport = TransportBuilder::new(conn_pool)
     .auth(aws_config.clone().try_into()?)
+    .service_name(service_name)
     .build()?;
 let client = OpenSearch::new(transport);
 ```

opensearch/examples/cat_indices.rs

@@ -65,7 +65,9 @@ fn create_client() -> Result<OpenSearch, Error> {
 
     /// Determines if Fiddler.exe proxy process is running
     fn running_proxy() -> bool {
-        let system = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::default()));
+        let system = System::new_with_specifics(
+            RefreshKind::new().with_processes(ProcessRefreshKind::default()),
+        );
         let has_fiddler = system.processes_by_name("Fiddler").next().is_some();
         has_fiddler
     }

opensearch/src/http/aws_auth.rs

@@ -12,29 +12,33 @@
 use std::time::SystemTime;
 
 use aws_credential_types::{
-    Credentials, 
-    provider::{ProvideCredentials, SharedCredentialsProvider}
+    provider::{ProvideCredentials, SharedCredentialsProvider},
+    Credentials,
 };
 use aws_sigv4::{
-    http_request::{sign, SignableBody, SignableRequest, SigningParams, SigningSettings},
+    http_request::{
+        sign, PayloadChecksumKind, SignableBody, SignableRequest, SigningParams, SigningSettings,
+    },
     signing_params::BuildError,
 };
 use aws_types::region::Region;
 use reqwest::Request;
 
-const SERVICE_NAME: &str = "es";
-
 fn get_signing_params<'a>(
     credentials: &'a Credentials,
+    service_name: &'a str,
     region: &'a Region,
 ) -> Result<SigningParams<'a>, BuildError> {
+    let mut signing_settings = SigningSettings::default();
+    signing_settings.payload_checksum_kind = PayloadChecksumKind::XAmzSha256; // required for OpenSearch Serverless
+
     let mut builder = SigningParams::builder()
         .access_key(credentials.access_key_id())
         .secret_key(credentials.secret_access_key())
-        .service_name(SERVICE_NAME)
+        .service_name(service_name)
         .region(region.as_ref())
         .time(SystemTime::now())
-        .settings(SigningSettings::default());
+        .settings(signing_settings);
 
     builder.set_security_token(credentials.session_token());
 
@@ -44,11 +48,12 @@ fn get_signing_params<'a>(
 pub async fn sign_request(
     request: &mut Request,
     credentials_provider: &SharedCredentialsProvider,
+    service_name: &str,
     region: &Region,
 ) -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
     let credentials = credentials_provider.provide_credentials().await?;
 
-    let params = get_signing_params(&credentials, region)?;
+    let params = get_signing_params(&credentials, service_name, region)?;
 
     let uri = request.url().as_str().parse()?;
 

opensearch/src/http/transport.rs

@@ -154,6 +154,8 @@ pub struct TransportBuilder {
     disable_proxy: bool,
     headers: HeaderMap,
     timeout: Option<Duration>,
+    #[cfg(feature = "aws-auth")]
+    service_name: String,
 }
 
 impl TransportBuilder {
@@ -174,6 +176,8 @@ impl TransportBuilder {
             disable_proxy: false,
             headers: HeaderMap::new(),
             timeout: None,
+            #[cfg(feature = "aws-auth")]
+            service_name: "es".to_string(),
         }
     }
 
@@ -241,6 +245,15 @@ impl TransportBuilder {
         self
     }
 
+    /// Sets a global AWS service name.
+    ///
+    /// Default is "es". Other supported services are "aoss" for OpenSearch Serverless.
+    #[cfg(feature = "aws-auth")]
+    pub fn service_name(mut self, service_name: &str) -> Self {
+        self.service_name = service_name.to_string();
+        self
+    }
+
     /// Builds a [Transport] to use to send API calls to Elasticsearch.
     pub fn build(self) -> Result<Transport, BuildError> {
         let mut client_builder = self.client_builder;
@@ -313,6 +326,8 @@ impl TransportBuilder {
             client,
             conn_pool: self.conn_pool,
             credentials: self.credentials,
+            #[cfg(feature = "aws-auth")]
+            service_name: self.service_name,
         })
     }
 }
@@ -352,6 +367,8 @@ pub struct Transport {
     client: reqwest::Client,
     credentials: Option<Credentials>,
     conn_pool: Box<dyn ConnectionPool>,
+    #[cfg(feature = "aws-auth")]
+    service_name: String,
 }
 
 impl Transport {
@@ -460,9 +477,14 @@ impl Transport {
 
         #[cfg(feature = "aws-auth")]
         if let Some(Credentials::AwsSigV4(credentials_provider, region)) = &self.credentials {
-            super::aws_auth::sign_request(&mut request, credentials_provider, region)
-                .await
-                .map_err(|e| crate::error::lib(format!("AWSV4 Signing Failed: {}", e)))?;
+            super::aws_auth::sign_request(
+                &mut request,
+                credentials_provider,
+                &self.service_name,
+                region,
+            )
+            .await
+            .map_err(|e| crate::error::lib(format!("AWSV4 Signing Failed: {}", e)))?;
         }
 
         let response = self.client.execute(request).await;

opensearch/src/lib.rs

@@ -58,7 +58,7 @@
 //! - **experimental-apis**: Enables experimental APIs. Experimental APIs are just that - an experiment. An experimental
 //!   API might have breaking changes in any future version, or it might even be removed entirely. This feature also
 //!   enables `beta-apis`.
-//! - **aws-auth**: Enables authentication with Amazon OpenSearch.
+//! - **aws-auth**: Enables authentication with Amazon OpenSearch and OpenSearch Serverless.
 //!   Performs AWS SigV4 signing using credential types from `aws-types`.
 //!
 //! # Getting started
@@ -327,9 +327,9 @@
 //! # }
 //! ```
 //!
-//! ## Amazon OpenSearch
+//! ## Amazon OpenSearch and OpenSearch Serverless
 //!
-//! For authenticating against an Amazon OpenSearch endpoint using AWS SigV4 request signing,
+//! For authenticating against an Amazon OpenSearch or OpenSearch Serverless endpoint using AWS SigV4 request signing,
 //! you must enable the `aws-auth` feature, then pass the AWS credentials to the [TransportBuilder](http::transport::TransportBuilder).
 //! The easiest way to retrieve AWS credentials in the required format is to use [aws-config](https://docs.rs/aws-config/latest/aws_config/).
 //!
@@ -349,14 +349,15 @@
 //! # use std::convert::TryInto;
 //! # #[tokio::main]
 //! # async fn main() -> Result<(), Box<dyn std::error::Error>> {
+//! # #[cfg(feature = "aws-auth")] {
 //! let creds = aws_config::load_from_env().await;
-//! let url = Url::parse("https://example.com")?;
+//! let url = Url::parse("https://...")?;
 //! let region_provider = RegionProviderChain::default_provider().or_else("us-east-1");
 //! let aws_config = aws_config::from_env().region(region_provider).load().await.clone();
 //! let conn_pool = SingleNodeConnectionPool::new(url);
-//! # #[cfg(feature = "aws-auth")] {
 //! let transport = TransportBuilder::new(conn_pool)
 //!     .auth(aws_config.clone().try_into()?)
+//!     .service_name("es") // use "aoss" for OpenSearch Serverless
 //!     .build()?;
 //! let client = OpenSearch::new(transport);
 //! # }

opensearch/tests/auth.rs

@@ -34,8 +34,6 @@ use common::*;
 use opensearch::auth::Credentials;
 
 use base64::{prelude::BASE64_STANDARD, write::EncoderWriter as Base64Encoder};
-// use std::fs::File;
-// use std::io::Read;
 use std::io::Write;
 
 #[tokio::test]

opensearch/tests/aws_auth.rs

@@ -0,0 +1,86 @@
+/*
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * The OpenSearch Contributors require contributions made to
+ * this file be licensed under the Apache-2.0 license or a
+ * compatible open source license.
+ *
+ * Modifications Copyright OpenSearch Contributors. See
+ * GitHub history for details.
+ */
+
+#![cfg(feature = "aws-auth")]
+
+pub mod common;
+use common::*;
+use opensearch::OpenSearch;
+use regex::Regex;
+
+use aws_config::SdkConfig;
+use aws_credential_types::provider::SharedCredentialsProvider;
+use aws_credential_types::Credentials;
+use aws_types::region::Region;
+use std::convert::TryInto;
+
+#[tokio::test]
+async fn aws_auth_get() -> Result<(), failure::Error> {
+    let server = server::http(move |req| async move {
+        let authorization_header = req.headers()["authorization"].to_str().unwrap();
+        let re = Regex::new(r"^AWS4-HMAC-SHA256 Credential=id/\d*/us-west-1/custom/aws4_request, SignedHeaders=accept;content-type;host;x-amz-content-sha256;x-amz-date, Signature=[a-f,0-9].*$").unwrap();
+        assert!(
+            re.is_match(authorization_header),
+            "{}",
+            authorization_header
+        );
+        let amz_content_sha256_header = req.headers()["x-amz-content-sha256"].to_str().unwrap();
+        assert_eq!(
+            amz_content_sha256_header,
+            "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"
+        ); // SHA of empty string
+        http::Response::default()
+    });
+
+    let client = create_aws_client(format!("http://{}", server.addr()).as_ref())?;
+    let _response = client.ping().send().await?;
+
+    Ok(())
+}
+
+#[tokio::test]
+async fn aws_auth_post() -> Result<(), failure::Error> {
+    let server = server::http(move |req| async move {
+        let amz_content_sha256_header = req.headers()["x-amz-content-sha256"].to_str().unwrap();
+        assert_eq!(
+            amz_content_sha256_header,
+            "f3a842f988a653a734ebe4e57c45f19293a002241a72f0b3abbff71e4f5297b9"
+        ); // SHA of the JSON
+        http::Response::default()
+    });
+
+    let client = create_aws_client(format!("http://{}", server.addr()).as_ref())?;
+    client
+        .index(opensearch::IndexParts::Index("movies"))
+        .body(serde_json::json!({
+                "title": "Moneyball",
+                "director": "Bennett Miller",
+                "year": 2011
+            }
+        ))
+        .send()
+        .await?;
+
+    Ok(())
+}
+
+fn create_aws_client(addr: &str) -> Result<OpenSearch, failure::Error> {
+    let aws_creds = Credentials::new("id", "secret", None, None, "token");
+    let creds_provider = SharedCredentialsProvider::new(aws_creds);
+    let aws_config = SdkConfig::builder()
+        .credentials_provider(creds_provider)
+        .region(Region::new("us-west-1"))
+        .build();
+    let builder = client::create_builder(addr)
+        .auth(aws_config.clone().try_into()?)
+        .service_name("custom");
+    Ok(client::create(builder))
+}

opensearch/tests/cert.rs

@@ -21,7 +21,10 @@
 //!
 //!
 //! DETACH=true .ci/run-opensearch.sh
-#![cfg(all(feature = "_integration", any(feature = "native-tls", feature = "rustls-tls")))]
+#![cfg(all(
+    feature = "_integration",
+    any(feature = "native-tls", feature = "rustls-tls")
+))]
 
 pub mod common;
 use common::*;

opensearch/tests/common/client.rs

@@ -56,7 +56,9 @@ pub fn cluster_addr() -> String {
 
 /// Checks if Fiddler proxy process is running
 fn running_proxy() -> bool {
-    let system = System::new_with_specifics(RefreshKind::new().with_processes(ProcessRefreshKind::default()));
+    let system = System::new_with_specifics(
+        RefreshKind::new().with_processes(ProcessRefreshKind::default()),
+    );
     let has_fiddler = system.processes_by_name("Fiddler").next().is_some();
     has_fiddler
 }

opensearch/tests/error.rs

@@ -28,7 +28,7 @@
  * GitHub history for details.
  */
 
- #![cfg(feature = "_integration")]
+#![cfg(feature = "_integration")]
 
 pub mod common;
 use common::*;

yaml_test_runner/Cargo.toml

@@ -31,7 +31,7 @@ serde_yaml = "0.9"
 serde_json = { version = "~1", features = ["arbitrary_precision"] }
 simple_logger = "4.0.0"
 syn = { version = "~1.0", features = ["full"] }
-sysinfo = "0.26.4"
+sysinfo = "0.26"
 url = "2.1.1"
 yaml-rust = "0.4.3"
 tar = "~0.4"