From 001f5c045efc833ef48efc69bbdeaa83a7e92ca5 Mon Sep 17 00:00:00 2001 From: Luis Moreno Date: Thu, 20 Apr 2023 13:16:27 -0400 Subject: [PATCH] Use trait to build context --- Cargo.lock | 1 + crates/fluvio-connector-common/src/lib.rs | 2 +- .../fluvio-connector-derive/src/generator.rs | 2 +- crates/fluvio-connector-package/Cargo.toml | 2 +- crates/fluvio-connector-package/src/lib.rs | 4 +- .../src/render/context.rs | 67 ++++++ .../src/render/mod.rs | 203 ++++++++++++++++++ .../fluvio-connector-package/src/renderer.rs | 170 --------------- 8 files changed, 276 insertions(+), 175 deletions(-) create mode 100644 crates/fluvio-connector-package/src/render/context.rs create mode 100644 crates/fluvio-connector-package/src/render/mod.rs delete mode 100644 crates/fluvio-connector-package/src/renderer.rs diff --git a/Cargo.lock b/Cargo.lock index 18095ded2fe..df045792461 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2725,6 +2725,7 @@ dependencies = [ "pretty_assertions", "regex", "serde", + "serde_json", "serde_yaml 0.9.21", "tempfile", "toml 0.7.3", diff --git a/crates/fluvio-connector-common/src/lib.rs b/crates/fluvio-connector-common/src/lib.rs index 6fa3dc722d6..67d4002ec4d 100644 --- a/crates/fluvio-connector-common/src/lib.rs +++ b/crates/fluvio-connector-common/src/lib.rs @@ -4,7 +4,7 @@ pub mod monitoring; pub mod consumer; pub mod config; -pub use fluvio_connector_package::resolve_config_str; +pub use fluvio_connector_package::render_config_str; pub use fluvio_connector_package::secret; #[cfg(feature = "derive")] diff --git a/crates/fluvio-connector-derive/src/generator.rs b/crates/fluvio-connector-derive/src/generator.rs index f27def542ad..d15591c91fa 100644 --- a/crates/fluvio-connector-derive/src/generator.rs +++ b/crates/fluvio-connector-derive/src/generator.rs @@ -117,7 +117,7 @@ fn init_and_parse_config(config_type_path: &Path) -> TokenStream { ::fluvio_connector_common::tracing::debug!(%config_str); /// Resolve any secrets/env in the config - let config_str_resolved =::fluvio_connector_common::resolve_config_str(&config_str)?; + let config_str_resolved =::fluvio_connector_common::render_config_str(&config_str)?; let config_value = ::fluvio_connector_common::config::value_from_reader(config_str_resolved.as_bytes())?; diff --git a/crates/fluvio-connector-package/Cargo.toml b/crates/fluvio-connector-package/Cargo.toml index f6e939bcda1..bce6ce104ec 100644 --- a/crates/fluvio-connector-package/Cargo.toml +++ b/crates/fluvio-connector-package/Cargo.toml @@ -14,13 +14,13 @@ default = ["toml"] anyhow = { workspace = true } bytesize = { workspace = true} humantime-serde = "1.1.1" -# Move to crates.io release, once https://github.com/mitsuhiko/minijinja/commit/78b9b87eaa68ac4c1b1c814d420a8e48e054fefc is released minijinja = { version = "0.32", default-features = false, features = ["custom_syntax"] } openapiv3 = { git = "https://github.com/galibey/openapiv3", rev = "bdd22f046d2bc19ede257504645d31f835545222", default-features = false } once_cell = { workspace = true } regex = { workspace = true } serde = { workspace = true, default-features = false, features = ["derive"] } serde_yaml = { workspace = true } +serde_json = { workspace = true } toml = { workspace = true , default-features = false, optional = true, features = ["preserve_order"] } tracing = { workspace = true } diff --git a/crates/fluvio-connector-package/src/lib.rs b/crates/fluvio-connector-package/src/lib.rs index 36f6200cf4c..84092de57db 100644 --- a/crates/fluvio-connector-package/src/lib.rs +++ b/crates/fluvio-connector-package/src/lib.rs @@ -1,6 +1,6 @@ pub mod metadata; pub mod config; pub mod secret; -mod renderer; +mod render; -pub use renderer::resolve_config_str; +pub use render::render_config_str; diff --git a/crates/fluvio-connector-package/src/render/context.rs b/crates/fluvio-connector-package/src/render/context.rs new file mode 100644 index 00000000000..cc1afa837cd --- /dev/null +++ b/crates/fluvio-connector-package/src/render/context.rs @@ -0,0 +1,67 @@ +use std::collections::HashMap; + +use serde::Serialize; + +use crate::secret::{SecretStore, detect_secrets_from_str}; + +/// Context for the template engine +/// This is the data that is available to the template engine +/// when it is rendering the template. +#[derive(Serialize, Default)] +pub(crate) struct Context(pub(crate) HashMap<&'static str, serde_json::Value>); + +/// ContextStore is a trait that allows for adding additional +/// context to the template engine. +pub(crate) trait ContextStore { + fn extract_context_values(&self, input: &str) -> anyhow::Result; + + fn context_name(&self) -> &'static str; +} +impl dyn ContextStore { + pub(crate) fn add_to_context(&self, context: &mut Context, input: &str) -> anyhow::Result<()> { + let value = self.extract_context_values(input)?; + context.0.insert(self.context_name(), value); + Ok(()) + } +} +impl ContextStore for &dyn SecretStore { + fn extract_context_values(&self, input: &str) -> anyhow::Result { + let secrets: HashMap<_, _> = detect_secrets_from_str(input)? + .into_iter() + .map(|s: String| self.read(&s).map(|secret_value| (s, secret_value))) + .collect::, _>>()?; + Ok(serde_json::to_value(secrets)?) + } + + fn context_name(&self) -> &'static str { + "secrets" + } +} + +#[cfg(test)] +mod test { + use super::ContextStore; + + pub struct TestStore; + + impl ContextStore for TestStore { + fn extract_context_values(&self, _input: &str) -> anyhow::Result { + Ok(serde_json::json!({"test": "value"})) + } + + fn context_name(&self) -> &'static str { + "test" + } + } + + #[test] + fn test_context_store() { + let store = &TestStore as &dyn ContextStore; + let mut context = super::Context::default(); + store.add_to_context(&mut context, "").unwrap(); + assert_eq!( + context.0.get("test").unwrap(), + &serde_json::json!({"test": "value"}) + ); + } +} diff --git a/crates/fluvio-connector-package/src/render/mod.rs b/crates/fluvio-connector-package/src/render/mod.rs new file mode 100644 index 00000000000..7b10682c391 --- /dev/null +++ b/crates/fluvio-connector-package/src/render/mod.rs @@ -0,0 +1,203 @@ +use minijinja::{Environment}; + +use crate::{ + secret::{self}, +}; + +use self::context::{Context, ContextStore}; + +mod context; + +/// Config renderer. This is the main entry point for rendering a config. +/// It is responsible for resolving secrets and rendering the config. +pub(crate) struct ConfigRenderer { + inner_renderer: Environment<'static>, + context_stores: Vec>, +} + +impl ConfigRenderer { + fn new( + inner_renderer: Environment<'static>, + context_stores: Vec>, + ) -> Self { + Self { + inner_renderer, + context_stores, + } + } + + fn new_with_context_stores(context_stores: Vec>) -> anyhow::Result { + let mut inner_renderer = Environment::new(); + inner_renderer.set_syntax(minijinja::Syntax { + block_start: "${%".into(), + block_end: "%}".into(), + variable_start: "${{".into(), + variable_end: "}}".into(), + comment_start: "${*".into(), + comment_end: "*}".into(), + })?; + + Ok(Self::new(inner_renderer, context_stores)) + } + + fn new_with_default_stores() -> anyhow::Result { + let context_stores = Self::default_stores()?; + + Self::new_with_context_stores(context_stores) + } + + fn render_str(&self, input: &str) -> anyhow::Result { + let context = self.build_context(input)?; + match self.inner_renderer.render_str(input, context) { + Ok(rendered) => Ok(rendered), + Err(e) => Err(anyhow::anyhow!("failed to render: `{}`. {}", input, e)), + } + } + + fn default_stores() -> anyhow::Result>> { + Ok(vec![Box::new(secret::default_secret_store()?)]) + } + + fn build_context(&self, input: &str) -> anyhow::Result { + let mut context = Context::default(); + for store in self.context_stores.iter() { + store.add_to_context(&mut context, input)?; + } + Ok(context) + } +} + +/// Render a config from a string. +/// This is the main entry point for rendering a config. +/// It is responsible for resolving secrets and rendering the config. +pub fn render_config_str(input: &str) -> anyhow::Result { + let renderer = ConfigRenderer::new_with_default_stores()?; + + let value = renderer.render_str(input)?; + + Ok(value) +} + +#[cfg(test)] +mod test { + use std::{io::Write, collections::HashMap}; + + use anyhow::Context; + use serde_yaml::Value; + + use crate::{ + secret::{FileSecretStore, detect_secrets_from_str}, + }; + + use super::{ConfigRenderer, context::ContextStore}; + + // mock secret store + pub struct SecretTestStore; + + impl ContextStore for SecretTestStore { + fn extract_context_values(&self, input: &str) -> anyhow::Result { + let values = + serde_json::json!({"foo": "bar", "api_key": "my_api_key", "interval": "10s"}); + let secrets: HashMap<_, _> = detect_secrets_from_str(input)? + .into_iter() + .map(|s: String| { + values + .get(&s) + .map(|secret_value| (s, secret_value)) + .context("not found in test values") + }) + .collect::, _>>()?; + Ok(serde_json::to_value(secrets)?) + } + + fn context_name(&self) -> &'static str { + "secrets" + } + } + + #[test] + fn test_build_context() { + let store = Box::new(SecretTestStore); + + let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap(); + let context: super::context::Context = + renderer.build_context("hello ${{ secrets.foo }}").unwrap(); + + assert_eq!(context.0["secrets"]["foo"], "bar"); + assert_eq!(context.0["secrets"].as_object().unwrap().len(), 1); + } + + #[test] + fn test_render_str() { + let store = Box::new(SecretTestStore); + let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap(); + let output = renderer.render_str("hello ${{ secrets.foo }}").unwrap(); + assert_eq!(output, "hello bar"); + } + + #[test] + fn test_render_str_undefined_secret() { + let store = Box::new(SecretTestStore); + + let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap(); + let output = renderer.render_str("hello ${{ secrets.undefined_var }}"); + assert!(output.is_err()); + } + + #[test] + fn test_invalid_syntax() { + let store = Box::new(SecretTestStore); + + let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap(); + let output = renderer.render_str("hello ${{"); + assert!(output.is_err()); + } + + #[test] + fn test_undefined_context_variable() { + let store = Box::new(SecretTestStore); + + let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap(); + let output = renderer.render_str("hello ${{ some_undefined.variable }}"); + assert!(output.is_err()); + } + + #[test] + fn test_resolve_config() { + use super::*; + let mut file = tempfile::NamedTempFile::new().expect("failed to create tmp file"); + file.write_all(b"foo=bar\napi_key=my_api_key\ninterval=10s") + .expect("file to write"); + let _ = secret::set_default_secret_store(::std::boxed::Box::new(FileSecretStore::from( + file.path(), + ))); + + let value_str = r#" + meta: + name: test + version: 0.1.0 + topic: test + type: http-source + consumer: + partition: 0 + my_service: + api_key: ${{secrets.api_key}} + interval: ${{ secrets.interval }} + "#; + + let value_str = render_config_str(value_str).unwrap(); + let value: Value = serde_yaml::from_str(&value_str).expect("should be yaml"); + assert_eq!( + value["my_service"]["api_key"] + .as_str() + .expect("should be str"), + "my_api_key" + ); + assert_eq!( + value["my_service"]["interval"] + .as_str() + .expect("should be str"), + "10s" + ); + } +} diff --git a/crates/fluvio-connector-package/src/renderer.rs b/crates/fluvio-connector-package/src/renderer.rs deleted file mode 100644 index 505474b54f0..00000000000 --- a/crates/fluvio-connector-package/src/renderer.rs +++ /dev/null @@ -1,170 +0,0 @@ -use std::collections::HashMap; - -use minijinja::Environment; -use serde::Serialize; - -use crate::{ - secret::{SecretStore, detect_secrets_from_str}, -}; - -/// Context for the template engine -/// This is the data that is available to the template engine -/// when it is rendering the template. -#[derive(Serialize)] -struct Context { - secrets: HashMap, -} - -/// Config renderer. This is the main entry point for rendering a config. -/// It is responsible for resolving secrets and rendering the config. -pub(crate) struct ConfigRenderer { - secret_store: &'static dyn SecretStore, - env: Environment<'static>, -} - -impl ConfigRenderer { - pub(crate) fn new(secret_store: &'static dyn SecretStore, env: Environment<'static>) -> Self { - Self { secret_store, env } - } - - pub fn new_with_default_stores() -> anyhow::Result { - let secret_store = crate::secret::default_secret_store()?; - let mut env = Environment::new(); - env.set_syntax(minijinja::Syntax { - block_start: "${%".into(), - block_end: "%}".into(), - variable_start: "${{".into(), - variable_end: "}}".into(), - comment_start: "${*".into(), - comment_end: "*}".into(), - })?; - - Ok(Self::new(secret_store, env)) - } - - pub fn render_str(&self, input: &str) -> anyhow::Result { - let context = self.build_context(input)?; - match self.env.render_str(input, context) { - Ok(rendered) => Ok(rendered), - - Err(e) => Err(anyhow::anyhow!("failed to render: `{}`. {}", input, e)), - } - } - - fn build_context(&self, input: &str) -> anyhow::Result { - let secrets: HashMap<_, _> = detect_secrets_from_str(input)? - .into_iter() - .map(|s| { - self.secret_store - .read(&s) - .map(|secret_value| (s, secret_value)) - }) - .collect::, _>>()?; - Ok(Context { secrets }) - } -} - -pub fn resolve_config_str(input: &str) -> anyhow::Result { - let renderer = ConfigRenderer::new_with_default_stores()?; - - let value = renderer.render_str(input)?; - - Ok(value) -} - -#[cfg(test)] -mod test { - use std::io::Write; - - use serde_yaml::Value; - - use crate::{ - secret::{self, FileSecretStore}, - renderer::Context, - }; - - use super::ConfigRenderer; - - // Calling all the test functions from a single test function - // so that we can set the default secret store once. - #[test] - fn test_renderer() { - let mut file = tempfile::NamedTempFile::new().expect("failed to create tmp file"); - file.write_all(b"foo=bar\napi_key=my_api_key\ninterval=10s") - .expect("file to write"); - secret::set_default_secret_store(::std::boxed::Box::new(FileSecretStore::from( - file.path(), - ))) - .expect("failed to set default secret store"); - - test_build_context(); - test_render_str(); - test_render_str_undefined_secret(); - test_invalid_syntax(); - test_undefined_context_variable(); - test_resolve_config(); - } - - fn test_build_context() { - let renderer = ConfigRenderer::new_with_default_stores().unwrap(); - let context: Context = renderer.build_context("hello ${{ secrets.foo }}").unwrap(); - - assert_eq!(context.secrets["foo"], "bar"); - assert_eq!(context.secrets.len(), 1); - } - - fn test_render_str() { - let renderer = ConfigRenderer::new_with_default_stores().unwrap(); - let output = renderer.render_str("hello ${{ secrets.foo }}").unwrap(); - assert_eq!(output, "hello bar"); - } - - fn test_render_str_undefined_secret() { - let renderer = ConfigRenderer::new_with_default_stores().unwrap(); - let output = renderer.render_str("hello ${{ secrets.undefined_var }}"); - assert!(output.is_err()); - } - - fn test_invalid_syntax() { - let renderer = ConfigRenderer::new_with_default_stores().unwrap(); - let output = renderer.render_str("hello ${{"); - assert!(output.is_err()); - } - - fn test_undefined_context_variable() { - let renderer = ConfigRenderer::new_with_default_stores().unwrap(); - let output = renderer.render_str("hello ${{ some_undefined.variable }}"); - assert!(output.is_err()); - } - - fn test_resolve_config() { - use super::*; - let value_str = r#" - meta: - name: test - version: 0.1.0 - topic: test - type: http-source - consumer: - partition: 0 - my_service: - api_key: ${{secrets.api_key}} - interval: ${{ secrets.interval }} - "#; - - let value_str = resolve_config_str(value_str).unwrap(); - let value: Value = serde_yaml::from_str(&value_str).expect("should be yaml"); - assert_eq!( - value["my_service"]["api_key"] - .as_str() - .expect("should be str"), - "my_api_key" - ); - assert_eq!( - value["my_service"]["interval"] - .as_str() - .expect("should be str"), - "10s" - ); - } -}