Use trait to build context

crates/fluvio-connector-common/src/lib.rs

@@ -4,7 +4,7 @@ pub mod monitoring;
 pub mod consumer;
 pub mod config;
 
-pub use fluvio_connector_package::resolve_config_str;
+pub use fluvio_connector_package::render_config_str;
 pub use fluvio_connector_package::secret;
 
 #[cfg(feature = "derive")]

crates/fluvio-connector-derive/src/generator.rs

@@ -117,7 +117,7 @@ fn init_and_parse_config(config_type_path: &Path) -> TokenStream {
         ::fluvio_connector_common::tracing::debug!(%config_str);
 
         /// Resolve any secrets/env in the config
-        let config_str_resolved =::fluvio_connector_common::resolve_config_str(&config_str)?;
+        let config_str_resolved =::fluvio_connector_common::render_config_str(&config_str)?;
 
         let config_value = ::fluvio_connector_common::config::value_from_reader(config_str_resolved.as_bytes())?;
 

crates/fluvio-connector-package/Cargo.toml

@@ -14,13 +14,13 @@ default = ["toml"]
 anyhow = { workspace = true }
 bytesize = { workspace = true}
 humantime-serde = "1.1.1"
-# Move to crates.io release, once https://github.com/mitsuhiko/minijinja/commit/78b9b87eaa68ac4c1b1c814d420a8e48e054fefc is released
 minijinja = { version = "0.32", default-features = false, features = ["custom_syntax"] }
 openapiv3 = { git = "https://github.com/galibey/openapiv3", rev = "bdd22f046d2bc19ede257504645d31f835545222", default-features = false }
 once_cell = { workspace = true }
 regex = { workspace = true }
 serde = { workspace = true, default-features = false, features = ["derive"] }
 serde_yaml = { workspace = true }
+serde_json = { workspace = true }
 toml = { workspace = true , default-features = false, optional = true, features = ["preserve_order"] }
 tracing = { workspace = true }
 

crates/fluvio-connector-package/src/lib.rs

@@ -1,6 +1,6 @@
 pub mod metadata;
 pub mod config;
 pub mod secret;
-mod renderer;
+mod render;
 
-pub use renderer::resolve_config_str;
+pub use render::render_config_str;

crates/fluvio-connector-package/src/render/context.rs

@@ -0,0 +1,67 @@
+use std::collections::HashMap;
+
+use serde::Serialize;
+
+use crate::secret::{SecretStore, detect_secrets_from_str};
+
+/// Context for the template engine
+/// This is the data that is available to the template engine
+/// when it is rendering the template.
+#[derive(Serialize, Default)]
+pub(crate) struct Context(pub(crate) HashMap<&'static str, serde_json::Value>);
+
+/// ContextStore is a trait that allows for adding additional
+/// context to the template engine.
+pub(crate) trait ContextStore {
+    fn extract_context_values(&self, input: &str) -> anyhow::Result<serde_json::Value>;
+
+    fn context_name(&self) -> &'static str;
+}
+impl dyn ContextStore {
+    pub(crate) fn add_to_context(&self, context: &mut Context, input: &str) -> anyhow::Result<()> {
+        let value = self.extract_context_values(input)?;
+        context.0.insert(self.context_name(), value);
+        Ok(())
+    }
+}
+impl ContextStore for &dyn SecretStore {
+    fn extract_context_values(&self, input: &str) -> anyhow::Result<serde_json::Value> {
+        let secrets: HashMap<_, _> = detect_secrets_from_str(input)?
+            .into_iter()
+            .map(|s: String| self.read(&s).map(|secret_value| (s, secret_value)))
+            .collect::<Result<HashMap<_, _>, _>>()?;
+        Ok(serde_json::to_value(secrets)?)
+    }
+
+    fn context_name(&self) -> &'static str {
+        "secrets"
+    }
+}
+
+#[cfg(test)]
+mod test {
+    use super::ContextStore;
+
+    pub struct TestStore;
+
+    impl ContextStore for TestStore {
+        fn extract_context_values(&self, _input: &str) -> anyhow::Result<serde_json::Value> {
+            Ok(serde_json::json!({"test": "value"}))
+        }
+
+        fn context_name(&self) -> &'static str {
+            "test"
+        }
+    }
+
+    #[test]
+    fn test_context_store() {
+        let store = &TestStore as &dyn ContextStore;
+        let mut context = super::Context::default();
+        store.add_to_context(&mut context, "").unwrap();
+        assert_eq!(
+            context.0.get("test").unwrap(),
+            &serde_json::json!({"test": "value"})
+        );
+    }
+}

crates/fluvio-connector-package/src/render/mod.rs

@@ -0,0 +1,203 @@
+use minijinja::{Environment};
+
+use crate::{
+    secret::{self},
+};
+
+use self::context::{Context, ContextStore};
+
+mod context;
+
+/// Config renderer. This is the main entry point for rendering a config.
+/// It is responsible for resolving secrets and rendering the config.
+pub(crate) struct ConfigRenderer {
+    inner_renderer: Environment<'static>,
+    context_stores: Vec<Box<dyn ContextStore>>,
+}
+
+impl ConfigRenderer {
+    fn new(
+        inner_renderer: Environment<'static>,
+        context_stores: Vec<Box<dyn ContextStore>>,
+    ) -> Self {
+        Self {
+            inner_renderer,
+            context_stores,
+        }
+    }
+
+    fn new_with_context_stores(context_stores: Vec<Box<dyn ContextStore>>) -> anyhow::Result<Self> {
+        let mut inner_renderer = Environment::new();
+        inner_renderer.set_syntax(minijinja::Syntax {
+            block_start: "${%".into(),
+            block_end: "%}".into(),
+            variable_start: "${{".into(),
+            variable_end: "}}".into(),
+            comment_start: "${*".into(),
+            comment_end: "*}".into(),
+        })?;
+
+        Ok(Self::new(inner_renderer, context_stores))
+    }
+
+    fn new_with_default_stores() -> anyhow::Result<Self> {
+        let context_stores = Self::default_stores()?;
+
+        Self::new_with_context_stores(context_stores)
+    }
+
+    fn render_str(&self, input: &str) -> anyhow::Result<String> {
+        let context = self.build_context(input)?;
+        match self.inner_renderer.render_str(input, context) {
+            Ok(rendered) => Ok(rendered),
+            Err(e) => Err(anyhow::anyhow!("failed to render: `{}`. {}", input, e)),
+        }
+    }
+
+    fn default_stores() -> anyhow::Result<Vec<Box<dyn ContextStore>>> {
+        Ok(vec![Box::new(secret::default_secret_store()?)])
+    }
+
+    fn build_context(&self, input: &str) -> anyhow::Result<Context> {
+        let mut context = Context::default();
+        for store in self.context_stores.iter() {
+            store.add_to_context(&mut context, input)?;
+        }
+        Ok(context)
+    }
+}
+
+/// Render a config from a string.
+/// This is the main entry point for rendering a config.
+/// It is responsible for resolving secrets and rendering the config.
+pub fn render_config_str(input: &str) -> anyhow::Result<String> {
+    let renderer = ConfigRenderer::new_with_default_stores()?;
+
+    let value = renderer.render_str(input)?;
+
+    Ok(value)
+}
+
+#[cfg(test)]
+mod test {
+    use std::{io::Write, collections::HashMap};
+
+    use anyhow::Context;
+    use serde_yaml::Value;
+
+    use crate::{
+        secret::{FileSecretStore, detect_secrets_from_str},
+    };
+
+    use super::{ConfigRenderer, context::ContextStore};
+
+    // mock secret store
+    pub struct SecretTestStore;
+
+    impl ContextStore for SecretTestStore {
+        fn extract_context_values(&self, input: &str) -> anyhow::Result<serde_json::Value> {
+            let values =
+                serde_json::json!({"foo": "bar", "api_key": "my_api_key", "interval": "10s"});
+            let secrets: HashMap<_, _> = detect_secrets_from_str(input)?
+                .into_iter()
+                .map(|s: String| {
+                    values
+                        .get(&s)
+                        .map(|secret_value| (s, secret_value))
+                        .context("not found in test values")
+                })
+                .collect::<Result<HashMap<_, _>, _>>()?;
+            Ok(serde_json::to_value(secrets)?)
+        }
+
+        fn context_name(&self) -> &'static str {
+            "secrets"
+        }
+    }
+
+    #[test]
+    fn test_build_context() {
+        let store = Box::new(SecretTestStore);
+
+        let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap();
+        let context: super::context::Context =
+            renderer.build_context("hello ${{ secrets.foo }}").unwrap();
+
+        assert_eq!(context.0["secrets"]["foo"], "bar");
+        assert_eq!(context.0["secrets"].as_object().unwrap().len(), 1);
+    }
+
+    #[test]
+    fn test_render_str() {
+        let store = Box::new(SecretTestStore);
+        let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap();
+        let output = renderer.render_str("hello ${{ secrets.foo }}").unwrap();
+        assert_eq!(output, "hello bar");
+    }
+
+    #[test]
+    fn test_render_str_undefined_secret() {
+        let store = Box::new(SecretTestStore);
+
+        let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap();
+        let output = renderer.render_str("hello ${{ secrets.undefined_var }}");
+        assert!(output.is_err());
+    }
+
+    #[test]
+    fn test_invalid_syntax() {
+        let store = Box::new(SecretTestStore);
+
+        let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap();
+        let output = renderer.render_str("hello ${{");
+        assert!(output.is_err());
+    }
+
+    #[test]
+    fn test_undefined_context_variable() {
+        let store = Box::new(SecretTestStore);
+
+        let renderer = ConfigRenderer::new_with_context_stores(vec![store]).unwrap();
+        let output = renderer.render_str("hello ${{ some_undefined.variable }}");
+        assert!(output.is_err());
+    }
+
+    #[test]
+    fn test_resolve_config() {
+        use super::*;
+        let mut file = tempfile::NamedTempFile::new().expect("failed to create tmp file");
+        file.write_all(b"foo=bar\napi_key=my_api_key\ninterval=10s")
+            .expect("file to write");
+        let _ = secret::set_default_secret_store(::std::boxed::Box::new(FileSecretStore::from(
+            file.path(),
+        )));
+
+        let value_str = r#"
+            meta:
+                name: test
+                version: 0.1.0
+                topic: test
+                type: http-source
+                consumer:
+                    partition: 0
+            my_service:
+                api_key: ${{secrets.api_key}}
+                interval: ${{ secrets.interval }}
+            "#;
+
+        let value_str = render_config_str(value_str).unwrap();
+        let value: Value = serde_yaml::from_str(&value_str).expect("should be yaml");
+        assert_eq!(
+            value["my_service"]["api_key"]
+                .as_str()
+                .expect("should be str"),
+            "my_api_key"
+        );
+        assert_eq!(
+            value["my_service"]["interval"]
+                .as_str()
+                .expect("should be str"),
+            "10s"
+        );
+    }
+}