From a7cd7d26979dc7265ad97d50be80ff9c8383a5ec Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 7 Mar 2025 13:57:33 +0100 Subject: [PATCH 1/6] add option to use users secrets --- custom-recipes/api-connect/recipe.json | 7 +++++++ custom-recipes/api-connect/recipe.py | 4 +++- .../api-connect_dataset/connector.json | 7 +++++++ .../api-connect_dataset/connector.py | 4 +++- python-lib/dku_utils.py | 16 ++++++++++++++++ 5 files changed, 36 insertions(+), 2 deletions(-) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index 94febd9..13e0149 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -297,6 +297,13 @@ "type": "BOOLEAN", "defaultValue": false }, + { + "name": "should_use_user_secrets", + "label": "User's secrets", + "description": "Use secrets from user's profile", + "type": "BOOLEAN", + "defaultValue": false + }, { "name": "timeout", "label": "Timeout (s)", diff --git a/custom-recipes/api-connect/recipe.py b/custom-recipes/api-connect/recipe.py index 4790d13..c2aed34 100644 --- a/custom-recipes/api-connect/recipe.py +++ b/custom-recipes/api-connect/recipe.py @@ -3,7 +3,7 @@ from dataiku.customrecipe import get_input_names_for_role, get_recipe_config, get_output_names_for_role import pandas as pd from safe_logger import SafeLogger -from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials +from dku_utils import get_dku_key_values, get_endpoint_parameters, get_secure_credentials, get_user_secrets from rest_api_recipe_session import RestApiRecipeSession from dku_constants import DKUConstants @@ -45,6 +45,8 @@ def get_partitioning_keys(id_list, dku_flow_variables): raise ValueError("There is no parameter column selected.") parameter_renamings = get_dku_key_values(config.get("parameter_renamings", {})) custom_key_values = get_dku_key_values(config.get("custom_key_values", {})) +user_secrets = get_user_secrets(config) +custom_key_values.update(user_secrets) display_metadata = config.get("display_metadata", False) maximum_number_rows = config.get("maximum_number_rows", -1) input_parameters_dataset = dataiku.Dataset(input_A_names[0]) diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index a0fac70..87939c0 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -245,6 +245,13 @@ "type": "BOOLEAN", "defaultValue": false }, + { + "name": "should_use_user_secrets", + "label": " ", + "description": "Use secrets from user's profile", + "type": "BOOLEAN", + "defaultValue": false + }, { "name": "timeout", "label": "Timeout (s)", diff --git a/python-connectors/api-connect_dataset/connector.py b/python-connectors/api-connect_dataset/connector.py index 55b0f20..fc155de 100644 --- a/python-connectors/api-connect_dataset/connector.py +++ b/python-connectors/api-connect_dataset/connector.py @@ -5,7 +5,7 @@ from dku_utils import ( get_dku_key_values, get_endpoint_parameters, parse_keys_for_json, get_value_from_path, get_secure_credentials, - decode_csv_data, decode_bytes + decode_csv_data, decode_bytes, get_user_secrets ) from dku_constants import DKUConstants import json @@ -24,6 +24,8 @@ def __init__(self, config, plugin_config): secure_credentials = get_secure_credentials(config) credential = config.get("credential", {}) custom_key_values = get_dku_key_values(config.get("custom_key_values", {})) + user_secrets = get_user_secrets(config) + custom_key_values.update(user_secrets) self.client = RestAPIClient(credential, secure_credentials, endpoint_parameters, custom_key_values) extraction_key = endpoint_parameters.get("extraction_key", None) self.extraction_key = extraction_key or '' diff --git a/python-lib/dku_utils.py b/python-lib/dku_utils.py index f9750d8..79c3b22 100644 --- a/python-lib/dku_utils.py +++ b/python-lib/dku_utils.py @@ -167,3 +167,19 @@ def decode_bytes(content): if isinstance(content, bytes): content = content.decode() return content + + +def get_user_secrets(configuration): + should_use_user_secrets = configuration.get("should_use_user_secrets", False) + if should_use_user_secrets: + import dataiku + logger.info("Using user's secrets:") + client = dataiku.api_client() + auth_info = client.get_auth_info(with_secrets=True) + secrets = auth_info.get("secrets", []) + user_secrets = {} + for secret in secrets: + logger.info("\t-'{}'".format(secret.get("key"))) + user_secrets[secret.get("key")] = secret.get("value") + return user_secrets + return {} From beb95dabca2c4908e0fa68634b5bdfafd91e109c Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 7 Mar 2025 13:59:26 +0100 Subject: [PATCH 2/6] update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 55ab9e6..004a315 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,7 @@ ## [Version 1.2.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.4) - Feature release - 2025-02-18 - Fix xml decoding for content type application/rss+xml +- Add option to use secrets stored in the user's profile ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature and bugfix release - 2024-11-25 From dff798d196f6f13745b2a5fc0e01e58f2bb614e6 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Fri, 7 Mar 2025 13:59:42 +0100 Subject: [PATCH 3/6] beta.1 --- python-lib/dku_constants.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/python-lib/dku_constants.py b/python-lib/dku_constants.py index 2145371..dacd8ba 100644 --- a/python-lib/dku_constants.py +++ b/python-lib/dku_constants.py @@ -2,6 +2,6 @@ class DKUConstants(object): API_RESPONSE_KEY = "api_response" FORBIDDEN_KEYS = ["token", "password", "api_key_value", "secure_token"] FORM_DATA_BODY_FORMAT = "FORM_DATA" - PLUGIN_VERSION = "1.2.4" + PLUGIN_VERSION = "1.2.4-beta.1" RAW_BODY_FORMAT = "RAW" REPONSE_ERROR_KEY = "dku_error" From 45dd8146e6b166ee473500f887d81ba670e6da69 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 2 Jun 2025 15:16:18 +0200 Subject: [PATCH 4/6] suggested changes in UX --- custom-recipes/api-connect/recipe.json | 4 ++-- python-connectors/api-connect_dataset/connector.json | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index 13e0149..55d8ee6 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -299,8 +299,8 @@ }, { "name": "should_use_user_secrets", - "label": "User's secrets", - "description": "Use secrets from user's profile", + "label": "User 'Other credentials'", + "description": "If checked, secrets under Profile > My account > Other credentials, are available to use as {{variables}}", "type": "BOOLEAN", "defaultValue": false }, diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index 87939c0..bd6c6b1 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -248,7 +248,7 @@ { "name": "should_use_user_secrets", "label": " ", - "description": "Use secrets from user's profile", + "description": "Use profile's 'other credentials'", "type": "BOOLEAN", "defaultValue": false }, From cad760e081d787294f6d237deb5f35a93446d1a9 Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 2 Jun 2025 15:27:28 +0200 Subject: [PATCH 5/6] Move "other creds" boolean to auth section --- custom-recipes/api-connect/recipe.json | 14 +++++++------- .../api-connect_dataset/connector.json | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/custom-recipes/api-connect/recipe.json b/custom-recipes/api-connect/recipe.json index 55d8ee6..5842cd0 100644 --- a/custom-recipes/api-connect/recipe.json +++ b/custom-recipes/api-connect/recipe.json @@ -65,6 +65,13 @@ "parameterSetId": "secure-basic", "visibilityCondition": "model.auth_type == 'secure_basic'" }, + { + "name": "should_use_user_secrets", + "label": "User 'Other credentials'", + "description": "If checked, secrets under Profile > My account > Other credentials, are available to use as {{variables}}", + "type": "BOOLEAN", + "defaultValue": false + }, { "type": "SEPARATOR", "label": "API call parameters" @@ -297,13 +304,6 @@ "type": "BOOLEAN", "defaultValue": false }, - { - "name": "should_use_user_secrets", - "label": "User 'Other credentials'", - "description": "If checked, secrets under Profile > My account > Other credentials, are available to use as {{variables}}", - "type": "BOOLEAN", - "defaultValue": false - }, { "name": "timeout", "label": "Timeout (s)", diff --git a/python-connectors/api-connect_dataset/connector.json b/python-connectors/api-connect_dataset/connector.json index bd6c6b1..754c507 100644 --- a/python-connectors/api-connect_dataset/connector.json +++ b/python-connectors/api-connect_dataset/connector.json @@ -44,6 +44,13 @@ "parameterSetId": "secure-basic", "visibilityCondition": "model.auth_type == 'secure_basic'" }, + { + "name": "should_use_user_secrets", + "label": " ", + "description": "Use profile's 'other credentials'", + "type": "BOOLEAN", + "defaultValue": false + }, { "type": "SEPARATOR", "label": "API call parameters" @@ -245,13 +252,6 @@ "type": "BOOLEAN", "defaultValue": false }, - { - "name": "should_use_user_secrets", - "label": " ", - "description": "Use profile's 'other credentials'", - "type": "BOOLEAN", - "defaultValue": false - }, { "name": "timeout", "label": "Timeout (s)", From bd5b3bc59534ead7643ec98705f7bbebf39f540a Mon Sep 17 00:00:00 2001 From: Alex Bourret Date: Mon, 2 Jun 2025 17:31:40 +0200 Subject: [PATCH 6/6] update changelog --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 004a315..5ba9c64 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## [Version 1.2.4](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.4) - Feature release - 2025-02-18 - Fix xml decoding for content type application/rss+xml -- Add option to use secrets stored in the user's profile +- Let use the *Other credentials* stored in the user's profile ## [Version 1.2.3](https://github.com/dataiku/dss-plugin-api-connect/releases/tag/v1.2.3) - Feature and bugfix release - 2024-11-25