Merge remote-tracking branch 'apache/branch-2.10' into retention-2.10

docker/pulsar/scripts/apply-config-from-env-with-prefix.py

@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env bash
 #
 # Licensed to the Apache Software Foundation (ASF) under one
 # or more contributor license agreements.  See the NOTICE file
@@ -18,85 +18,22 @@
 # under the License.
 #
 
-##
-## Edit a properties config file and replace values based on
-## the ENV variables
-## export prefix_my-key=new-value
-## ./apply-config-from-env-with-prefix prefix_ file.conf
-##
-
-import os, sys
-
-if len(sys.argv) < 3:
-    print('Usage: %s' % (sys.argv[0]))
-    sys.exit(1)
-
-# Always apply env config to env scripts as well
-prefix = sys.argv[1]
-conf_files = sys.argv[2:]
-
-PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1')
-
-for conf_filename in conf_files:
-    lines = []  # List of config file lines
-    keys = {} # Map a key to its line number in the file
-
-    # Load conf file
-    for line in open(conf_filename):
-        lines.append(line)
-        line = line.strip()
-        if not line or line.startswith('#'):
-            continue
-
-        try:
-            k,v = line.split('=', 1)
-            keys[k] = len(lines) - 1
-        except:
-            if PF_ENV_DEBUG:
-                print("[%s] skip Processing %s" % (conf_filename, line))
-
-    # Update values from Env
-    for k in sorted(os.environ.keys()):
-        v = os.environ[k].strip()
-
-        # Hide the value in logs if is password.
-        if "password" in k.lower():
-            displayValue = "********"
-        else:
-            displayValue = v
-
-        if k.startswith(prefix):
-            k = k[len(prefix):]
-        if k in keys:
-            print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue))
-            idx = keys[k]
-            lines[idx] = '%s=%s\n' % (k, v)
-
-
-    # Add new keys from Env
-    for k in sorted(os.environ.keys()):
-        v = os.environ[k]
-        if not k.startswith(prefix):
-            continue
-
-        # Hide the value in logs if is password.
-        if "password" in k.lower():
-            displayValue = "********"
-        else:
-            displayValue = v
-
-        k = k[len(prefix):]
-        if k not in keys:
-            print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue))
-            lines.append('%s=%s\n' % (k, v))
-        else:
-            print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue))
-            lines[keys[k]] = '%s=%s\n' % (k, v)
-
+############################################################
+# Edit a properties config file and replace values based on
+# the ENV variables
+# export prefix_my-key=new-value
+# ./apply-config-from-env-with-prefix prefix_ file.conf
+#
+# Environment variables that are prefixed with the command
+# line prefix will be used to updated file properties if
+# they exist and create new ones if they don't.
+#
+# Environment variables not prefixed will be used only to
+# update if they exist and ignored if they don't.
+############################################################
 
-    # Store back the updated config in the same file
-    f = open(conf_filename, 'w')
-    for line in lines:
-        f.write(line)
-    f.close()
+# DEPRECATED: Use "apply-config-from-env.py --prefix MY_PREFIX_ conf_file" instead
 
+# this is not a python script, but a bash script. Call apply-config-from-env.py with the prefix argument
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" >/dev/null 2>&1 && pwd)"
+"${SCRIPT_DIR}/apply-config-from-env.py" --prefix "$1" "${@:2}"

docker/pulsar/scripts/apply-config-from-env.py

@@ -25,18 +25,29 @@
 ## ./apply-config-from-env file.conf
 ##
 
-import os, sys
-
-if len(sys.argv) < 2:
-    print('Usage: %s' % (sys.argv[0]))
+import os, sys, argparse
+
+parser = argparse.ArgumentParser(description='Pulsar configuration file customizer based on environment variables')
+parser.add_argument('--prefix', default='PULSAR_PREFIX_', help='Prefix for environment variables, default is PULSAR_PREFIX_')
+parser.add_argument('conf_files', nargs='*', help='Configuration files')
+args = parser.parse_args()
+if not args.conf_files:
+    parser.print_help()
     sys.exit(1)
 
-# Always apply env config to env scripts as well
-conf_files = sys.argv[1:]
+env_prefix = args.prefix
+conf_files = args.conf_files
 
-PF_ENV_PREFIX = 'PULSAR_PREFIX_'
 PF_ENV_DEBUG = (os.environ.get('PF_ENV_DEBUG','0') == '1')
 
+# List of keys where the value should not be displayed in logs
+sensitive_keys = ["brokerClientAuthenticationParameters", "bookkeeperClientAuthenticationParameters", "tokenSecretKey"]
+
+def sanitize_display_value(k, v):
+    if "password" in k.lower() or k in sensitive_keys or (k == "tokenSecretKey" and v.startswith("data:")):
+        return "********"
+    return v
+
 for conf_filename in conf_files:
     lines = []  # List of config file lines
     keys = {} # Map a key to its line number in the file
@@ -47,7 +58,6 @@
         line = line.strip()
         if not line:
             continue
-
         try:
             k,v = line.split('=', 1)
             if k.startswith('#'):
@@ -61,48 +71,35 @@
     for k in sorted(os.environ.keys()):
         v = os.environ[k].strip()
 
-        # Hide the value in logs if is password.
-        if "password" in k.lower():
-            displayValue = "********"
-        else:
-            displayValue = v
-
-        if k.startswith(PF_ENV_PREFIX):
-            k = k[len(PF_ENV_PREFIX):]
         if k in keys:
+            displayValue = sanitize_display_value(k, v)
             print('[%s] Applying config %s = %s' % (conf_filename, k, displayValue))
             idx = keys[k]
             lines[idx] = '%s=%s\n' % (k, v)
 
-
     # Ensure we have a new-line at the end of the file, to avoid issue
     # when appending more lines to the config
     lines.append('\n')
-    
-    # Add new keys from Env    
+
+    # Add new keys from Env
     for k in sorted(os.environ.keys()):
-        v = os.environ[k]
-        if not k.startswith(PF_ENV_PREFIX):
+        if not k.startswith(env_prefix):
             continue
 
-        # Hide the value in logs if is password.
-        if "password" in k.lower():
-            displayValue = "********"
-        else:
-            displayValue = v
+        v = os.environ[k].strip()
+        k = k[len(env_prefix):]
+
+        displayValue = sanitize_display_value(k, v)
 
-        k = k[len(PF_ENV_PREFIX):]
         if k not in keys:
             print('[%s] Adding config %s = %s' % (conf_filename, k, displayValue))
             lines.append('%s=%s\n' % (k, v))
         else:
             print('[%s] Updating config %s = %s' % (conf_filename, k, displayValue))
             lines[keys[k]] = '%s=%s\n' % (k, v)
 
-
     # Store back the updated config in the same file
     f = open(conf_filename, 'w')
     for line in lines:
         f.write(line)
-    f.close()
-
+    f.close()