config: parse environment variables in the config file

closes #663

CHANGELOG.md

@@ -1,6 +1,7 @@
 ## v0.12.0 [unreleased]
 
 ### Features
+- [#951](https://github.com/influxdata/telegraf/pull/951): Parse environment variables in the config file.
 - [#948](https://github.com/influxdata/telegraf/pull/948): Cleanup config file and make default package version include all plugins (but commented).
 - [#927](https://github.com/influxdata/telegraf/pull/927): Adds parsing of tags to the statsd input when using DataDog's dogstatsd extension
 - [#863](https://github.com/influxdata/telegraf/pull/863): AMQP output: allow external auth. Thanks @ekini!

docs/CONFIGURATION.md

@@ -9,6 +9,12 @@ To generate a file with specific inputs and outputs, you can use the
 -input-filter and -output-filter flags:
 `telegraf -sample-config -input-filter cpu:mem:net:swap -output-filter influxdb:kafka`
 
+## Environment Variables
+
+Environment variables can be used anywhere in the config file, simply prepend
+them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
+
 ## `[global_tags]` Configuration
 
 Global tags can be specific in the `[global_tags]` section of the config file in

etc/telegraf.conf

@@ -8,12 +8,18 @@
 #
 # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
 # file would generate.
+#
+# Environment variables can be used anywhere in this config file, simply prepend
+# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
 
 
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # dc = "us-east-1" # will tag all metrics with dc=us-east-1
   # rack = "1a"
+  ## Environment variables can be used as tags and throughout the config file
+  # user = "$USER"
 
 
 # Configuration for telegraf agent
@@ -1114,27 +1120,6 @@
 #                            SERVICE INPUT PLUGINS                            #
 ###############################################################################
 
-# # Generic UDP listener
-# [[inputs.udp_listener]]
-#   ## Address and port to host UDP listener on
-#   service_address = ":8092"
-# 
-#   ## Number of UDP messages allowed to queue up. Once filled, the
-#   ## UDP listener will start dropping packets.
-#   allowed_pending_messages = 10000
-# 
-#   ## UDP packet size for the server to listen for. This will depend
-#   ## on the size of the packets that the client is sending, which is
-#   ## usually 1500 bytes, but can be as large as 65,535 bytes.
-#   udp_packet_size = 1500
-# 
-#   ## Data format to consume.
-#   ## Each data format has it's own unique set of configuration options, read
-#   ## more about them here:
-#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
-#   data_format = "influx"
-
-
 # # A Github Webhook Event collector
 # [[inputs.github_webhooks]]
 #   ## Address and port to host Webhook listener on
@@ -1277,3 +1262,24 @@
 #   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
 #   data_format = "influx"
 
+
+# # Generic UDP listener
+# [[inputs.udp_listener]]
+#   ## Address and port to host UDP listener on
+#   service_address = ":8092"
+# 
+#   ## Number of UDP messages allowed to queue up. Once filled, the
+#   ## UDP listener will start dropping packets.
+#   allowed_pending_messages = 10000
+# 
+#   ## UDP packet size for the server to listen for. This will depend
+#   ## on the size of the packets that the client is sending, which is
+#   ## usually 1500 bytes, but can be as large as 65,535 bytes.
+#   udp_packet_size = 1500
+# 
+#   ## Data format to consume.
+#   ## Each data format has it's own unique set of configuration options, read
+#   ## more about them here:
+#   ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+#   data_format = "influx"
+

internal/config/config.go

@@ -1,11 +1,14 @@
 package config
 
 import (
+	"bytes"
 	"errors"
 	"fmt"
 	"io/ioutil"
 	"log"
+	"os"
 	"path/filepath"
+	"regexp"
 	"sort"
 	"strings"
 	"time"
@@ -19,6 +22,7 @@ import (
 	"github.com/influxdata/telegraf/plugins/serializers"
 
 	"github.com/influxdata/config"
+	"github.com/influxdata/toml"
 	"github.com/influxdata/toml/ast"
 )
 
@@ -29,6 +33,9 @@ var (
 
 	// Default output plugins
 	outputDefaults = []string{"influxdb"}
+
+	// envVarRe is a regex to find environment variables in the config file
+	envVarRe = regexp.MustCompile(`\$\w+`)
 )
 
 // Config specifies the URL/user/password for the database that telegraf
@@ -153,12 +160,18 @@ var header = `# Telegraf Configuration
 #
 # Use 'telegraf -config telegraf.conf -test' to see what metrics a config
 # file would generate.
+#
+# Environment variables can be used anywhere in this config file, simply prepend
+# them with $. For strings the variable must be within quotes (ie, "$STR_VAR"),
+# for numbers and booleans they should be plain (ie, $INT_VAR, $BOOL_VAR)
 
 
 # Global tags can be specified here in key="value" format.
 [global_tags]
   # dc = "us-east-1" # will tag all metrics with dc=us-east-1
   # rack = "1a"
+  ## Environment variables can be used as tags and throughout the config file
+  # user = "$USER"
 
 
 # Configuration for telegraf agent
@@ -264,15 +277,20 @@ func printFilteredInputs(inputFilters []string, commented bool) {
 	}
 	sort.Strings(pnames)
 
-	// Print Inputs
+	// cache service inputs to print them at the end
 	servInputs := make(map[string]telegraf.ServiceInput)
+	// for alphabetical looping:
+	servInputNames := []string{}
+
+	// Print Inputs
 	for _, pname := range pnames {
 		creator := inputs.Inputs[pname]
 		input := creator()
 
 		switch p := input.(type) {
 		case telegraf.ServiceInput:
 			servInputs[pname] = p
+			servInputNames = append(servInputNames, pname)
 			continue
 		}
 
@@ -283,9 +301,10 @@ func printFilteredInputs(inputFilters []string, commented bool) {
 	if len(servInputs) == 0 {
 		return
 	}
+	sort.Strings(servInputNames)
 	fmt.Printf(serviceInputHeader)
-	for name, input := range servInputs {
-		printConfig(name, input, "inputs", commented)
+	for _, name := range servInputNames {
+		printConfig(name, servInputs[name], "inputs", commented)
 	}
 }
 
@@ -387,7 +406,7 @@ func (c *Config) LoadDirectory(path string) error {
 
 // LoadConfig loads the given config file and applies it to c
 func (c *Config) LoadConfig(path string) error {
-	tbl, err := config.ParseFile(path)
+	tbl, err := parseFile(path)
 	if err != nil {
 		return fmt.Errorf("Error parsing %s, %s", path, err)
 	}
@@ -456,6 +475,26 @@ func (c *Config) LoadConfig(path string) error {
 	return nil
 }
 
+// parseFile loads a TOML configuration from a provided path and
+// returns the AST produced from the TOML parser. When loading the file, it
+// will find environment variables and replace them.
+func parseFile(fpath string) (*ast.Table, error) {
+	contents, err := ioutil.ReadFile(fpath)
+	if err != nil {
+		return nil, err
+	}
+
+	env_vars := envVarRe.FindAll(contents, -1)
+	for _, env_var := range env_vars {
+		env_val := os.Getenv(strings.TrimPrefix(string(env_var), "$"))
+		if env_val != "" {
+			contents = bytes.Replace(contents, env_var, []byte(env_val), 1)
+		}
+	}
+
+	return toml.Parse(contents)
+}
+
 func (c *Config) addOutput(name string, table *ast.Table) error {
 	if len(c.OutputFilters) > 0 && !sliceContains(name, c.OutputFilters) {
 		return nil

internal/config/config_test.go

@@ -1,6 +1,7 @@
 package config
 
 import (
+	"os"
 	"testing"
 	"time"
 
@@ -10,9 +11,52 @@ import (
 	"github.com/influxdata/telegraf/plugins/inputs/memcached"
 	"github.com/influxdata/telegraf/plugins/inputs/procstat"
 	"github.com/influxdata/telegraf/plugins/parsers"
+
 	"github.com/stretchr/testify/assert"
 )
 
+func TestConfig_LoadSingleInputWithEnvVars(t *testing.T) {
+	c := NewConfig()
+	err := os.Setenv("MY_TEST_SERVER", "192.168.1.1")
+	assert.NoError(t, err)
+	err = os.Setenv("TEST_INTERVAL", "10s")
+	assert.NoError(t, err)
+	c.LoadConfig("./testdata/single_plugin_env_vars.toml")
+
+	memcached := inputs.Inputs["memcached"]().(*memcached.Memcached)
+	memcached.Servers = []string{"192.168.1.1"}
+
+	mConfig := &internal_models.InputConfig{
+		Name: "memcached",
+		Filter: internal_models.Filter{
+			NameDrop:  []string{"metricname2"},
+			NamePass:  []string{"metricname1"},
+			FieldDrop: []string{"other", "stuff"},
+			FieldPass: []string{"some", "strings"},
+			TagDrop: []internal_models.TagFilter{
+				internal_models.TagFilter{
+					Name:   "badtag",
+					Filter: []string{"othertag"},
+				},
+			},
+			TagPass: []internal_models.TagFilter{
+				internal_models.TagFilter{
+					Name:   "goodtag",
+					Filter: []string{"mytag"},
+				},
+			},
+			IsActive: true,
+		},
+		Interval: 10 * time.Second,
+	}
+	mConfig.Tags = make(map[string]string)
+
+	assert.Equal(t, memcached, c.Inputs[0].Input,
+		"Testdata did not produce a correct memcached struct.")
+	assert.Equal(t, mConfig, c.Inputs[0].Config,
+		"Testdata did not produce correct memcached metadata.")
+}
+
 func TestConfig_LoadSingleInput(t *testing.T) {
 	c := NewConfig()
 	c.LoadConfig("./testdata/single_plugin.toml")

internal/config/testdata/single_plugin_env_vars.toml

@@ -0,0 +1,11 @@
+[[inputs.memcached]]
+  servers = ["$MY_TEST_SERVER"]
+  namepass = ["metricname1"]
+  namedrop = ["metricname2"]
+  fieldpass = ["some", "strings"]
+  fielddrop = ["other", "stuff"]
+  interval = "$TEST_INTERVAL"
+  [inputs.memcached.tagpass]
+    goodtag = ["mytag"]
+  [inputs.memcached.tagdrop]
+    badtag = ["othertag"]