Merge branch 'master' into master

CHANGELOG.md

@@ -5,12 +5,15 @@
 ### Features
 
 - [#1564](https://github.com/influxdata/telegraf/issues/1564): Use RFC3339 timestamps in log output.
+- [#1997](https://github.com/influxdata/telegraf/issues/1997): Non-default HTTP timeouts for RabbitMQ plugin.
 
 ### Bugfixes
 
+- [#2049](https://github.com/influxdata/telegraf/pull/2049): Fix the Value data format not trimming null characters from input.
 - [#1949](https://github.com/influxdata/telegraf/issues/1949): Fix windows `net` plugin.
+- [#1775](https://github.com/influxdata/telegraf/issues/1775): Cache & expire metrics for delivery to prometheus
 
-## v1.1.1 [unreleased]
+## v1.1.1 [2016-11-14]
 
 ### Bugfixes
 
@@ -29,6 +32,9 @@ documentation for configuring journald. There is also a [`logfile` config option
 available in 1.1, which will allow users to easily configure telegraf to
 continue sending logs to /var/log/telegraf/telegraf.log.
 
+- The JSON parser can now parse JSON data where the root object is an array.
+The parsing configuration is applied to each element of the array.
+
 ### Features
 
 - [#1726](https://github.com/influxdata/telegraf/issues/1726): Processor & Aggregator plugin support.
@@ -91,6 +97,7 @@ continue sending logs to /var/log/telegraf/telegraf.log.
 - [#1771](https://github.com/influxdata/telegraf/issues/1771): Delete nil fields in the metric maker.
 - [#870](https://github.com/influxdata/telegraf/issues/870): Fix MySQL special characters in DSN parsing.
 - [#1742](https://github.com/influxdata/telegraf/issues/1742): Ping input odd timeout behavior.
+- [#1950](https://github.com/influxdata/telegraf/pull/1950): Switch to github.com/kballard/go-shellquote.
 
 ## v1.0.1 [2016-09-26]
 

Godeps

@@ -20,7 +20,6 @@ github.com/go-sql-driver/mysql 1fca743146605a172a266e1654e01e5cd5669bee
 github.com/gobwas/glob 49571a1557cd20e6a2410adc6421f85b66c730b5
 github.com/golang/protobuf 552c7b9542c194800fd493123b3798ef0a832032
 github.com/golang/snappy d9eb7a3d35ec988b8585d4a0068e462c27d28380
-github.com/gonuts/go-shellquote e842a11b24c6abfb3dd27af69a17f482e4b483c2
 github.com/gorilla/context 1ea25387ff6f684839d82767c1733ff4d4d15d0a
 github.com/gorilla/mux c9e326e2bdec29039a3761c07bece13133863e1e
 github.com/hailocab/go-hostpool e80d13ce29ede4452c43dea11e79b9bc8a15b478
@@ -32,6 +31,7 @@ github.com/influxdata/toml af4df43894b16e3fd2b788d01bd27ad0776ef2d0
 github.com/influxdata/wlog 7c63b0a71ef8300adc255344d275e10e5c3a71ec
 github.com/kardianos/osext 29ae4ffbc9a6fe9fb2bc5029050ce6996ea1d3bc
 github.com/kardianos/service 5e335590050d6d00f3aa270217d288dda1c94d0a
+github.com/kballard/go-shellquote d8ec1a69a250a17bb0e419c386eac1f3711dc142
 github.com/klauspost/crc32 19b0b332c9e4516a6370a0456e6182c3b5036720
 github.com/lib/pq e182dc4027e2ded4b19396d638610f2653295f36
 github.com/matttproud/golang_protobuf_extensions d0c3fe89de86839aecf2e0579c40ba3bb336a453

docs/DATA_FORMATS_INPUT.md

@@ -147,6 +147,62 @@ Your Telegraf metrics would get tagged with "my_tag_1"
 exec_mycollector,my_tag_1=foo a=5,b_c=6
 ```
 
+If the JSON data is an array, then each element of the array is parsed with the configured settings.
+Each resulting metric will be output with the same timestamp.
+
+For example, if the following configuration:
+
+```toml
+[[inputs.exec]]
+  ## Commands array
+  commands = ["/usr/bin/mycollector --foo=bar"]
+
+  ## measurement name suffix (for separating different commands)
+  name_suffix = "_mycollector"
+
+  ## Data format to consume.
+  ## Each data format has it's own unique set of configuration options, read
+  ## more about them here:
+  ## https://github.com/influxdata/telegraf/blob/master/docs/DATA_FORMATS_INPUT.md
+  data_format = "json"
+
+  ## List of tag names to extract from top-level of JSON server response
+  tag_keys = [
+    "my_tag_1",
+    "my_tag_2"
+  ]
+```
+
+with this JSON output from a command:
+
+```json
+[
+    {
+        "a": 5,
+        "b": {
+            "c": 6
+        },
+        "my_tag_1": "foo",
+        "my_tag_2": "baz"
+    },
+    {
+        "a": 7,
+        "b": {
+            "c": 8
+        },
+        "my_tag_1": "bar",
+        "my_tag_2": "baz"
+    }
+]
+```
+
+Your Telegraf metrics would get tagged with "my_tag_1" and "my_tag_2"
+
+```
+exec_mycollector,my_tag_1=foo,my_tag_2=baz a=5,b_c=6
+exec_mycollector,my_tag_1=bar,my_tag_2=baz a=7,b_c=8
+```
+
 # Value:
 
 The "value" data format translates single values into Telegraf metrics. This

docs/LICENSE_OF_DEPENDENCIES.md

@@ -12,11 +12,11 @@
 - github.com/gogo/protobuf [BSD LICENSE](https://github.com/gogo/protobuf/blob/master/LICENSE)
 - github.com/golang/protobuf [BSD LICENSE](https://github.com/golang/protobuf/blob/master/LICENSE)
 - github.com/golang/snappy [BSD LICENSE](https://github.com/golang/snappy/blob/master/LICENSE)
-- github.com/gonuts/go-shellquote (No License, but the project it was forked from https://github.com/kballard/go-shellquote is [MIT](https://github.com/kballard/go-shellquote/blob/master/LICENSE)).
 - github.com/hashicorp/go-msgpack [BSD LICENSE](https://github.com/hashicorp/go-msgpack/blob/master/LICENSE)
 - github.com/hashicorp/raft [MPL LICENSE](https://github.com/hashicorp/raft/blob/master/LICENSE)
 - github.com/hashicorp/raft-boltdb [MPL LICENSE](https://github.com/hashicorp/raft-boltdb/blob/master/LICENSE)
 - github.com/kardianos/service [ZLIB LICENSE](https://github.com/kardianos/service/blob/master/LICENSE) (License not named but matches word for word with ZLib)
+- github.com/kballard/go-shellquote [MIT LICENSE](https://github.com/kballard/go-shellquote/blob/master/LICENSE)
 - github.com/lib/pq [MIT LICENSE](https://github.com/lib/pq/blob/master/LICENSE.md)
 - github.com/matttproud/golang_protobuf_extensions [APACHE LICENSE](https://github.com/matttproud/golang_protobuf_extensions/blob/master/LICENSE)
 - github.com/naoina/go-stringutil [MIT LICENSE](https://github.com/naoina/go-stringutil/blob/master/LICENSE)

etc/telegraf_windows.conf

@@ -130,6 +130,7 @@
     Counters = [
       "Context Switches/sec",
       "System Calls/sec",
+      "Processor Queue Length",
     ]
     Instances = ["------"]
     Measurement = "win_system"

plugins/inputs/apache/README.md

@@ -2,6 +2,16 @@
 
 #### Plugin arguments:
 - **urls** []string: List of apache-status URLs to collect from. Default is "http://localhost/server-status?auto".
+- **username** string: Username for HTTP basic authentication
+- **password** string: Password for HTTP basic authentication
+- **timeout** duration: time that the HTTP connection will remain waiting for response. Defalt 4 seconds ("4s")
+
+##### Optional SSL Config
+
+- **ssl_ca** string: the full path for the SSL CA certicate
+- **ssl_cert** string: the full path for the SSL certificate
+- **ssl_key** string: the full path for the key file
+- **insecure_skip_verify** bool: if true HTTP client will skip all SSL verifications related to peer and host. Default to false
 
 #### Description
 

plugins/inputs/apache/apache.go

@@ -11,17 +11,42 @@ import (
 	"time"
 
 	"github.com/influxdata/telegraf"
+	"github.com/influxdata/telegraf/internal"
 	"github.com/influxdata/telegraf/plugins/inputs"
 )
 
 type Apache struct {
-	Urls []string
+	Urls            []string
+	Username        string
+	Password        string
+	ResponseTimeout internal.Duration
+	// Path to CA file
+	SSLCA string `toml:"ssl_ca"`
+	// Path to host cert file
+	SSLCert string `toml:"ssl_cert"`
+	// Path to cert key file
+	SSLKey string `toml:"ssl_key"`
+	// Use SSL but skip chain & host verification
+	InsecureSkipVerify bool
 }
 
 var sampleConfig = `
   ## An array of Apache status URI to gather stats.
   ## Default is "http://localhost/server-status?auto".
   urls = ["http://localhost/server-status?auto"]
+  ## user credentials for basic HTTP authentication
+  username = "myuser"
+  password = "mypassword"
+
+  ## Timeout to the complete conection and reponse time in seconds
+  response_timeout = "25s" ## default to 5 seconds
+
+  ## Optional SSL Config
+  # ssl_ca = "/etc/telegraf/ca.pem"
+  # ssl_cert = "/etc/telegraf/cert.pem"
+  # ssl_key = "/etc/telegraf/key.pem"
+  ## Use SSL but skip chain & host verification
+  # insecure_skip_verify = false
 `
 
 func (n *Apache) SampleConfig() string {
@@ -36,6 +61,9 @@ func (n *Apache) Gather(acc telegraf.Accumulator) error {
 	if len(n.Urls) == 0 {
 		n.Urls = []string{"http://localhost/server-status?auto"}
 	}
+	if n.ResponseTimeout.Duration < time.Second {
+		n.ResponseTimeout.Duration = time.Second * 5
+	}
 
 	var outerr error
 	var errch = make(chan error)
@@ -61,21 +89,46 @@ func (n *Apache) Gather(acc telegraf.Accumulator) error {
 	return outerr
 }
 
-var tr = &http.Transport{
-	ResponseHeaderTimeout: time.Duration(3 * time.Second),
-}
+func (n *Apache) gatherUrl(addr *url.URL, acc telegraf.Accumulator) error {
 
-var client = &http.Client{
-	Transport: tr,
-	Timeout:   time.Duration(4 * time.Second),
-}
+	var tr *http.Transport
 
-func (n *Apache) gatherUrl(addr *url.URL, acc telegraf.Accumulator) error {
-	resp, err := client.Get(addr.String())
+	if addr.Scheme == "https" {
+		tlsCfg, err := internal.GetTLSConfig(
+			n.SSLCert, n.SSLKey, n.SSLCA, n.InsecureSkipVerify)
+		if err != nil {
+			return err
+		}
+		tr = &http.Transport{
+			ResponseHeaderTimeout: time.Duration(3 * time.Second),
+			TLSClientConfig:       tlsCfg,
+		}
+	} else {
+		tr = &http.Transport{
+			ResponseHeaderTimeout: time.Duration(3 * time.Second),
+		}
+	}
+
+	client := &http.Client{
+		Transport: tr,
+		Timeout:   n.ResponseTimeout.Duration,
+	}
+
+	req, err := http.NewRequest("GET", addr.String(), nil)
 	if err != nil {
-		return fmt.Errorf("error making HTTP request to %s: %s", addr.String(), err)
+		return fmt.Errorf("error on new request to %s : %s\n", addr.String(), err)
+	}
+
+	if len(n.Username) != 0 && len(n.Password) != 0 {
+		req.SetBasicAuth(n.Username, n.Password)
+	}
+
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("error on request to %s : %s\n", addr.String(), err)
 	}
 	defer resp.Body.Close()
+
 	if resp.StatusCode != http.StatusOK {
 		return fmt.Errorf("%s returned HTTP status %s", addr.String(), resp.Status)
 	}

plugins/inputs/exec/exec.go

@@ -11,7 +11,7 @@ import (
 	"syscall"
 	"time"
 
-	"github.com/gonuts/go-shellquote"
+	"github.com/kballard/go-shellquote"
 
 	"github.com/influxdata/telegraf"
 	"github.com/influxdata/telegraf/internal"

plugins/inputs/httpjson/README.md

@@ -37,6 +37,8 @@ You can also specify which keys from server response should be considered tags:
   ]
 ```
 
+If the JSON response is an array of objects, then each object will be parsed with the same configuration.
+
 You can also specify additional request parameters for the service:
 
 ```
@@ -150,3 +152,53 @@ httpjson_mycollector1_b_e,server='http://my.service.com/_stats' value=5
 httpjson_mycollector2_load,server='http://service.net/json/stats' value=100
 httpjson_mycollector2_users,server='http://service.net/json/stats' value=1335
 ```
+
+# Example 3, Multiple Metrics in Response:
+
+The response JSON can be treated as an array of data points that are all parsed with the same configuration.
+
+```
+[[inputs.httpjson]]
+  name = "mycollector"
+  servers = [
+    "http://my.service.com/_stats"
+  ]
+  # HTTP method to use (case-sensitive)
+  method = "GET"
+  tag_keys = ["service"]
+```
+
+which responds with the following JSON:
+
+```json
+[
+    {
+        "service": "service01",
+        "a": 0.5,
+        "b": {
+            "c": "some text",
+            "d": 0.1,
+            "e": 5
+        }
+    },
+    {
+        "service": "service02",
+        "a": 0.6,
+        "b": {
+            "c": "some text",
+            "d": 0.2,
+            "e": 6
+        }
+    }
+]
+```
+
+The collected metrics will be:
+```
+httpjson_mycollector_a,service='service01',server='http://my.service.com/_stats' value=0.5
+httpjson_mycollector_b_d,service='service01',server='http://my.service.com/_stats' value=0.1
+httpjson_mycollector_b_e,service='service01',server='http://my.service.com/_stats' value=5
+httpjson_mycollector_a,service='service02',server='http://my.service.com/_stats' value=0.6
+httpjson_mycollector_b_d,service='service02',server='http://my.service.com/_stats' value=0.2
+httpjson_mycollector_b_e,service='service02',server='http://my.service.com/_stats' value=6
+```

plugins/inputs/httpjson/httpjson_test.go

@@ -511,3 +511,52 @@ func TestHttpJson200Tags(t *testing.T) {
 		}
 	}
 }
+
+const validJSONArrayTags = `
+[
+	{
+		"value": 15,
+		"role": "master",
+		"build": "123"
+	},
+	{
+		"value": 17,
+		"role": "slave",
+		"build": "456"
+	}
+]`
+
+// Test that array data is collected correctly
+func TestHttpJsonArray200Tags(t *testing.T) {
+	httpjson := genMockHttpJson(validJSONArrayTags, 200)
+
+	for _, service := range httpjson {
+		if service.Name == "other_webapp" {
+			var acc testutil.Accumulator
+			err := service.Gather(&acc)
+			// Set responsetime
+			for _, p := range acc.Metrics {
+				p.Fields["response_time"] = 1.0
+			}
+			require.NoError(t, err)
+			assert.Equal(t, 8, acc.NFields())
+			assert.Equal(t, uint64(4), acc.NMetrics())
+
+			for _, m := range acc.Metrics {
+				if m.Tags["role"] == "master" {
+					assert.Equal(t, "123", m.Tags["build"])
+					assert.Equal(t, float64(15), m.Fields["value"])
+					assert.Equal(t, float64(1), m.Fields["response_time"])
+					assert.Equal(t, "httpjson_"+service.Name, m.Measurement)
+				} else if m.Tags["role"] == "slave" {
+					assert.Equal(t, "456", m.Tags["build"])
+					assert.Equal(t, float64(17), m.Fields["value"])
+					assert.Equal(t, float64(1), m.Fields["response_time"])
+					assert.Equal(t, "httpjson_"+service.Name, m.Measurement)
+				} else {
+					assert.FailNow(t, "unknown metric")
+				}
+			}
+		}
+	}
+}