Add Stackdriver Metrics Exporter

Add Opencensus Stackdriver Exporter functionality for Agones metrics.
New docs on how to set Stackdriver Dashboard and configure permissions.
Add helm config variable as well as change reporting period to 1 minute
 if StackdriveExporter is enabled.

Gopkg.toml

@@ -95,3 +95,7 @@
 [[constraint]]
   name = "github.com/evanphx/json-patch"
   version = "4.1.0"
+
+[[constraint]]
+  name = "contrib.go.opencensus.io/exporter/stackdriver"
+  version = "v0.8.0"

cmd/controller/main.go

@@ -49,18 +49,20 @@ import (
 )
 
 const (
-	enableMetricsFlag     = "metrics"
-	sidecarImageFlag      = "sidecar-image"
-	sidecarCPURequestFlag = "sidecar-cpu-request"
-	sidecarCPULimitFlag   = "sidecar-cpu-limit"
-	pullSidecarFlag       = "always-pull-sidecar"
-	minPortFlag           = "min-port"
-	maxPortFlag           = "max-port"
-	certFileFlag          = "cert-file"
-	keyFileFlag           = "key-file"
-	kubeconfigFlag        = "kubeconfig"
-	workers               = 2
-	defaultResync         = 30 * time.Second
+	enableStackdriverMetricsFlag = "stackdriver-exporter"
+	enablePrometheusMetricsFlag  = "prometheus-exporter"
+	projectIDFlag                = "gcp-project-id"
+	sidecarImageFlag             = "sidecar-image"
+	sidecarCPURequestFlag        = "sidecar-cpu-request"
+	sidecarCPULimitFlag          = "sidecar-cpu-limit"
+	pullSidecarFlag              = "always-pull-sidecar"
+	minPortFlag                  = "min-port"
+	maxPortFlag                  = "max-port"
+	certFileFlag                 = "cert-file"
+	keyFileFlag                  = "key-file"
+	kubeconfigFlag               = "kubeconfig"
+	workers                      = 2
+	defaultResync                = 30 * time.Second
 )
 
 var (
@@ -106,19 +108,39 @@ func main() {
 	var rs []runner
 	var health healthcheck.Handler
 
-	if ctlConf.Metrics {
+	// Stackdriver metrics
+	if ctlConf.Stackdriver {
+		sd, err := metrics.RegisterStackdriverExporter(ctlConf.GCPProjectID)
+		if err != nil {
+			logger.WithError(err).Fatal("Could not register stackdriver exporter")
+		}
+		// It is imperative to invoke flush before your main function exits
+		defer sd.Flush()
+	}
+
+	// Prometheus metrics
+	if ctlConf.PrometheusMetrics {
 		registry := prom.NewRegistry()
 		metricHandler, err := metrics.RegisterPrometheusExporter(registry)
 		if err != nil {
-			logger.WithError(err).Fatal("Could not create register prometheus exporter")
+			logger.WithError(err).Fatal("Could not register prometheus exporter")
 		}
 		server.Handle("/metrics", metricHandler)
 		health = healthcheck.NewMetricsHandler(registry, "agones")
-		rs = append(rs, metrics.NewController(kubeClient, agonesClient, agonesInformerFactory))
 	} else {
 		health = healthcheck.NewHandler()
 	}
 
+	// If we are using Prometheus only exporter we can make reporting more often,
+	// every 1 seconds, if we are using Stackdriver we would use 60 seconds reporting period,
+	// which is a requirements of Stackdriver, otherwise most of time series would be invalid for Stackdriver
+	metrics.SetReportingPeriod(ctlConf.PrometheusMetrics, ctlConf.Stackdriver)
+
+	// Add metrics controller only if we configure one of metrics exporters
+	if ctlConf.PrometheusMetrics || ctlConf.Stackdriver {
+		rs = append(rs, metrics.NewController(kubeClient, agonesClient, agonesInformerFactory))
+	}
+
 	server.Handle("/", health)
 
 	allocationMutex := &sync.Mutex{}
@@ -170,7 +192,9 @@ func parseEnvFlags() config {
 	viper.SetDefault(pullSidecarFlag, false)
 	viper.SetDefault(certFileFlag, filepath.Join(base, "certs/server.crt"))
 	viper.SetDefault(keyFileFlag, filepath.Join(base, "certs/server.key"))
-	viper.SetDefault(enableMetricsFlag, true)
+	viper.SetDefault(enablePrometheusMetricsFlag, true)
+	viper.SetDefault(enableStackdriverMetricsFlag, false)
+	viper.SetDefault(projectIDFlag, "")
 
 	pflag.String(sidecarImageFlag, viper.GetString(sidecarImageFlag), "Flag to overwrite the GameServer sidecar image that is used. Can also use SIDECAR env variable")
 	pflag.String(sidecarCPULimitFlag, viper.GetString(sidecarCPULimitFlag), "Flag to overwrite the GameServer sidecar container's cpu limit. Can also use SIDECAR_CPU_LIMIT env variable")
@@ -181,7 +205,9 @@ func parseEnvFlags() config {
 	pflag.String(keyFileFlag, viper.GetString(keyFileFlag), "Optional. Path to the key file")
 	pflag.String(certFileFlag, viper.GetString(certFileFlag), "Optional. Path to the crt file")
 	pflag.String(kubeconfigFlag, viper.GetString(kubeconfigFlag), "Optional. kubeconfig to run the controller out of the cluster. Only use it for debugging as webhook won't works.")
-	pflag.Bool(enableMetricsFlag, viper.GetBool(enableMetricsFlag), "Flag to activate metrics of Agones. Can also use METRICS env variable.")
+	pflag.Bool(enablePrometheusMetricsFlag, viper.GetBool(enablePrometheusMetricsFlag), "Flag to activate metrics of Agones. Can also use PROMETHEUS_EXPORTER env variable.")
+	pflag.Bool(enableStackdriverMetricsFlag, viper.GetBool(enableStackdriverMetricsFlag), "Flag to activate stackdriver monitoring metrics for Agones. Can also use STACKDRIVER_EXPORTER env variable.")
+	pflag.String(projectIDFlag, viper.GetString(projectIDFlag), "GCP ProjectID used for Stackdriver, if not specified ProjectID from Application Default Credentials would be used. Can also use GCP_PROJECT_ID env variable.")
 	pflag.Parse()
 
 	viper.SetEnvKeyReplacer(strings.NewReplacer("-", "_"))
@@ -194,7 +220,9 @@ func parseEnvFlags() config {
 	runtime.Must(viper.BindEnv(keyFileFlag))
 	runtime.Must(viper.BindEnv(certFileFlag))
 	runtime.Must(viper.BindEnv(kubeconfigFlag))
-	runtime.Must(viper.BindEnv(enableMetricsFlag))
+	runtime.Must(viper.BindEnv(enablePrometheusMetricsFlag))
+	runtime.Must(viper.BindEnv(enableStackdriverMetricsFlag))
+	runtime.Must(viper.BindEnv(projectIDFlag))
 	runtime.Must(viper.BindPFlags(pflag.CommandLine))
 
 	request, err := resource.ParseQuantity(viper.GetString(sidecarCPURequestFlag))
@@ -217,7 +245,9 @@ func parseEnvFlags() config {
 		KeyFile:           viper.GetString(keyFileFlag),
 		CertFile:          viper.GetString(certFileFlag),
 		KubeConfig:        viper.GetString(kubeconfigFlag),
-		Metrics:           viper.GetBool(enableMetricsFlag),
+		PrometheusMetrics: viper.GetBool(enablePrometheusMetricsFlag),
+		Stackdriver:       viper.GetBool(enableStackdriverMetricsFlag),
+		GCPProjectID:      viper.GetString(projectIDFlag),
 	}
 }
 
@@ -229,10 +259,12 @@ type config struct {
 	SidecarCPURequest resource.Quantity
 	SidecarCPULimit   resource.Quantity
 	AlwaysPullSidecar bool
-	Metrics           bool
+	PrometheusMetrics bool
+	Stackdriver       bool
 	KeyFile           string
 	CertFile          string
 	KubeConfig        string
+	GCPProjectID      string
 }
 
 // validate ensures the ctlConfig data is valid.

install/helm/agones/templates/controller.yaml

@@ -40,7 +40,7 @@ spec:
 {{- if .Values.agones.controller.generateTLS }}
         revision/tls-cert: {{ .Release.Revision | quote }}
 {{- end }}
-{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.enabled) }}
+{{- if and (.Values.agones.metrics.prometheusServiceDiscovery) (.Values.agones.metrics.prometheusEnabled) }}
         prometheus.io/scrape: "true"
         prometheus.io/port: {{ .Values.agones.controller.http.port | quote }}
         prometheus.io/path: "/metrics"
@@ -82,8 +82,12 @@ spec:
           value: {{ .Values.agones.image.sdk.alwaysPull | quote }}
         - name: SIDECAR_CPU_REQUEST
           value: {{ .Values.agones.image.sdk.cpuRequest | quote }}
-        - name: METRICS
-          value: {{ .Values.agones.metrics.enabled | quote }}
+        - name: PROMETHEUS_EXPORTER
+          value: {{ .Values.agones.metrics.prometheusEnabled | quote }}
+        - name: STACKDRIVER_EXPORTER
+          value: {{ .Values.agones.metrics.stackdriverEnabled | quote }}
+        - name: GCP_PROJECT_ID
+          value: {{ .Values.agones.metrics.stackdriverProjectID | quote }}
         - name: SIDECAR_CPU_LIMIT
           value: {{ .Values.agones.image.sdk.cpuLimit | quote }}
         livenessProbe:

install/helm/agones/values.yaml

@@ -16,8 +16,10 @@
 
 agones:
   metrics:
-    enabled: true
+    prometheusEnabled: true
     prometheusServiceDiscovery: true
+    stackdriverEnabled: false
+    stackdriverProjectID: ""
   rbacEnabled: true
   crds:
     install: true

install/yaml/install.yaml

@@ -1047,8 +1047,12 @@ spec:
           value: "false"
         - name: SIDECAR_CPU_REQUEST
           value: "30m"
-        - name: METRICS
+        - name: PROMETHEUS_EXPORTER
           value: "true"
+        - name: STACKDRIVER_EXPORTER
+          value: "false"
+        - name: GCP_PROJECT_ID
+          value: ""
         - name: SIDECAR_CPU_LIMIT
           value: "0"
         livenessProbe:

pkg/metrics/exporter.go

@@ -18,6 +18,7 @@ import (
 	"net/http"
 	"time"
 
+	"contrib.go.opencensus.io/exporter/stackdriver"
 	prom "github.com/prometheus/client_golang/prometheus"
 	"go.opencensus.io/exporter/prometheus"
 	"go.opencensus.io/stats/view"
@@ -41,7 +42,40 @@ func RegisterPrometheusExporter(registry *prom.Registry) (http.Handler, error) {
 		return nil, err
 	}
 	view.RegisterExporter(pe)
-	// since we're using prometheus we can report faster as we're only exposing metrics in memory
-	view.SetReportingPeriod(1 * time.Second)
+
 	return pe, nil
 }
+
+// RegisterStackdriverExporter register a Stackdriver exporter to OpenCensus.
+// It will add Agones metrics into Stackdriver on Google Cloud.
+func RegisterStackdriverExporter(projectID string) (sd *stackdriver.Exporter, err error) {
+	// Default project will be used
+	sd, err = stackdriver.NewExporter(stackdriver.Options{
+		ProjectID: projectID,
+		// MetricPrefix helps uniquely identify your metrics.
+		MetricPrefix: "agones",
+	})
+	if err != nil {
+		return
+	}
+
+	// Register it as a metrics exporter
+	view.RegisterExporter(sd)
+	return
+}
+
+// SetReportingPeriod set appropriate reporting period which depends on exporters
+// we are going to use
+func SetReportingPeriod(prometheus, stackdriver bool) {
+	// if we're using only prometheus we can report faster as we're only exposing metrics in memory
+	reportingPeriod := 1 * time.Second
+	if stackdriver {
+		// There is a limitation on Stackdriver that reporting should
+		// be equal or more than 1 minute
+		reportingPeriod = 60 * time.Second
+	}
+
+	if stackdriver || prometheus {
+		view.SetReportingPeriod(reportingPeriod)
+	}
+}

site/content/en/docs/Contribute/_index.md

@@ -60,7 +60,7 @@ or to hide a section from 0.8.0 onward:
 
 ```markdown
 {{\% feature expiryVersion="0.8.0" %}}
-  This is my special content that she be hidden <= 0.8.0
+  This is my special content that will be hidden >= 0.8.0
 {{\% /feature %}}
 ```
 

site/content/en/docs/Guides/metrics.md

@@ -23,13 +23,14 @@ Table of Contents
   - [Installation](#installation)
     - [Prometheus installation](#prometheus-installation)
     - [Grafana installation](#grafana-installation)
+    - [Stackdriver installation](#stackdriver-installation)
   - [Adding more metrics](#adding-more-metrics)
 
 ## Backend integrations
 
 ### Prometheus
 
-If you are running a [Prometheus](https://prometheus.io/) instance you just need to ensure that metrics and kubernetes service discovery are enabled. (helm chart values `agones.metrics.enabled` and `agones.metrics.prometheusServiceDiscovery`). This will automatically add annotations required by Prometheus to discover Agones metrics and start collecting them. (see [example](https://github.com/prometheus/prometheus/tree/master/documentation/examples/kubernetes-rabbitmq))
+If you are running a [Prometheus](https://prometheus.io/) instance you just need to ensure that metrics and kubernetes service discovery are enabled. (helm chart values {{% feature expiryVersion="0.8.0" %}}`agones.metrics.enabled`{{% /feature %}}{{% feature publishVersion="0.8.0" %}}`agones.metrics.prometheusEnabled`{{% /feature %}} and `agones.metrics.prometheusServiceDiscovery`). This will automatically add annotations required by Prometheus to discover Agones metrics and start collecting them. (see [example](https://github.com/prometheus/prometheus/tree/master/documentation/examples/kubernetes-rabbitmq))
 
 ### Prometheus Operator
 
@@ -54,10 +55,17 @@ Finally include that `ServiceMonitor` in your [Prometheus instance CRD](https://
 
 ### Stackdriver
 
+{{% feature expiryVersion="0.8.0" %}}
 We don't yet support the [OpenCensus Stackdriver exporter](https://opencensus.io/exporters/supported-exporters/go/stackdriver/)
 but you can still use the Prometheus Stackdriver integration by following these [instructions](https://cloud.google.com/monitoring/kubernetes-engine/prometheus).
 Annotations required by this integration can be activated by setting the `agones.metrics.prometheusServiceDiscovery` 
 to true (default) via the [helm chart value]({{< relref "../Installation/helm.md" >}}).
+{{% /feature %}}
+{{% feature publishVersion="0.8.0" %}}
+We support the [OpenCensus Stackdriver exporter](https://opencensus.io/exporters/supported-exporters/go/stackdriver/). 
+In order to use it you should enable [Stackdriver Monitoring API](https://cloud.google.com/monitoring/api/enable-api) in Google Cloud Console.
+Follow the [Stackdriver Installation steps](#stackdriver-installation) to see your metrics on Stackdriver Monitoring website.
+{{% /feature %}}
 
 ## Metrics available
 
@@ -174,6 +182,39 @@ Open a web browser to [http://127.0.0.1:3000](http://127.0.0.1:3000), you should
 
 > Makefile targets `make grafana-portforward`,`make kind-grafana-portforward` and `make minikube-grafana-portforward`.
 
+{{% feature publishVersion="0.8.0" %}}
+### Stackdriver installation
+
+In order to use [Stackdriver monitoring](https://app.google.stackdriver.com) you should [enable Stackdriver Monitoring API](https://cloud.google.com/monitoring/api/enable-api) on Google Cloud Console. You need to grant all the necessary permissions to the users (see [Access Control Guide](https://cloud.google.com/monitoring/access-control)). Stackdriver exporter uses a strategy called Application Default Credentials (ADC) to find your application's credentials. Details could be found here [Setting Up Authentication for Server to Server Production Applications](https://cloud.google.com/docs/authentication/production).
+
+Note that Stackdriver monitoring is enabled by default on GKE clusters, however you can follow this [guide](https://cloud.google.com/kubernetes-engine/docs/how-to/monitoring#enabling_stackdriver_monitoring) if it was disabled on your GKE cluster.
+
+Default metrics exporter is Prometheus. If you are using the [Helm installation]({{< ref "/docs/Installation/helm.md" >}}), you can install or upgrade Agones to use Stackdriver, using the following chart parameters:
+```
+helm upgrade --install --wait --set agones.metrics.stackdriverEnabled=true --set agones.metrics.prometheusEnabled=false --set agones.metrics.prometheusServiceDiscovery=false my-release-name agones/agones
+```
+
+With this configuration only Stackdriver exporter would be used instead of Prometheus exporter.
+
+Create a Fleet or a Gameserver in order to check that connection with stackdriver API is configured properly and so that you will be able to see the metrics data.
+
+Visit [Stackdriver monitoring](https://app.google.stackdriver.com) website, select your project, or choose `Create a new Workspace` and select GCP project where your cluster resides. In [Stackdriver metrics explorer](https://cloud.google.com/monitoring/charts/metrics-explorer) you should be able to find new metrics with prefix `agones/` (resource type is `Global`) after a couple of minutes. Choose the metrics you are interested in and add to a single or separate graphs. You can create multiple graphs, save them into your dashboard and use various aggregation parameters and reducers for each graph.
+
+Example of the dashboard appearance is provided below:
+
+![stackdriver monitoring dashboard](../../../images/stackdriver-metrics-dashboard.png)
+
+Currently there exists only manual way of configuring Stackdriver Dashboard. So it is up to you to set an Alignment Period (minimal is 1 minute), GroupBy, Filter parameters and other graph settings.
+
+#### Troubleshooting
+If you can't see Agones metrics you should have a look at the controller logs for connection errors. Also ensure that your cluster has the necessary credentials to interact with Stackdriver Monitoring. You can configure `stackdriverProjectID` manually, if the automatic discovery is not working.
+
+Permissions problem example from controller logs:
+```
+Failed to export to Stackdriver: rpc error: code = PermissionDenied desc = Permission monitoring.metricDescriptors.create denied (or the resource may not exist).
+```
+{{% /feature %}}
+
 ## Adding more metrics
 
 If you want to contribute and add more metrics we recommend to use shared informers (cache) as it is currently implemented in the {{< ghlink href="pkg/metrics/controller.go" branch="master" >}}metrics controller{{< /ghlink >}}. 

site/content/en/docs/Installation/helm.md

@@ -106,7 +106,7 @@ The following tables lists the configurable parameters of the Agones chart and t
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------- |
 | `agones.rbacEnabled`                                | Creates RBAC resources. Must be set for any cluster configured with RBAC                        | `true`                 |
 | `agones.crds.install`                               | Install the CRDs with this chart. Useful to disable if you want to subchart (since crd-install hook is broken), so you can copy the CRDs into your own chart. | `true` |
-| `agones.crds.cleanupOnDelete`                       | Run the pre-delete hook to delete all GameServers and their backing Pods when deleting the helm chart, so that all CRDs can be removed on chart deletion | `true`          |  
+| `agones.crds.cleanupOnDelete`                       | Run the pre-delete hook to delete all GameServers and their backing Pods when deleting the helm chart, so that all CRDs can be removed on chart deletion | `true`          |
 | `agones.metrics.enabled`                            | Enables controller metrics on port `8080` and path `/metrics`                                   | `true`                 |
 | `agones.metrics.prometheusServiceDiscovery`         | Adds annotations for Prometheus ServiceDiscovery (and also Strackdriver)                        | `true`                 |
 | `agones.serviceaccount.controller`                  | Service account name for the controller                                                         | `agones-controller`    |
@@ -152,6 +152,9 @@ The following tables lists the configurable parameters of the Agones chart and t
 
 | Parameter                                           | Description                                                                                     | Default                |
 | --------------------------------------------------- | ----------------------------------------------------------------------------------------------- | ---------------------- |
+| `agones.metrics.prometheusEnabled`                  | Enables controller metrics on port `8080` and path `/metrics`                                   | `true`                 |
+| `agones.metrics.stackdriverEnabled`                 | Enables Stackdriver exporter of controller metrics              | `false`                 |
+| `agones.metrics.stackdriverProjectID`               | This overrides the default gcp project id for use with stackdriver            | ``                 |
 | `agones.controller.nodeSelector`                    | Controller [node labels](nodeSelector) for pod assignment                                       | `{}`                   |
 | `agones.controller.tolerations`                     | Controller [toleration][toleration] labels for pod assignment                                   | `[]`                   |
 | `agones.controller.affinity`                        | Controller [affinity](affinity) settings for pod assignment                                     | `{}`                   |