Merge pull request #1155 from brancz/prometheus

Add Prometheus metrics

bill-of-materials.json

@@ -8,6 +8,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/beorn7/perks/quantile",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
 	{
 		"project": "github.com/cockroachdb/cockroach-go/crdb",
 		"licenses": [
@@ -44,6 +53,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/felixge/httpsnoop",
+		"licenses": [
+			{
+				"type": "MIT License",
+				"confidence": 0.9891304347826086
+			}
+		]
+	},
 	{
 		"project": "github.com/ghodss/yaml",
 		"licenses": [
@@ -89,6 +107,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/grpc-ecosystem/go-grpc-prometheus",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "github.com/gtank/cryptopasta",
 		"licenses": [
@@ -125,6 +152,15 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/matttproud/golang_protobuf_extensions/pbutil",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 0.9988925802879292
+			}
+		]
+	},
 	{
 		"project": "github.com/pquerna/cachecontrol",
 		"licenses": [
@@ -134,6 +170,42 @@
 			}
 		]
 	},
+	{
+		"project": "github.com/prometheus/client_golang/prometheus",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/client_model/go",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/common",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
+	{
+		"project": "github.com/prometheus/procfs",
+		"licenses": [
+			{
+				"type": "Apache License 2.0",
+				"confidence": 1
+			}
+		]
+	},
 	{
 		"project": "github.com/russellhaering/goxmldsig",
 		"licenses": [

cmd/dex/config.go

@@ -19,13 +19,14 @@ import (
 
 // Config is the config format for the main application.
 type Config struct {
-	Issuer  string  `json:"issuer"`
-	Storage Storage `json:"storage"`
-	Web     Web     `json:"web"`
-	OAuth2  OAuth2  `json:"oauth2"`
-	GRPC    GRPC    `json:"grpc"`
-	Expiry  Expiry  `json:"expiry"`
-	Logger  Logger  `json:"logger"`
+	Issuer    string    `json:"issuer"`
+	Storage   Storage   `json:"storage"`
+	Web       Web       `json:"web"`
+	Telemetry Telemetry `json:"telemetry"`
+	OAuth2    OAuth2    `json:"oauth2"`
+	GRPC      GRPC      `json:"grpc"`
+	Expiry    Expiry    `json:"expiry"`
+	Logger    Logger    `json:"logger"`
 
 	Frontend server.WebConfig `json:"frontend"`
 
@@ -104,6 +105,11 @@ type Web struct {
 	AllowedOrigins []string `json:"allowedOrigins"`
 }
 
+// Telemetry is the config format for telemetry including the HTTP server config.
+type Telemetry struct {
+	HTTP string `json:"http"`
+}
+
 // GRPC is the config for the gRPC API.
 type GRPC struct {
 	// The port to listen on.

cmd/dex/serve.go

@@ -14,6 +14,9 @@ import (
 	"time"
 
 	"github.com/ghodss/yaml"
+	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
@@ -93,7 +96,25 @@ func serve(cmd *cobra.Command, args []string) error {
 
 	logger.Infof("config issuer: %s", c.Issuer)
 
+	prometheusRegistry := prometheus.NewRegistry()
+	err = prometheusRegistry.Register(prometheus.NewGoCollector())
+	if err != nil {
+		return fmt.Errorf("failed to register Go runtime metrics: %v", err)
+	}
+
+	err = prometheusRegistry.Register(prometheus.NewProcessCollector(os.Getpid(), ""))
+	if err != nil {
+		return fmt.Errorf("failed to register process metrics: %v", err)
+	}
+
+	grpcMetrics := grpcprometheus.NewServerMetrics()
+	err = prometheusRegistry.Register(grpcMetrics)
+	if err != nil {
+		return fmt.Errorf("failed to register gRPC server metrics: %v", err)
+	}
+
 	var grpcOptions []grpc.ServerOption
+
 	if c.GRPC.TLSCert != "" {
 		if c.GRPC.TLSClientCA != "" {
 			// Parse certificates from certificate file and key file for server.
@@ -117,7 +138,11 @@ func serve(cmd *cobra.Command, args []string) error {
 				ClientAuth:   tls.RequireAndVerifyClientCert,
 				ClientCAs:    cPool,
 			}
-			grpcOptions = append(grpcOptions, grpc.Creds(credentials.NewTLS(&tlsConfig)))
+			grpcOptions = append(grpcOptions,
+				grpc.Creds(credentials.NewTLS(&tlsConfig)),
+				grpc.StreamInterceptor(grpcMetrics.StreamServerInterceptor()),
+				grpc.UnaryInterceptor(grpcMetrics.UnaryServerInterceptor()),
+			)
 		} else {
 			opt, err := credentials.NewServerTLSFromFile(c.GRPC.TLSCert, c.GRPC.TLSKey)
 			if err != nil {
@@ -199,6 +224,7 @@ func serve(cmd *cobra.Command, args []string) error {
 		Web:                    c.Frontend,
 		Logger:                 logger,
 		Now:                    now,
+		PrometheusRegistry:     prometheusRegistry,
 	}
 	if c.Expiry.SigningKeys != "" {
 		signingKeys, err := time.ParseDuration(c.Expiry.SigningKeys)
@@ -222,7 +248,17 @@ func serve(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to initialize server: %v", err)
 	}
 
+	telemetryServ := http.NewServeMux()
+	telemetryServ.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}))
+
 	errc := make(chan error, 3)
+	if c.Telemetry.HTTP != "" {
+		logger.Infof("listening (http/telemetry) on %s", c.Telemetry.HTTP)
+		go func() {
+			err := http.ListenAndServe(c.Telemetry.HTTP, telemetryServ)
+			errc <- fmt.Errorf("listening on %s failed: %v", c.Telemetry.HTTP, err)
+		}()
+	}
 	if c.Web.HTTP != "" {
 		logger.Infof("listening (http) on %s", c.Web.HTTP)
 		go func() {
@@ -247,6 +283,7 @@ func serve(cmd *cobra.Command, args []string) error {
 				}
 				s := grpc.NewServer(grpcOptions...)
 				api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger))
+				grpcMetrics.InitializeMetrics(s)
 				err = s.Serve(list)
 				return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
 			}()

examples/config-dev.yaml

@@ -20,6 +20,10 @@ web:
   # tlsCert: /etc/dex/tls.crt
   # tlsKey: /etc/dex/tls.key
 
+# Configuration for telemetry
+telemetry:
+  http: 0.0.0.0:5558
+
 # Uncomment this block to enable the gRPC API. This values MUST be different
 # from the HTTP endpoints.
 # grpc:

glide.yaml

@@ -158,3 +158,27 @@ import:
 # License bill of materials generator.
 - package: github.com/coreos/license-bill-of-materials
   version: d70207c33a3c79a1c0479b208f8b7ab6215144c7
+
+# monitoring packages
+- package: github.com/beorn7/perks/quantile
+  version: 3ac7bf7a47d159a033b107610db8a1b6575507a4
+- package: github.com/felixge/httpsnoop
+  version: eadd4fad6aac69ae62379194fe0219f3dbc80fd3
+- package: github.com/grpc-ecosystem/go-grpc-prometheus
+  version: 0dafe0d496ea71181bf2dd039e7e3f44b6bd11a7
+- package: github.com/matttproud/golang_protobuf_extensions/pbutil
+  version: fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a
+- package: github.com/prometheus/client_golang/prometheus
+  version: 967789050ba94deca04a5e84cce8ad472ce313c1
+- package: github.com/prometheus/client_golang/prometheus/promhttp
+  version: 967789050ba94deca04a5e84cce8ad472ce313c1
+- package: github.com/prometheus/client_model/go
+  version: fa8ad6fec33561be4280a8f0514318c79d7f6cb6
+- package: github.com/prometheus/common/expfmt
+  version: 49fee292b27bfff7f354ee0f64e1bc4850462edf
+- package: github.com/prometheus/common/internal/bitbucket.org/ww/goautoneg
+  version: 49fee292b27bfff7f354ee0f64e1bc4850462edf
+- package: github.com/prometheus/common/model
+  version: 49fee292b27bfff7f354ee0f64e1bc4850462edf
+- package: github.com/prometheus/procfs/xfs
+  version: a1dba9ce8baed984a2495b658c82687f8157b98f

server/server.go

@@ -8,15 +8,18 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
 	"time"
 
 	"golang.org/x/crypto/bcrypt"
 
+	"github.com/felixge/httpsnoop"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	"github.com/coreos/dex/connector"
@@ -75,6 +78,8 @@ type Config struct {
 	Web WebConfig
 
 	Logger logrus.FieldLogger
+
+	PrometheusRegistry *prometheus.Registry
 }
 
 // WebConfig holds the server's frontend templates and asset configuration.
@@ -214,9 +219,26 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 		}
 	}
 
+	requestCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "http_requests_total",
+		Help: "Count of all HTTP requests.",
+	}, []string{"handler", "code", "method"})
+
+	err = c.PrometheusRegistry.Register(requestCounter)
+	if err != nil {
+		return nil, fmt.Errorf("server: Failed to register Prometheus HTTP metrics: %v", err)
+	}
+
+	instrumentHandlerCounter := func(handlerName string, handler http.Handler) http.HandlerFunc {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			m := httpsnoop.CaptureMetrics(handler, w, r)
+			requestCounter.With(prometheus.Labels{"handler": handlerName, "code": strconv.Itoa(m.Code), "method": r.Method}).Inc()
+		})
+	}
+
 	r := mux.NewRouter()
 	handleFunc := func(p string, h http.HandlerFunc) {
-		r.HandleFunc(path.Join(issuerURL.Path, p), h)
+		r.HandleFunc(path.Join(issuerURL.Path, p), instrumentHandlerCounter(p, h))
 	}
 	handlePrefix := func(p string, h http.Handler) {
 		prefix := path.Join(issuerURL.Path, p)