*: Add go runtime, process, HTTP and gRPC metrics

cmd/dex/config.go

@@ -19,13 +19,14 @@ import (
 
 // Config is the config format for the main application.
 type Config struct {
-	Issuer  string  `json:"issuer"`
+	Issuer    string    `json:"issuer"`
-	Storage Storage `json:"storage"`
+	Storage   Storage   `json:"storage"`
-	Web     Web     `json:"web"`
+	Web       Web       `json:"web"`
-	OAuth2  OAuth2  `json:"oauth2"`
+	Telemetry Telemetry `json:"telemetry"`
-	GRPC    GRPC    `json:"grpc"`
+	OAuth2    OAuth2    `json:"oauth2"`
-	Expiry  Expiry  `json:"expiry"`
+	GRPC      GRPC      `json:"grpc"`
-	Logger  Logger  `json:"logger"`
+	Expiry    Expiry    `json:"expiry"`
+	Logger    Logger    `json:"logger"`
 
 	Frontend server.WebConfig `json:"frontend"`
 
@@ -104,6 +105,11 @@ type Web struct {
 	AllowedOrigins []string `json:"allowedOrigins"`
 }
 
+// Telemetry is the config format for telemetry including the HTTP server config.
+type Telemetry struct {
+	HTTP string `json:"http"`
+}
+
 // GRPC is the config for the gRPC API.
 type GRPC struct {
 	// The port to listen on.

cmd/dex/serve.go

@@ -14,6 +14,9 @@ import (
 	"time"
 
 	"github.com/ghodss/yaml"
+	grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	"google.golang.org/grpc"
@@ -93,7 +96,25 @@ func serve(cmd *cobra.Command, args []string) error {
 
 	logger.Infof("config issuer: %s", c.Issuer)
 
+	prometheusRegistry := prometheus.NewRegistry()
+	err = prometheusRegistry.Register(prometheus.NewGoCollector())
+	if err != nil {
+		return fmt.Errorf("failed to register Go runtime metrics: %v", err)
+	}
+
+	err = prometheusRegistry.Register(prometheus.NewProcessCollector(os.Getpid(), ""))
+	if err != nil {
+		return fmt.Errorf("failed to register process metrics: %v", err)
+	}
+
+	grpcMetrics := grpcprometheus.NewServerMetrics()
+	err = prometheusRegistry.Register(grpcMetrics)
+	if err != nil {
+		return fmt.Errorf("failed to register gRPC server metrics: %v", err)
+	}
+
 	var grpcOptions []grpc.ServerOption
+
 	if c.GRPC.TLSCert != "" {
 		if c.GRPC.TLSClientCA != "" {
 			// Parse certificates from certificate file and key file for server.
@@ -117,7 +138,11 @@ func serve(cmd *cobra.Command, args []string) error {
 				ClientAuth:   tls.RequireAndVerifyClientCert,
 				ClientCAs:    cPool,
 			}
-			grpcOptions = append(grpcOptions, grpc.Creds(credentials.NewTLS(&tlsConfig)))
+			grpcOptions = append(grpcOptions,
+				grpc.Creds(credentials.NewTLS(&tlsConfig)),
+				grpc.StreamInterceptor(grpcMetrics.StreamServerInterceptor()),
+				grpc.UnaryInterceptor(grpcMetrics.UnaryServerInterceptor()),
+			)
 		} else {
 			opt, err := credentials.NewServerTLSFromFile(c.GRPC.TLSCert, c.GRPC.TLSKey)
 			if err != nil {
@@ -199,6 +224,7 @@ func serve(cmd *cobra.Command, args []string) error {
 		Web:                    c.Frontend,
 		Logger:                 logger,
 		Now:                    now,
+		PrometheusRegistry:     prometheusRegistry,
 	}
 	if c.Expiry.SigningKeys != "" {
 		signingKeys, err := time.ParseDuration(c.Expiry.SigningKeys)
@@ -222,7 +248,17 @@ func serve(cmd *cobra.Command, args []string) error {
 		return fmt.Errorf("failed to initialize server: %v", err)
 	}
 
+	telemetryServ := http.NewServeMux()
+	telemetryServ.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{}))
+
 	errc := make(chan error, 3)
+	if c.Telemetry.HTTP != "" {
+		logger.Infof("listening (http/telemetry) on %s", c.Telemetry.HTTP)
+		go func() {
+			err := http.ListenAndServe(c.Telemetry.HTTP, telemetryServ)
+			errc <- fmt.Errorf("listening on %s failed: %v", c.Telemetry.HTTP, err)
+		}()
+	}
 	if c.Web.HTTP != "" {
 		logger.Infof("listening (http) on %s", c.Web.HTTP)
 		go func() {
@@ -247,6 +283,7 @@ func serve(cmd *cobra.Command, args []string) error {
 				}
 				s := grpc.NewServer(grpcOptions...)
 				api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger))
+				grpcMetrics.InitializeMetrics(s)
 				err = s.Serve(list)
 				return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err)
 			}()

examples/config-dev.yaml

@@ -20,6 +20,10 @@ web:
   # tlsCert: /etc/dex/tls.crt
   # tlsKey: /etc/dex/tls.key
 
+# Configuration for telemetry
+telemetry:
+  http: 0.0.0.0:5558
+
 # Uncomment this block to enable the gRPC API. This values MUST be different
 # from the HTTP endpoints.
 # grpc:

server/server.go

@@ -8,6 +8,7 @@ import (
 	"net/http"
 	"net/url"
 	"path"
+	"strconv"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -15,8 +16,10 @@ import (
 
 	"golang.org/x/crypto/bcrypt"
 
+	"github.com/felixge/httpsnoop"
 	"github.com/gorilla/handlers"
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 
 	"github.com/coreos/dex/connector"
@@ -75,6 +78,8 @@ type Config struct {
 	Web WebConfig
 
 	Logger logrus.FieldLogger
+
+	PrometheusRegistry *prometheus.Registry
 }
 
 // WebConfig holds the server's frontend templates and asset configuration.
@@ -214,9 +219,26 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy)
 		}
 	}
 
+	requestCounter := prometheus.NewCounterVec(prometheus.CounterOpts{
+		Name: "http_requests_total",
+		Help: "Count of all HTTP requests.",
+	}, []string{"handler", "code", "method"})
+
+	err = c.PrometheusRegistry.Register(requestCounter)
+	if err != nil {
+		return nil, fmt.Errorf("server: Failed to register Prometheus HTTP metrics: %v", err)
+	}
+
+	instrumentHandlerCounter := func(handlerName string, handler http.Handler) http.HandlerFunc {
+		return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+			m := httpsnoop.CaptureMetrics(handler, w, r)
+			requestCounter.With(prometheus.Labels{"handler": handlerName, "code": strconv.Itoa(m.Code), "method": r.Method}).Inc()
+		})
+	}
+
 	r := mux.NewRouter()
 	handleFunc := func(p string, h http.HandlerFunc) {
-		r.HandleFunc(path.Join(issuerURL.Path, p), h)
+		r.HandleFunc(path.Join(issuerURL.Path, p), instrumentHandlerCounter(p, h))
 	}
 	handlePrefix := func(p string, h http.Handler) {
 		prefix := path.Join(issuerURL.Path, p)

server/server_test.go

@@ -23,6 +23,7 @@ import (
 
 	oidc "github.com/coreos/go-oidc"
 	"github.com/kylelemons/godebug/pretty"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/sirupsen/logrus"
 	"golang.org/x/crypto/bcrypt"
 	"golang.org/x/oauth2"
@@ -92,7 +93,8 @@ func newTestServer(ctx context.Context, t *testing.T, updateConfig func(c *Confi
 		Web: WebConfig{
 			Dir: filepath.Join(os.Getenv("GOPATH"), "src/github.com/coreos/dex/web"),
 		},
-		Logger: logger,
+		Logger:             logger,
+		PrometheusRegistry: prometheus.NewRegistry(),
 	}
 	if updateConfig != nil {
 		updateConfig(&config)