From 5f03479d29d8cd8d6c14bdc884df4d7829eb983e Mon Sep 17 00:00:00 2001 From: Frederic Branczyk Date: Wed, 20 Dec 2017 16:03:32 +0100 Subject: [PATCH] *: Add go runtime, process, HTTP and gRPC metrics --- cmd/dex/config.go | 20 +++++++++++++------- cmd/dex/serve.go | 39 ++++++++++++++++++++++++++++++++++++++- examples/config-dev.yaml | 4 ++++ server/server.go | 24 +++++++++++++++++++++++- server/server_test.go | 4 +++- 5 files changed, 81 insertions(+), 10 deletions(-) diff --git a/cmd/dex/config.go b/cmd/dex/config.go index a4e1338c..416991ab 100644 --- a/cmd/dex/config.go +++ b/cmd/dex/config.go @@ -19,13 +19,14 @@ import ( // Config is the config format for the main application. type Config struct { - Issuer string `json:"issuer"` - Storage Storage `json:"storage"` - Web Web `json:"web"` - OAuth2 OAuth2 `json:"oauth2"` - GRPC GRPC `json:"grpc"` - Expiry Expiry `json:"expiry"` - Logger Logger `json:"logger"` + Issuer string `json:"issuer"` + Storage Storage `json:"storage"` + Web Web `json:"web"` + Telemetry Telemetry `json:"telemetry"` + OAuth2 OAuth2 `json:"oauth2"` + GRPC GRPC `json:"grpc"` + Expiry Expiry `json:"expiry"` + Logger Logger `json:"logger"` Frontend server.WebConfig `json:"frontend"` @@ -104,6 +105,11 @@ type Web struct { AllowedOrigins []string `json:"allowedOrigins"` } +// Telemetry is the config format for telemetry including the HTTP server config. +type Telemetry struct { + HTTP string `json:"http"` +} + // GRPC is the config for the gRPC API. type GRPC struct { // The port to listen on. diff --git a/cmd/dex/serve.go b/cmd/dex/serve.go index 1a4a6c82..3b0a0d7f 100644 --- a/cmd/dex/serve.go +++ b/cmd/dex/serve.go @@ -14,6 +14,9 @@ import ( "time" "github.com/ghodss/yaml" + grpcprometheus "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/sirupsen/logrus" "github.com/spf13/cobra" "google.golang.org/grpc" @@ -93,7 +96,25 @@ func serve(cmd *cobra.Command, args []string) error { logger.Infof("config issuer: %s", c.Issuer) + prometheusRegistry := prometheus.NewRegistry() + err = prometheusRegistry.Register(prometheus.NewGoCollector()) + if err != nil { + return fmt.Errorf("failed to register Go runtime metrics: %v", err) + } + + err = prometheusRegistry.Register(prometheus.NewProcessCollector(os.Getpid(), "")) + if err != nil { + return fmt.Errorf("failed to register process metrics: %v", err) + } + + grpcMetrics := grpcprometheus.NewServerMetrics() + err = prometheusRegistry.Register(grpcMetrics) + if err != nil { + return fmt.Errorf("failed to register gRPC server metrics: %v", err) + } + var grpcOptions []grpc.ServerOption + if c.GRPC.TLSCert != "" { if c.GRPC.TLSClientCA != "" { // Parse certificates from certificate file and key file for server. @@ -117,7 +138,11 @@ func serve(cmd *cobra.Command, args []string) error { ClientAuth: tls.RequireAndVerifyClientCert, ClientCAs: cPool, } - grpcOptions = append(grpcOptions, grpc.Creds(credentials.NewTLS(&tlsConfig))) + grpcOptions = append(grpcOptions, + grpc.Creds(credentials.NewTLS(&tlsConfig)), + grpc.StreamInterceptor(grpcMetrics.StreamServerInterceptor()), + grpc.UnaryInterceptor(grpcMetrics.UnaryServerInterceptor()), + ) } else { opt, err := credentials.NewServerTLSFromFile(c.GRPC.TLSCert, c.GRPC.TLSKey) if err != nil { @@ -199,6 +224,7 @@ func serve(cmd *cobra.Command, args []string) error { Web: c.Frontend, Logger: logger, Now: now, + PrometheusRegistry: prometheusRegistry, } if c.Expiry.SigningKeys != "" { signingKeys, err := time.ParseDuration(c.Expiry.SigningKeys) @@ -222,7 +248,17 @@ func serve(cmd *cobra.Command, args []string) error { return fmt.Errorf("failed to initialize server: %v", err) } + telemetryServ := http.NewServeMux() + telemetryServ.Handle("/metrics", promhttp.HandlerFor(prometheusRegistry, promhttp.HandlerOpts{})) + errc := make(chan error, 3) + if c.Telemetry.HTTP != "" { + logger.Infof("listening (http/telemetry) on %s", c.Telemetry.HTTP) + go func() { + err := http.ListenAndServe(c.Telemetry.HTTP, telemetryServ) + errc <- fmt.Errorf("listening on %s failed: %v", c.Telemetry.HTTP, err) + }() + } if c.Web.HTTP != "" { logger.Infof("listening (http) on %s", c.Web.HTTP) go func() { @@ -247,6 +283,7 @@ func serve(cmd *cobra.Command, args []string) error { } s := grpc.NewServer(grpcOptions...) api.RegisterDexServer(s, server.NewAPI(serverConfig.Storage, logger)) + grpcMetrics.InitializeMetrics(s) err = s.Serve(list) return fmt.Errorf("listening on %s failed: %v", c.GRPC.Addr, err) }() diff --git a/examples/config-dev.yaml b/examples/config-dev.yaml index 542c0cae..87783917 100644 --- a/examples/config-dev.yaml +++ b/examples/config-dev.yaml @@ -20,6 +20,10 @@ web: # tlsCert: /etc/dex/tls.crt # tlsKey: /etc/dex/tls.key +# Configuration for telemetry +telemetry: + http: 0.0.0.0:5558 + # Uncomment this block to enable the gRPC API. This values MUST be different # from the HTTP endpoints. # grpc: diff --git a/server/server.go b/server/server.go index 6b609cc7..3b586d8e 100644 --- a/server/server.go +++ b/server/server.go @@ -8,6 +8,7 @@ import ( "net/http" "net/url" "path" + "strconv" "strings" "sync" "sync/atomic" @@ -15,8 +16,10 @@ import ( "golang.org/x/crypto/bcrypt" + "github.com/felixge/httpsnoop" "github.com/gorilla/handlers" "github.com/gorilla/mux" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "github.com/coreos/dex/connector" @@ -75,6 +78,8 @@ type Config struct { Web WebConfig Logger logrus.FieldLogger + + PrometheusRegistry *prometheus.Registry } // WebConfig holds the server's frontend templates and asset configuration. @@ -214,9 +219,26 @@ func newServer(ctx context.Context, c Config, rotationStrategy rotationStrategy) } } + requestCounter := prometheus.NewCounterVec(prometheus.CounterOpts{ + Name: "http_requests_total", + Help: "Count of all HTTP requests.", + }, []string{"handler", "code", "method"}) + + err = c.PrometheusRegistry.Register(requestCounter) + if err != nil { + return nil, fmt.Errorf("server: Failed to register Prometheus HTTP metrics: %v", err) + } + + instrumentHandlerCounter := func(handlerName string, handler http.Handler) http.HandlerFunc { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + m := httpsnoop.CaptureMetrics(handler, w, r) + requestCounter.With(prometheus.Labels{"handler": handlerName, "code": strconv.Itoa(m.Code), "method": r.Method}).Inc() + }) + } + r := mux.NewRouter() handleFunc := func(p string, h http.HandlerFunc) { - r.HandleFunc(path.Join(issuerURL.Path, p), h) + r.HandleFunc(path.Join(issuerURL.Path, p), instrumentHandlerCounter(p, h)) } handlePrefix := func(p string, h http.Handler) { prefix := path.Join(issuerURL.Path, p) diff --git a/server/server_test.go b/server/server_test.go index b5f73363..c1046afe 100644 --- a/server/server_test.go +++ b/server/server_test.go @@ -23,6 +23,7 @@ import ( oidc "github.com/coreos/go-oidc" "github.com/kylelemons/godebug/pretty" + "github.com/prometheus/client_golang/prometheus" "github.com/sirupsen/logrus" "golang.org/x/crypto/bcrypt" "golang.org/x/oauth2" @@ -92,7 +93,8 @@ func newTestServer(ctx context.Context, t *testing.T, updateConfig func(c *Confi Web: WebConfig{ Dir: filepath.Join(os.Getenv("GOPATH"), "src/github.com/coreos/dex/web"), }, - Logger: logger, + Logger: logger, + PrometheusRegistry: prometheus.NewRegistry(), } if updateConfig != nil { updateConfig(&config)