diff --git a/Makefile b/Makefile index 249d8c3f..5ab9b9c2 100644 --- a/Makefile +++ b/Makefile @@ -20,7 +20,7 @@ all: prometheus-adapter SRC_DEPS=$(shell find pkg cmd -type f -name "*.go") prometheus-adapter: $(SRC_DEPS) - CGO_ENABLED=0 GOARCH=$(ARCH) go build sigs.k8s.io/prometheus-adapter/cmd/adapter + CGO_ENABLED=0 GOARCH=$(ARCH) go build -o prometheus-adapter-custom sigs.k8s.io/prometheus-adapter/cmd/adapter .PHONY: container container: diff --git a/cmd/adapter/adapter.go b/cmd/adapter/adapter.go index 13ef3f8d..55961eaf 100644 --- a/cmd/adapter/adapter.go +++ b/cmd/adapter/adapter.go @@ -271,11 +271,23 @@ func (cmd *PrometheusAdapter) addResourceMetricsAPI(promClient prom.Client, stop return err } + config, err := cmd.Config() + if err != nil { + return err + } + config.GenericConfig.EnableMetrics = false + server, err := cmd.Server() if err != nil { return err } + metricsHandler, err := mprom.MetricsHandler() + if err != nil { + return err + } + server.GenericAPIServer.Handler.NonGoRestfulMux.HandleFunc("/metrics", metricsHandler) + if err := api.Install(provider, podInformer.Lister(), informer.Core().V1().Nodes().Lister(), server.GenericAPIServer, nil); err != nil { return err } @@ -457,4 +469,4 @@ func parseHeaderArgs(args []string) http.Header { headers.Add(parts[0], value) } return headers -} +} \ No newline at end of file diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index b54067f2..8067872a 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -18,11 +18,16 @@ package metrics import ( "context" + "net/http" "net/url" "time" "github.com/prometheus/client_golang/prometheus" + apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "sigs.k8s.io/prometheus-adapter/pkg/client" ) @@ -30,18 +35,47 @@ var ( // queryLatency is the total latency of any query going through the // various endpoints (query, range-query, series). It includes some deserialization // overhead and HTTP overhead. - queryLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "cmgateway_prometheus_query_latency_seconds", - Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", - Buckets: prometheus.ExponentialBuckets(0.0001, 2, 10), + queryLatency = metrics.NewHistogramVec( + &metrics.HistogramOpts{ + Namespace: "prometheus_adapter", + Subsystem: "prometheus_client", + Name: "request_duration_seconds", + Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", + Buckets: prometheus.DefBuckets, }, - []string{"endpoint", "server"}, + []string{"path", "server"}, ) + + // define a counter for API errors for various ErrorTypes + apiErrorCount = metrics.NewCounterVec( + &metrics.CounterOpts{ + Namespace: "prometheus_adapter", + Subsystem: "prometheus_client", + Name: "api_errors_total", + Help: "Total number of API errors", + }, + []string{"error_code", "path", "server"}, + ) ) -func init() { - prometheus.MustRegister(queryLatency) +func MetricsHandler() (http.HandlerFunc, error) { + registry := metrics.NewKubeRegistry() + + errRegisterQueryLatency := registry.Register(queryLatency) + if errRegisterQueryLatency != nil { + return nil, errRegisterQueryLatency + } + + errRegisterAPIErrorCount := registry.Register(apiErrorCount) + if errRegisterAPIErrorCount != nil { + return nil, errRegisterAPIErrorCount + } + + apimetrics.Register() + return func(w http.ResponseWriter, req *http.Request) { + legacyregistry.Handler().ServeHTTP(w, req) + metrics.HandlerFor(registry, metrics.HandlerOpts{}).ServeHTTP(w, req) + }, nil } // instrumentedClient is a client.GenericAPIClient which instruments calls to Do, @@ -58,12 +92,16 @@ func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint strin endTime := time.Now() // skip calls where we don't make the actual request if err != nil { - if _, wasAPIErr := err.(*client.Error); !wasAPIErr { - // TODO: measure API errors by code? - return - } + if apiErr, wasAPIErr := err.(*client.Error); wasAPIErr { + // Measure API errors + apiErrorCount.With(prometheus.Labels{"error_code": string(apiErr.Type), "path": endpoint, "server": c.serverName}).Inc() + } else { + // Increment a generic error code counter + apiErrorCount.With(prometheus.Labels{"error_code": "generic", "path": endpoint, "server": c.serverName}).Inc() + } + return } - queryLatency.With(prometheus.Labels{"endpoint": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds()) + queryLatency.With(prometheus.Labels{"path": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds()) }() var resp client.APIResponse @@ -76,4 +114,4 @@ func InstrumentGenericAPIClient(client client.GenericAPIClient, serverName strin serverName: serverName, client: client, } -} +} \ No newline at end of file