From 966ef227febcd6cd5de605a8b340837bcfb729e0 Mon Sep 17 00:00:00 2001 From: bogo Date: Thu, 17 Aug 2023 20:23:01 +0800 Subject: [PATCH 1/7] Fix metric unregistered --- pkg/client/metrics/metrics.go | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index b54067f2..43ed11bd 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -23,6 +23,8 @@ import ( "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" "sigs.k8s.io/prometheus-adapter/pkg/client" ) @@ -30,18 +32,18 @@ var ( // queryLatency is the total latency of any query going through the // various endpoints (query, range-query, series). It includes some deserialization // overhead and HTTP overhead. - queryLatency = prometheus.NewHistogramVec( - prometheus.HistogramOpts{ + queryLatency = metrics.NewHistogramVec( + &metrics.HistogramOpts{ Name: "cmgateway_prometheus_query_latency_seconds", Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", - Buckets: prometheus.ExponentialBuckets(0.0001, 2, 10), + Buckets: prometheus.ExponentialBuckets(0.01, 2, 10), }, []string{"endpoint", "server"}, ) ) func init() { - prometheus.MustRegister(queryLatency) + legacyregistry.MustRegister(queryLatency) } // instrumentedClient is a client.GenericAPIClient which instruments calls to Do, From 7cf3ac5d9028f928798bafa9906f96537184cff3 Mon Sep 17 00:00:00 2001 From: bogo <70442600+bogo-y@users.noreply.github.com> Date: Thu, 17 Aug 2023 20:57:47 +0800 Subject: [PATCH 2/7] Update metric buckets --- pkg/client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index 43ed11bd..03297853 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -36,7 +36,7 @@ var ( &metrics.HistogramOpts{ Name: "cmgateway_prometheus_query_latency_seconds", Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", - Buckets: prometheus.ExponentialBuckets(0.01, 2, 10), + Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), }, []string{"endpoint", "server"}, ) From 0032610ace8cb30da993aada8f23a4a3c6923735 Mon Sep 17 00:00:00 2001 From: bogo Date: Tue, 29 Aug 2023 17:19:47 +0800 Subject: [PATCH 3/7] change the latency metric and dependency inject prometheus registry --- cmd/adapter/adapter.go | 6 ++++++ pkg/client/metrics/metrics.go | 25 +++++++++++++++++++------ 2 files changed, 25 insertions(+), 6 deletions(-) diff --git a/cmd/adapter/adapter.go b/cmd/adapter/adapter.go index ed14e4f6..c20a6f59 100644 --- a/cmd/adapter/adapter.go +++ b/cmd/adapter/adapter.go @@ -273,6 +273,12 @@ func (cmd *PrometheusAdapter) addResourceMetricsAPI(promClient prom.Client, stop return err } + metricsHandler, err := mprom.MetricsHandler() + if err != nil { + return err + } + server.GenericAPIServer.Handler.NonGoRestfulMux.HandleFunc("/metrics", metricsHandler) + if err := api.Install(provider, podInformer.Lister(), informer.Core().V1().Nodes().Lister(), server.GenericAPIServer, nil); err != nil { return err } diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index 03297853..e9ad89f4 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -18,13 +18,15 @@ package metrics import ( "context" + "k8s.io/component-base/metrics/legacyregistry" + "net/http" "net/url" "time" "github.com/prometheus/client_golang/prometheus" + apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/component-base/metrics" - "k8s.io/component-base/metrics/legacyregistry" "sigs.k8s.io/prometheus-adapter/pkg/client" ) @@ -34,16 +36,27 @@ var ( // overhead and HTTP overhead. queryLatency = metrics.NewHistogramVec( &metrics.HistogramOpts{ - Name: "cmgateway_prometheus_query_latency_seconds", - Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", - Buckets: prometheus.ExponentialBuckets(0.001, 2, 10), + Namespace: "prometheus_adapter", + Subsystem: "prometheus_client", + Name: "request_duration_seconds", + Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", + Buckets: prometheus.DefBuckets, }, []string{"endpoint", "server"}, ) ) -func init() { - legacyregistry.MustRegister(queryLatency) +func MetricsHandler() (http.HandlerFunc, error) { + registry := metrics.NewKubeRegistry() + err := registry.Register(queryLatency) + if err != nil { + return nil, err + } + apimetrics.Register() + return func(w http.ResponseWriter, req *http.Request) { + legacyregistry.Handler().ServeHTTP(w, req) + metrics.HandlerFor(registry, metrics.HandlerOpts{}).ServeHTTP(w, req) + }, nil } // instrumentedClient is a client.GenericAPIClient which instruments calls to Do, From e772844ed858e696584c21922b6c3325259602d8 Mon Sep 17 00:00:00 2001 From: bogo <70442600+bogo-y@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:01:25 +0800 Subject: [PATCH 4/7] normalize import order --- pkg/client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index e9ad89f4..c5ea3778 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -18,13 +18,13 @@ package metrics import ( "context" - "k8s.io/component-base/metrics/legacyregistry" "net/http" "net/url" "time" "github.com/prometheus/client_golang/prometheus" + "k8s.io/component-base/metrics/legacyregistry" apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/component-base/metrics" "sigs.k8s.io/prometheus-adapter/pkg/client" From a53ee9eed169541718afd23fbdcc99d65051d157 Mon Sep 17 00:00:00 2001 From: bogo <70442600+bogo-y@users.noreply.github.com> Date: Sat, 7 Oct 2023 10:40:52 +0800 Subject: [PATCH 5/7] replace "endpoint" with "path" Co-authored-by: Simon Pasquier --- pkg/client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index c5ea3778..9b2d9e30 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -42,7 +42,7 @@ var ( Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", Buckets: prometheus.DefBuckets, }, - []string{"endpoint", "server"}, + []string{"path", "server"}, ) ) From 2a4a4316dd6b577b90f22064403038f1d97074ac Mon Sep 17 00:00:00 2001 From: bogo Date: Fri, 13 Oct 2023 20:14:34 +0800 Subject: [PATCH 6/7] run make update-lint and set EnabledMetrics=false in the server config --- cmd/adapter/adapter.go | 6 ++++++ pkg/client/metrics/metrics.go | 3 ++- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/cmd/adapter/adapter.go b/cmd/adapter/adapter.go index c20a6f59..c5208461 100644 --- a/cmd/adapter/adapter.go +++ b/cmd/adapter/adapter.go @@ -268,6 +268,12 @@ func (cmd *PrometheusAdapter) addResourceMetricsAPI(promClient prom.Client, stop return err } + config, err := cmd.Config() + if err != nil { + return err + } + config.GenericConfig.EnableMetrics = false + server, err := cmd.Server() if err != nil { return err diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index 9b2d9e30..0016d112 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -24,9 +24,10 @@ import ( "github.com/prometheus/client_golang/prometheus" - "k8s.io/component-base/metrics/legacyregistry" apimetrics "k8s.io/apiserver/pkg/endpoints/metrics" "k8s.io/component-base/metrics" + "k8s.io/component-base/metrics/legacyregistry" + "sigs.k8s.io/prometheus-adapter/pkg/client" ) From a5bcb3904635113d47d939f714a4c451cd4cc754 Mon Sep 17 00:00:00 2001 From: bogo <70442600+bogo-y@users.noreply.github.com> Date: Mon, 23 Oct 2023 11:15:51 +0800 Subject: [PATCH 7/7] replace "endpoint" with "path" --- pkg/client/metrics/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/client/metrics/metrics.go b/pkg/client/metrics/metrics.go index 0016d112..0faaed82 100644 --- a/pkg/client/metrics/metrics.go +++ b/pkg/client/metrics/metrics.go @@ -79,7 +79,7 @@ func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint strin return } } - queryLatency.With(prometheus.Labels{"endpoint": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds()) + queryLatency.With(prometheus.Labels{"path": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds()) }() var resp client.APIResponse