Adding extra metric till now, not impacting path

This commit is contained in:
Vivek Kumar 2024-02-22 11:44:48 +05:30
parent 9156bf3fbc
commit e70cb42780
No known key found for this signature in database
GPG key ID: 81605FDFBA4EF440
3 changed files with 66 additions and 16 deletions

View file

@ -20,7 +20,7 @@ all: prometheus-adapter
SRC_DEPS=$(shell find pkg cmd -type f -name "*.go") SRC_DEPS=$(shell find pkg cmd -type f -name "*.go")
prometheus-adapter: $(SRC_DEPS) prometheus-adapter: $(SRC_DEPS)
CGO_ENABLED=0 GOARCH=$(ARCH) go build sigs.k8s.io/prometheus-adapter/cmd/adapter CGO_ENABLED=0 GOARCH=$(ARCH) go build -o prometheus-adapter-custom sigs.k8s.io/prometheus-adapter/cmd/adapter
.PHONY: container .PHONY: container
container: container:

View file

@ -271,11 +271,23 @@ func (cmd *PrometheusAdapter) addResourceMetricsAPI(promClient prom.Client, stop
return err return err
} }
config, err := cmd.Config()
if err != nil {
return err
}
config.GenericConfig.EnableMetrics = false
server, err := cmd.Server() server, err := cmd.Server()
if err != nil { if err != nil {
return err return err
} }
metricsHandler, err := mprom.MetricsHandler()
if err != nil {
return err
}
server.GenericAPIServer.Handler.NonGoRestfulMux.HandleFunc("/metrics", metricsHandler)
if err := api.Install(provider, podInformer.Lister(), informer.Core().V1().Nodes().Lister(), server.GenericAPIServer, nil); err != nil { if err := api.Install(provider, podInformer.Lister(), informer.Core().V1().Nodes().Lister(), server.GenericAPIServer, nil); err != nil {
return err return err
} }
@ -457,4 +469,4 @@ func parseHeaderArgs(args []string) http.Header {
headers.Add(parts[0], value) headers.Add(parts[0], value)
} }
return headers return headers
} }

View file

@ -18,11 +18,16 @@ package metrics
import ( import (
"context" "context"
"net/http"
"net/url" "net/url"
"time" "time"
"github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus"
apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/component-base/metrics"
"k8s.io/component-base/metrics/legacyregistry"
"sigs.k8s.io/prometheus-adapter/pkg/client" "sigs.k8s.io/prometheus-adapter/pkg/client"
) )
@ -30,18 +35,47 @@ var (
// queryLatency is the total latency of any query going through the // queryLatency is the total latency of any query going through the
// various endpoints (query, range-query, series). It includes some deserialization // various endpoints (query, range-query, series). It includes some deserialization
// overhead and HTTP overhead. // overhead and HTTP overhead.
queryLatency = prometheus.NewHistogramVec( queryLatency = metrics.NewHistogramVec(
prometheus.HistogramOpts{ &metrics.HistogramOpts{
Name: "cmgateway_prometheus_query_latency_seconds", Namespace: "prometheus_adapter",
Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server", Subsystem: "prometheus_client",
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 10), Name: "request_duration_seconds",
Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server",
Buckets: prometheus.DefBuckets,
}, },
[]string{"endpoint", "server"}, []string{"path", "server"},
) )
// define a counter for API errors for various ErrorTypes
apiErrorCount = metrics.NewCounterVec(
&metrics.CounterOpts{
Namespace: "prometheus_adapter",
Subsystem: "prometheus_client",
Name: "api_errors_total",
Help: "Total number of API errors",
},
[]string{"error_code", "path", "server"},
)
) )
func init() { func MetricsHandler() (http.HandlerFunc, error) {
prometheus.MustRegister(queryLatency) registry := metrics.NewKubeRegistry()
errRegisterQueryLatency := registry.Register(queryLatency)
if errRegisterQueryLatency != nil {
return nil, errRegisterQueryLatency
}
errRegisterAPIErrorCount := registry.Register(apiErrorCount)
if errRegisterAPIErrorCount != nil {
return nil, errRegisterAPIErrorCount
}
apimetrics.Register()
return func(w http.ResponseWriter, req *http.Request) {
legacyregistry.Handler().ServeHTTP(w, req)
metrics.HandlerFor(registry, metrics.HandlerOpts{}).ServeHTTP(w, req)
}, nil
} }
// instrumentedClient is a client.GenericAPIClient which instruments calls to Do, // instrumentedClient is a client.GenericAPIClient which instruments calls to Do,
@ -58,12 +92,16 @@ func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint strin
endTime := time.Now() endTime := time.Now()
// skip calls where we don't make the actual request // skip calls where we don't make the actual request
if err != nil { if err != nil {
if _, wasAPIErr := err.(*client.Error); !wasAPIErr { if apiErr, wasAPIErr := err.(*client.Error); wasAPIErr {
// TODO: measure API errors by code? // Measure API errors
return apiErrorCount.With(prometheus.Labels{"error_code": string(apiErr.Type), "path": endpoint, "server": c.serverName}).Inc()
} } else {
// Increment a generic error code counter
apiErrorCount.With(prometheus.Labels{"error_code": "generic", "path": endpoint, "server": c.serverName}).Inc()
}
return
} }
queryLatency.With(prometheus.Labels{"endpoint": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds()) queryLatency.With(prometheus.Labels{"path": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds())
}() }()
var resp client.APIResponse var resp client.APIResponse
@ -76,4 +114,4 @@ func InstrumentGenericAPIClient(client client.GenericAPIClient, serverName strin
serverName: serverName, serverName: serverName,
client: client, client: client,
} }
} }