Instrument Prometheus query times

This commit instruments prometheus query times by prometheus endpoint,
allowing us to see how quickly prometheus answers the queries that
back the custom-metrics API endpoints.

cmd/app/start.go

@@ -33,6 +33,7 @@ import (
 	"k8s.io/custom-metrics-boilerplate/pkg/cmd/server"
 	cmprov "github.com/directxman12/k8s-prometheus-adapter/pkg/custom-provider"
 	prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
+	mprom "github.com/directxman12/k8s-prometheus-adapter/pkg/client/metrics"
 )
 
 // NewCommandStartPrometheusAdapterServer provides a CLI handler for 'start master' command
@@ -87,6 +88,8 @@ func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-c
 		return err
 	}
 
+	config.GenericConfig.EnableMetrics = true
+
 	var clientConfig *rest.Config
 	if len(o.RemoteKubeConfigFile) > 0 {
 		loadingRules := &clientcmd.ClientConfigLoadingRules{ExplicitPath: o.RemoteKubeConfigFile}
@@ -122,7 +125,9 @@ func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-c
 	if err != nil {
 		return fmt.Errorf("invalid Prometheus URL %q: %v", baseURL, err)
 	}
-	promClient := prom.NewClient(http.DefaultClient, baseURL)
+	genericPromClient := prom.NewGenericAPIClient(http.DefaultClient, baseURL)
+	instrumentedGenericPromClient := mprom.InstrumentGenericAPIClient(genericPromClient, baseURL.String())
+	promClient := prom.NewClientForAPI(instrumentedGenericPromClient)
 
 	cmProvider := cmprov.NewPrometheusProvider(dynamicMapper, clientPool, promClient, o.MetricsRelistInterval, o.RateInterval)
 

pkg/client/metrics/metrics.go

@@ -0,0 +1,78 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+package metrics
+
+import (
+	"context"
+	"net/url"
+	"time"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/directxman12/k8s-prometheus-adapter/pkg/client"
+)
+
+var (
+	// queryLatency is the total latency of any query going through the
+	// various endpoints (query, range-query, series).  It includes some deserialization
+	// overhead and HTTP overhead.
+	queryLatency = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: "cmgateway_prometheus_query_latency_seconds",
+			Help: "Prometheus client query latency in seconds.  Broken down by target prometheus endpoint and target server",
+			Buckets: prometheus.ExponentialBuckets(0.0001, 2, 10),
+		},
+		[]string{"endpoint", "server"},
+	)
+)
+
+func init() {
+	prometheus.MustRegister(queryLatency)
+}
+
+// instrumentedClient is a client.GenericAPIClient which instruments calls to Do,
+// capturing request latency.
+type instrumentedGenericClient struct {
+	serverName string
+	client client.GenericAPIClient
+}
+
+func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint string, query url.Values) (client.APIResponse, error) {
+	startTime := time.Now()
+	var err error
+	defer func() {
+		endTime := time.Now()
+		// skip calls where we don't make the actual request
+		if err != nil {
+			if _, wasAPIErr := err.(*client.Error); !wasAPIErr {
+				// TODO: measure API errors by code?
+				return
+			}
+		}
+		queryLatency.With(prometheus.Labels{"endpoint": endpoint, "server": c.serverName}).Observe(endTime.Sub(startTime).Seconds())
+	}()
+
+	var resp client.APIResponse
+	resp, err = c.client.Do(ctx, verb, endpoint, query)
+	return resp, err
+}
+
+func InstrumentGenericAPIClient(client client.GenericAPIClient, serverName string) client.GenericAPIClient {
+	return &instrumentedGenericClient{
+		serverName: serverName,
+		client: client,
+	}
+}