kubernetes-sigs/prometheus-adapter
1
0
Fork 1
mirror of https://github.com/kubernetes-sigs/prometheus-adapter.git synced 2026-04-05 17:27:51 +00:00
Code Issues Projects Releases Packages Activity Actions

Merge pull request #14 from DirectXMan12/build/initial-travis-setup

Browse source 
[build] set up Travis CI to run the unit tests
This commit is contained in:
Solly Ross 2017-06-27 19:07:18 -04:00 • committed by GitHub
commit 895183e503
15 changed files with 350 additions and 256 deletions
Download patch file Download diff file Expand all files Collapse all files

23
.travis.yml Normal file
View file

@ -0,0 +1,23 @@
language: go
go:
- 1.8
# blech, Travis downloads with capitals in DirectXMan12, which confuses go
go_import_path: github.com/directxman12/k8s-prometheus-adapter
addons:
apt:
sources:
- sourceline: 'ppa:masterminds/glide'
packages:
- glide
install:
- make -B vendor
script: make verify
cache:
directories:
- ~/.glide

10
Makefile
View file

@ -24,7 +24,7 @@ ifeq ($(ARCH),s390x)
BASEIMAGE?=s390x/busybox
endif
.PHONY: all build docker-build push-% push test
.PHONY: all build docker-build push-% push test verify-gofmt gofmt verify
all: build
build: vendor
@ -56,3 +56,11 @@ vendor: glide.lock
test: vendor
CGO_ENABLED=0 go test ./pkg/...
verify-gofmt:
./hack/gofmt-all.sh -v
gofmt:
./hack/gofmt-all.sh
verify: verify-gofmt test

30
cmd/adapter/adapter.go
View file

@ -17,27 +17,27 @@ limitations under the License.
package main
import (
"flag"
"os"
"runtime"
"flag"
"os"
"runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/util/logs"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/util/logs"
"github.com/directxman12/k8s-prometheus-adapter/cmd/adapter/app"
)
func main() {
logs.InitLogs()
defer logs.FlushLogs()
logs.InitLogs()
defer logs.FlushLogs()
if len(os.Getenv("GOMAXPROCS")) == 0 {
runtime.GOMAXPROCS(runtime.NumCPU())
}
if len(os.Getenv("GOMAXPROCS")) == 0 {
runtime.GOMAXPROCS(runtime.NumCPU())
}
cmd := app.NewCommandStartPrometheusAdapterServer(os.Stdout, os.Stderr, wait.NeverStop)
cmd.Flags().AddGoFlagSet(flag.CommandLine)
if err := cmd.Execute(); err != nil {
panic(err)
}
cmd := app.NewCommandStartPrometheusAdapterServer(os.Stdout, os.Stderr, wait.NeverStop)
cmd.Flags().AddGoFlagSet(flag.CommandLine)
if err := cmd.Execute(); err != nil {
panic(err)
}
}

20
cmd/adapter/app/start.go
View file

@ -17,23 +17,23 @@ limitations under the License.
package app
import (
"net/http"
"net/url"
"fmt"
"io"
"net/http"
"net/url"
"time"
"github.com/spf13/cobra"
"k8s.io/client-go/rest"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/discovery"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/pkg/api"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"github.com/directxman12/custom-metrics-boilerplate/pkg/cmd/server"
cmprov "github.com/directxman12/k8s-prometheus-adapter/pkg/custom-provider"
prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
mprom "github.com/directxman12/k8s-prometheus-adapter/pkg/client/metrics"
cmprov "github.com/directxman12/k8s-prometheus-adapter/pkg/custom-provider"
)
// NewCommandStartPrometheusAdapterServer provides a CLI handler for 'start master' command
@ -41,9 +41,9 @@ func NewCommandStartPrometheusAdapterServer(out, errOut io.Writer, stopCh <-chan
baseOpts := server.NewCustomMetricsAdapterServerOptions(out, errOut)
o := PrometheusAdapterServerOptions{
CustomMetricsAdapterServerOptions: baseOpts,
MetricsRelistInterval: 10 * time.Minute,
RateInterval: 5 * time.Minute,
PrometheusURL: "https://localhost",
MetricsRelistInterval: 10 * time.Minute,
RateInterval: 5 * time.Minute,
PrometheusURL: "https://localhost",
}
cmd := &cobra.Command{

9
glide.lock generated
View file

@ -1,5 +1,5 @@
hash: 5301bfa390fea808bce92640a9c3ea2165b8484a75a394144d242d49f4a8e6e6
updated: 2017-06-23T21:03:10.447897871-04:00
hash: 4432fd0d13b3a79f1febb9040cad9576793e44ab1c4d6e40abc664ad0582999f
updated: 2017-06-27T18:59:14.961201169-04:00
imports:
- name: bitbucket.org/ww/goautoneg
version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675
@ -386,10 +386,12 @@ imports:
- plugin/pkg/authenticator/token/webhook
- plugin/pkg/authorizer/webhook
- name: k8s.io/client-go
version: 450baa5d60f8d6a251c7682cb6f86e939b750b2d
version: b932a6d0e02c2b5afe156f4dd193a095efd8e954
repo: https://github.com/directxman12/client-go.git
subpackages:
- discovery
- dynamic
- dynamic/fake
- informers
- informers/apps
- informers/apps/v1beta1
@ -488,6 +490,7 @@ imports:
- pkg/version
- rest
- rest/watch
- testing
- tools/auth
- tools/cache
- tools/clientcmd

2
glide.yaml
View file

@ -8,6 +8,8 @@ import:
subpackages:
- pkg/util/logs
- package: k8s.io/client-go
repo: https://github.com/directxman12/client-go.git
version: feature/fake-dynamic-client
subpackages:
- kubernetes/typed/core/v1
- rest

41
hack/gofmt-all.sh Executable file
View file

@ -0,0 +1,41 @@
#!/bin/bash
# Copyright 2017 The Kubernetes Authors.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
set -o errexit
set -o nounset
set -o pipefail
verify=0
if [[ ${1:-} = "--verify" || ${1:-} = "-v" ]]; then
verify=1
fi
find_files() {
find . -not \( \( \
-wholename './_output' \
-o -wholename './vendor' \
\) -prune \) -name '*.go'
}
if [[ $verify -eq 1 ]]; then
diff=$(find_files | xargs gofmt -s -d 2>&1)
if [[ -n "${diff}" ]]; then
echo "gofmt -s -w $(echo "${diff}" | awk '/^diff / { print $2 }' | tr '\n' ' ')"
exit 1
fi
else
find_files | xargs gofmt -s -w
fi

18
pkg/client/api.go
View file

@ -20,14 +20,14 @@ import (
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/url"
"path"
"io/ioutil"
"io"
"github.com/prometheus/common/model"
"github.com/golang/glog"
"github.com/prometheus/common/model"
)
// APIClient is a raw client to the Prometheus Query API.
@ -44,7 +44,7 @@ type GenericAPIClient interface {
// httpAPIClient is a GenericAPIClient implemented in terms of an underlying http.Client.
type httpAPIClient struct {
client *http.Client
client *http.Client
baseURL *url.URL
}
@ -80,7 +80,7 @@ func (c *httpAPIClient) Do(ctx context.Context, verb, endpoint string, query url
if code/100 != 2 && code != 400 && code != 422 && code != 503 {
return APIResponse{}, &Error{
Type: ErrBadResponse,
Msg: fmt.Sprintf("unknown response code %d", code),
Msg: fmt.Sprintf("unknown response code %d", code),
}
}
@ -106,7 +106,7 @@ func (c *httpAPIClient) Do(ctx context.Context, verb, endpoint string, query url
if res.Status == ResponseError {
return res, &Error{
Type: res.ErrorType,
Msg: res.Error,
Msg: res.Error,
}
}
@ -116,15 +116,15 @@ func (c *httpAPIClient) Do(ctx context.Context, verb, endpoint string, query url
// NewGenericAPIClient builds a new generic Prometheus API client for the given base URL and HTTP Client.
func NewGenericAPIClient(client *http.Client, baseURL *url.URL) GenericAPIClient {
return &httpAPIClient{
client: client,
client: client,
baseURL: baseURL,
}
}
const (
queryURL = "/api/v1/query"
queryURL = "/api/v1/query"
queryRangeURL = "/api/v1/query_range"
seriesURL = "/api/v1/series"
seriesURL = "/api/v1/series"
)
// queryClient is a Client that connects to the Prometheus HTTP API.

2
pkg/client/interfaces.go
View file

@ -99,7 +99,7 @@ func (qr *QueryResult) UnmarshalJSON(b []byte) error {
// Series is roughly equivalent to model.Metrics, but has easy access to name
// and the set of non-name labels.
type Series struct {
Name string
Name string
Labels model.LabelSet
}

8
pkg/client/metrics/metrics.go
View file

@ -31,8 +31,8 @@ var (
// overhead and HTTP overhead.
queryLatency = prometheus.NewHistogramVec(
prometheus.HistogramOpts{
Name: "cmgateway_prometheus_query_latency_seconds",
Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server",
Name: "cmgateway_prometheus_query_latency_seconds",
Help: "Prometheus client query latency in seconds. Broken down by target prometheus endpoint and target server",
Buckets: prometheus.ExponentialBuckets(0.0001, 2, 10),
},
[]string{"endpoint", "server"},
@ -47,7 +47,7 @@ func init() {
// capturing request latency.
type instrumentedGenericClient struct {
serverName string
client client.GenericAPIClient
client client.GenericAPIClient
}
func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint string, query url.Values) (client.APIResponse, error) {
@ -73,6 +73,6 @@ func (c *instrumentedGenericClient) Do(ctx context.Context, verb, endpoint strin
func InstrumentGenericAPIClient(client client.GenericAPIClient, serverName string) client.GenericAPIClient {
return &instrumentedGenericClient{
serverName: serverName,
client: client,
client: client,
}
}

1
pkg/client/types.go
View file

@ -43,6 +43,7 @@ func (e *Error) Error() string {
// ResponseStatus is the type of response from the API: succeeded or error.
type ResponseStatus string
const (
ResponseSucceeded ResponseStatus = "succeeded"
ResponseError = "error"

31
pkg/custom-provider/metric_namer.go
View file

@ -21,12 +21,12 @@ import (
"strings"
"sync"
"k8s.io/apimachinery/pkg/runtime/schema"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/golang/glog"
prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
"github.com/golang/glog"
pmodel "github.com/prometheus/common/model"
)
@ -36,6 +36,7 @@ import (
// SeriesType represents the kind of series backing a metric.
type SeriesType int
const (
CounterSeries SeriesType = iota
SecondsCounterSeries
@ -97,7 +98,7 @@ func (r *basicSeriesRegistry) SetSeries(newSeries []prom.Series) error {
}
newMetrics := make([]provider.MetricInfo, 0, len(newInfo))
for info, _ := range newInfo {
for info := range newInfo {
newMetrics = append(newMetrics, info)
}
@ -248,13 +249,13 @@ func (n *metricNamer) processContainerSeries(series prom.Series, infos map[provi
info := provider.MetricInfo{
// TODO: is the plural correct?
GroupResource: schema.GroupResource{Resource: "pods"},
Namespaced: true,
Metric: name,
Namespaced: true,
Metric: name,
}
infos[info] = seriesInfo{
kind: metricKind,
baseSeries: prom.Series{Name: originalName},
kind: metricKind,
baseSeries: prom.Series{Name: originalName},
isContainer: true,
}
}
@ -272,8 +273,8 @@ func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[prov
for _, resource := range resources {
info := provider.MetricInfo{
GroupResource: resource,
Namespaced: true,
Metric: name,
Namespaced: true,
Metric: name,
}
// metrics describing namespaces aren't considered to be namespaced
@ -282,7 +283,7 @@ func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[prov
}
infos[info] = seriesInfo{
kind: metricKind,
kind: metricKind,
baseSeries: prom.Series{Name: series.Name},
}
}
@ -303,12 +304,12 @@ func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[prov
for _, resource := range resources {
info := provider.MetricInfo{
GroupResource: resource,
Namespaced: false,
Metric: name,
Namespaced: false,
Metric: name,
}
infos[info] = seriesInfo{
kind: metricKind,
kind: metricKind,
baseSeries: prom.Series{Name: series.Name},
}
}
@ -324,7 +325,7 @@ func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[prov
func (n *metricNamer) groupResourcesFromSeries(series prom.Series) ([]schema.GroupResource, error) {
// TODO: do we need to cache this, or is ResourceFor good enough?
var res []schema.GroupResource
for label, _ := range series.Labels {
for label := range series.Labels {
// TODO: figure out a way to let people specify a fully-qualified name in label-form
// TODO: will this work when missing a group?
gvr, err := n.mapper.ResourceFor(schema.GroupVersionResource{Resource: string(label)})

260
pkg/custom-provider/metric_namer_test.go
View file

@ -20,11 +20,12 @@ import (
"sort"
"testing"
"k8s.io/client-go/pkg/api"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
pmodel "github.com/prometheus/common/model"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
"k8s.io/client-go/pkg/api"
// install extensions so that our RESTMapper knows about it
_ "k8s.io/client-go/pkg/apis/extensions/install"
@ -35,9 +36,9 @@ import (
func setupMetricNamer(t *testing.T) *metricNamer {
return &metricNamer{
overrides: map[string]seriesSpec{
"container_actually_gauge_seconds_total": seriesSpec{
"container_actually_gauge_seconds_total": {
metricName: "actually_gauge",
kind: GaugeSeries,
kind: GaugeSeries,
},
},
mapper: api.Registry.RESTMapper(),
@ -45,56 +46,56 @@ func setupMetricNamer(t *testing.T) *metricNamer {
}
func TestMetricNamerContainerSeries(t *testing.T) {
testCases := []struct{
input prom.Series
testCases := []struct {
input prom.Series
outputMetricName string
outputInfo seriesInfo
outputInfo seriesInfo
}{
{
input: prom.Series{
Name: "container_actually_gauge_seconds_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_actually_gauge_seconds_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
outputMetricName: "actually_gauge",
outputInfo: seriesInfo{
baseSeries: prom.Series{Name: "container_actually_gauge_seconds_total"},
kind: GaugeSeries,
baseSeries: prom.Series{Name: "container_actually_gauge_seconds_total"},
kind: GaugeSeries,
isContainer: true,
},
},
{
input: prom.Series{
Name: "container_some_usage",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_usage",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
outputMetricName: "some_usage",
outputInfo: seriesInfo{
baseSeries: prom.Series{Name: "container_some_usage"},
kind: GaugeSeries,
baseSeries: prom.Series{Name: "container_some_usage"},
kind: GaugeSeries,
isContainer: true,
},
},
{
input: prom.Series{
Name: "container_some_count_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_count_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
outputMetricName: "some_count",
outputInfo: seriesInfo{
baseSeries: prom.Series{Name: "container_some_count_total"},
kind: CounterSeries,
baseSeries: prom.Series{Name: "container_some_count_total"},
kind: CounterSeries,
isContainer: true,
},
},
{
input: prom.Series{
Name: "container_some_time_seconds_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_time_seconds_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
outputMetricName: "some_time",
outputInfo: seriesInfo{
baseSeries: prom.Series{Name: "container_some_time_seconds_total"},
kind: SecondsCounterSeries,
baseSeries: prom.Series{Name: "container_some_time_seconds_total"},
kind: SecondsCounterSeries,
isContainer: true,
},
},
@ -108,9 +109,9 @@ func TestMetricNamerContainerSeries(t *testing.T) {
for _, test := range testCases {
namer.processContainerSeries(test.input, resMap)
metric := provider.MetricInfo{
Metric: test.outputMetricName,
Metric: test.outputMetricName,
GroupResource: schema.GroupResource{Resource: "pods"},
Namespaced: true,
Namespaced: true,
}
if assert.Contains(resMap, metric) {
assert.Equal(test.outputInfo, resMap[metric])
@ -130,64 +131,64 @@ func TestSeriesRegistry(t *testing.T) {
inputSeries := []prom.Series{
// container series
{
Name: "container_actually_gauge_seconds_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_actually_gauge_seconds_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
{
Name: "container_some_usage",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_usage",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
{
Name: "container_some_count_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_count_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
{
Name: "container_some_time_seconds_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_time_seconds_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
// namespaced series
// a series that should turn into multiple metrics
{
Name: "ingress_hits_total",
Labels: map[string]string{"ingress": "someingress", "service": "somesvc", "pod": "backend1", "namespace": "somens"},
Name: "ingress_hits_total",
Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend1", "namespace": "somens"},
},
{
Name: "ingress_hits_total",
Labels: map[string]string{"ingress": "someingress", "service": "somesvc", "pod": "backend2", "namespace": "somens"},
Name: "ingress_hits_total",
Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend2", "namespace": "somens"},
},
{
Name: "service_proxy_packets",
Labels: map[string]string{"service": "somesvc", "namespace": "somens"},
Name: "service_proxy_packets",
Labels: pmodel.LabelSet{"service": "somesvc", "namespace": "somens"},
},
{
Name: "work_queue_wait_seconds_total",
Labels: map[string]string{"deployment": "somedep", "namespace": "somens"},
Name: "work_queue_wait_seconds_total",
Labels: pmodel.LabelSet{"deployment": "somedep", "namespace": "somens"},
},
// non-namespaced series
{
Name: "node_gigawatts",
Labels: map[string]string{"node": "somenode"},
Name: "node_gigawatts",
Labels: pmodel.LabelSet{"node": "somenode"},
},
{
Name: "volume_claims_total",
Labels: map[string]string{"persistentvolume": "somepv"},
Name: "volume_claims_total",
Labels: pmodel.LabelSet{"persistentvolume": "somepv"},
},
{
Name: "node_fan_seconds_total",
Labels: map[string]string{"node": "somenode"},
Name: "node_fan_seconds_total",
Labels: pmodel.LabelSet{"node": "somenode"},
},
// unrelated series
{
Name: "admin_coffee_liters_total",
Labels: map[string]string{"admin": "some-admin"},
Name: "admin_coffee_liters_total",
Labels: pmodel.LabelSet{"admin": "some-admin"},
},
{
Name: "admin_unread_emails",
Labels: map[string]string{"admin": "some-admin"},
Name: "admin_unread_emails",
Labels: pmodel.LabelSet{"admin": "some-admin"},
},
{
Name: "admin_reddit_seconds_total",
Labels: map[string]string{"admin": "some-admin"},
Name: "admin_reddit_seconds_total",
Labels: pmodel.LabelSet{"admin": "some-admin"},
},
}
@ -195,152 +196,163 @@ func TestSeriesRegistry(t *testing.T) {
require.NoError(registry.SetSeries(inputSeries))
// make sure each metric got registered and can form queries
testCases := []struct{
title string
info provider.MetricInfo
namespace string
testCases := []struct {
title string
info provider.MetricInfo
namespace string
resourceNames []string
expectedKind SeriesType
expectedQuery string
expectedKind SeriesType
expectedQuery string
expectedGroupBy string
}{
// container metrics
{
title: "container metrics overrides / single resource name",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
namespace: "somens",
title: "container metrics overrides / single resource name",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
namespace: "somens",
resourceNames: []string{"somepod"},
expectedKind: GaugeSeries,
expectedQuery: "container_actually_gauge_seconds_total{pod_name=\"somepod\",container_name!=\"POD\",namespace=\"somens\"}",
expectedKind: GaugeSeries,
expectedQuery: "container_actually_gauge_seconds_total{pod_name=\"somepod\",container_name!=\"POD\",namespace=\"somens\"}",
expectedGroupBy: "pod_name",
},
{
title: "container metrics gauge / multiple resource names",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
namespace: "somens",
title: "container metrics gauge / multiple resource names",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
namespace: "somens",
resourceNames: []string{"somepod1", "somepod2"},
expectedKind: GaugeSeries,
expectedQuery: "container_some_usage{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedKind: GaugeSeries,
expectedQuery: "container_some_usage{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedGroupBy: "pod_name",
},
{
title: "container metrics counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"},
namespace: "somens",
title: "container metrics counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"},
namespace: "somens",
resourceNames: []string{"somepod1", "somepod2"},
expectedKind: CounterSeries,
expectedQuery: "container_some_count_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedKind: CounterSeries,
expectedQuery: "container_some_count_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedGroupBy: "pod_name",
},
{
title: "container metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"},
namespace: "somens",
title: "container metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"},
namespace: "somens",
resourceNames: []string{"somepod1", "somepod2"},
expectedKind: SecondsCounterSeries,
expectedQuery: "container_some_time_seconds_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedKind: SecondsCounterSeries,
expectedQuery: "container_some_time_seconds_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}",
expectedGroupBy: "pod_name",
},
// namespaced metrics
{
title: "namespaced metrics counter / multidimensional (service)",
info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "ingress_hits"},
namespace: "somens",
title: "namespaced metrics counter / multidimensional (service)",
info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "ingress_hits"},
namespace: "somens",
resourceNames: []string{"somesvc"},
expectedKind: CounterSeries,
expectedKind: CounterSeries,
expectedQuery: "ingress_hits_total{service=\"somesvc\",namespace=\"somens\"}",
},
{
title: "namespaced metrics counter / multidimensional (ingress)",
info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingress"}, true, "ingress_hits"},
namespace: "somens",
title: "namespaced metrics counter / multidimensional (ingress)",
info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingress"}, true, "ingress_hits"},
namespace: "somens",
resourceNames: []string{"someingress"},
expectedKind: CounterSeries,
expectedKind: CounterSeries,
expectedQuery: "ingress_hits_total{ingress=\"someingress\",namespace=\"somens\"}",
},
{
title: "namespaced metrics counter / multidimensional (pod)",
info: provider.MetricInfo{schema.GroupResource{Resource: "pod"}, true, "ingress_hits"},
namespace: "somens",
title: "namespaced metrics counter / multidimensional (pod)",
info: provider.MetricInfo{schema.GroupResource{Resource: "pod"}, true, "ingress_hits"},
namespace: "somens",
resourceNames: []string{"somepod"},
expectedKind: CounterSeries,
expectedKind: CounterSeries,
expectedQuery: "ingress_hits_total{pod=\"somepod\",namespace=\"somens\"}",
},
{
title: "namespaced metrics gauge",
info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "service_proxy_packets"},
namespace: "somens",
title: "namespaced metrics gauge",
info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "service_proxy_packets"},
namespace: "somens",
resourceNames: []string{"somesvc"},
expectedKind: GaugeSeries,
expectedKind: GaugeSeries,
expectedQuery: "service_proxy_packets{service=\"somesvc\",namespace=\"somens\"}",
},
{
title: "namespaced metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployment"}, true, "work_queue_wait"},
namespace: "somens",
title: "namespaced metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployment"}, true, "work_queue_wait"},
namespace: "somens",
resourceNames: []string{"somedep"},
expectedKind: SecondsCounterSeries,
expectedKind: SecondsCounterSeries,
expectedQuery: "work_queue_wait_seconds_total{deployment=\"somedep\",namespace=\"somens\"}",
},
// non-namespaced series
{
title: "root scoped metrics gauge",
info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_gigawatts"},
title: "root scoped metrics gauge",
info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_gigawatts"},
resourceNames: []string{"somenode"},
expectedKind: GaugeSeries,
expectedKind: GaugeSeries,
expectedQuery: "node_gigawatts{node=\"somenode\"}",
},
{
title: "root scoped metrics counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "persistentvolume"}, false, "volume_claims"},
title: "root scoped metrics counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "persistentvolume"}, false, "volume_claims"},
resourceNames: []string{"somepv"},
expectedKind: CounterSeries,
expectedKind: CounterSeries,
expectedQuery: "volume_claims_total{persistentvolume=\"somepv\"}",
},
{
title: "root scoped metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_fan"},
title: "root scoped metrics seconds counter",
info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_fan"},
resourceNames: []string{"somenode"},
expectedKind: SecondsCounterSeries,
expectedKind: SecondsCounterSeries,
expectedQuery: "node_fan_seconds_total{node=\"somenode\"}",
},
}
for _, testCase := range testCases {
outputKind, outputQuery, found := registry.QueryForMetric(testCase.info, testCase.namespace, testCase.resourceNames...)
outputKind, outputQuery, groupBy, found := registry.QueryForMetric(testCase.info, testCase.namespace, testCase.resourceNames...)
if !assert.True(found, "%s: metric %v should available", testCase.title, testCase.info) {
continue
}
assert.Equal(testCase.expectedKind, outputKind, "%s: metric %v should have had the right series type", testCase.title, testCase.info)
assert.Equal(prom.Selector(testCase.expectedQuery), outputQuery, "%s: metric %v should have produced the correct query for %v in namespace %s", testCase.title, testCase.info, testCase.resourceNames, testCase.namespace)
expectedGroupBy := testCase.expectedGroupBy
if expectedGroupBy == "" {
expectedGroupBy = testCase.info.GroupResource.Resource
}
assert.Equal(expectedGroupBy, groupBy, "%s: metric %v should have produced the correct groupBy clause", testCase.title)
}
allMetrics := registry.ListAllMetrics()
expectedMetrics := []provider.MetricInfo{
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"},
provider.MetricInfo{schema.GroupResource{Resource: "services"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "services"}, true, "service_proxy_packets"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"},
provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"},
provider.MetricInfo{schema.GroupResource{Resource: "nodes"}, false, "node_gigawatts"},
provider.MetricInfo{schema.GroupResource{Resource: "persistentvolumes"}, false, "volume_claims"},
provider.MetricInfo{schema.GroupResource{Resource: "nodes"}, false, "node_fan"},
{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
{schema.GroupResource{Resource: "pods"}, true, "some_count"},
{schema.GroupResource{Resource: "pods"}, true, "some_time"},
{schema.GroupResource{Resource: "services"}, true, "ingress_hits"},
{schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"},
{schema.GroupResource{Resource: "pods"}, true, "ingress_hits"},
{schema.GroupResource{Resource: "namespaces"}, false, "ingress_hits"},
{schema.GroupResource{Resource: "services"}, true, "service_proxy_packets"},
{schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"},
{schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"},
{schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"},
{schema.GroupResource{Resource: "nodes"}, false, "node_gigawatts"},
{schema.GroupResource{Resource: "persistentvolumes"}, false, "volume_claims"},
{schema.GroupResource{Resource: "nodes"}, false, "node_fan"},
}
// sort both for easy comparison

83
pkg/custom-provider/provider.go
View file

@ -18,27 +18,27 @@ package provider
import (
"context"
"time"
"fmt"
"net/http"
"github.com/golang/glog"
"net/http"
"time"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
pmodel "github.com/prometheus/common/model"
apierr "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime/schema"
apimeta "k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/metrics/pkg/apis/custom_metrics"
"k8s.io/apimachinery/pkg/runtime/schema"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/dynamic"
"k8s.io/client-go/pkg/api"
_ "k8s.io/client-go/pkg/api/install"
"k8s.io/apimachinery/pkg/api/resource"
"k8s.io/client-go/dynamic"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
"k8s.io/apimachinery/pkg/util/wait"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
pmodel "github.com/prometheus/common/model"
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/metrics/pkg/apis/custom_metrics"
prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
)
@ -47,9 +47,9 @@ import (
// It is similar to NewNotFound, but more specialized
func newMetricNotFoundError(resource schema.GroupResource, metricName string) *apierr.StatusError {
return &apierr.StatusError{metav1.Status{
Status: metav1.StatusFailure,
Code: int32(http.StatusNotFound),
Reason: metav1.StatusReasonNotFound,
Status: metav1.StatusFailure,
Code: int32(http.StatusNotFound),
Reason: metav1.StatusReasonNotFound,
Message: fmt.Sprintf("the server could not find the metric %s for %s", metricName, resource.String()),
}}
}
@ -58,15 +58,15 @@ func newMetricNotFoundError(resource schema.GroupResource, metricName string) *a
// the given named object. It is similar to NewNotFound, but more specialized
func newMetricNotFoundForError(resource schema.GroupResource, metricName string, resourceName string) *apierr.StatusError {
return &apierr.StatusError{metav1.Status{
Status: metav1.StatusFailure,
Code: int32(http.StatusNotFound),
Reason: metav1.StatusReasonNotFound,
Status: metav1.StatusFailure,
Code: int32(http.StatusNotFound),
Reason: metav1.StatusReasonNotFound,
Message: fmt.Sprintf("the server could not find the metric %s for %s %s", metricName, resource.String(), resourceName),
}}
}
type prometheusProvider struct {
mapper apimeta.RESTMapper
mapper apimeta.RESTMapper
kubeClient dynamic.ClientPool
promClient prom.Client
@ -78,13 +78,13 @@ type prometheusProvider struct {
func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.ClientPool, promClient prom.Client, updateInterval time.Duration, rateInterval time.Duration) provider.CustomMetricsProvider {
lister := &cachingMetricsLister{
updateInterval: updateInterval,
promClient: promClient,
promClient: promClient,
SeriesRegistry: &basicSeriesRegistry{
namer: metricNamer{
// TODO: populate this...
overrides: nil,
mapper: mapper,
mapper: mapper,
},
},
}
@ -93,7 +93,7 @@ func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.ClientP
lister.Run()
return &prometheusProvider{
mapper: mapper,
mapper: mapper,
kubeClient: kubeClient,
promClient: promClient,
@ -111,14 +111,14 @@ func (p *prometheusProvider) metricFor(value pmodel.SampleValue, groupResource s
return &custom_metrics.MetricValue{
DescribedObject: api.ObjectReference{
APIVersion: groupResource.Group+"/"+runtime.APIVersionInternal,
Kind: kind.Kind,
Name: name,
Namespace: namespace,
APIVersion: groupResource.Group + "/" + runtime.APIVersionInternal,
Kind: kind.Kind,
Name: name,
Namespace: namespace,
},
MetricName: metricName,
Timestamp: metav1.Time{time.Now()},
Value: *resource.NewMilliQuantity(int64(value * 1000.0), resource.DecimalSI),
Timestamp: metav1.Time{time.Now()},
Value: *resource.NewMilliQuantity(int64(value*1000.0), resource.DecimalSI),
}, nil
}
@ -227,7 +227,7 @@ func (p *prometheusProvider) getMultiple(info provider.MetricInfo, namespace str
// we can construct a this APIResource ourself, since the dynamic client only uses Name and Namespaced
// TODO: use discovery information instead
apiRes := &metav1.APIResource{
Name: info.GroupResource.Resource,
Name: info.GroupResource.Resource,
Namespaced: info.Namespaced,
}
@ -267,19 +267,18 @@ func (p *prometheusProvider) getMultiple(info provider.MetricInfo, namespace str
func (p *prometheusProvider) GetRootScopedMetricByName(groupResource schema.GroupResource, name string, metricName string) (*custom_metrics.MetricValue, error) {
info := provider.MetricInfo{
GroupResource: groupResource,
Metric: metricName,
Namespaced: false,
Metric: metricName,
Namespaced: false,
}
return p.getSingle(info, "", name)
}
func (p *prometheusProvider) GetRootScopedMetricBySelector(groupResource schema.GroupResource, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error) {
info := provider.MetricInfo{
GroupResource: groupResource,
Metric: metricName,
Namespaced: false,
Metric: metricName,
Namespaced: false,
}
return p.getMultiple(info, "", selector)
}
@ -287,8 +286,8 @@ func (p *prometheusProvider) GetRootScopedMetricBySelector(groupResource schema.
func (p *prometheusProvider) GetNamespacedMetricByName(groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error) {
info := provider.MetricInfo{
GroupResource: groupResource,
Metric: metricName,
Namespaced: true,
Metric: metricName,
Namespaced: true,
}
return p.getSingle(info, namespace, name)
@ -297,8 +296,8 @@ func (p *prometheusProvider) GetNamespacedMetricByName(groupResource schema.Grou
func (p *prometheusProvider) GetNamespacedMetricBySelector(groupResource schema.GroupResource, namespace string, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error) {
info := provider.MetricInfo{
GroupResource: groupResource,
Metric: metricName,
Namespaced: true,
Metric: metricName,
Namespaced: true,
}
return p.getMultiple(info, namespace, selector)
}
@ -306,12 +305,12 @@ func (p *prometheusProvider) GetNamespacedMetricBySelector(groupResource schema.
type cachingMetricsLister struct {
SeriesRegistry
promClient prom.Client
promClient prom.Client
updateInterval time.Duration
}
func (l *cachingMetricsLister) Run() {
go wait.Forever(func () {
go wait.Forever(func() {
if err := l.updateMetrics(); err != nil {
utilruntime.HandleError(err)
}
@ -319,7 +318,7 @@ func (l *cachingMetricsLister) Run() {
}
func (l *cachingMetricsLister) updateMetrics() error {
startTime := pmodel.Now().Add(-1*l.updateInterval)
startTime := pmodel.Now().Add(-1 * l.updateInterval)
// TODO: figure out a good way to add all Kubernetes-related metrics at once
// (i.e. how do we determine if something is a Kubernetes-related metric?)

68
pkg/custom-provider/provider_test.go
View file

@ -17,17 +17,18 @@ limitations under the License.
package provider
import (
"context"
"fmt"
"sort"
"time"
"testing"
"time"
fakedyn "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/pkg/api"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"
"k8s.io/apimachinery/pkg/runtime/schema"
"github.com/directxman12/custom-metrics-boilerplate/pkg/provider"
fakedyn "k8s.io/client-go/dynamic/fake"
"k8s.io/client-go/pkg/api"
// install extensions so that our RESTMapper knows about it
_ "k8s.io/client-go/pkg/apis/extensions/install"
@ -36,7 +37,7 @@ import (
pmodel "github.com/prometheus/common/model"
)
const fakeProviderUpdateInterval = 2*time.Second
const fakeProviderUpdateInterval = 2 * time.Second
// fakePromClient is a fake instance of prom.Client
type fakePromClient struct {
@ -50,7 +51,7 @@ type fakePromClient struct {
queryResults map[prom.Selector]prom.QueryResult
}
func (c *fakePromClient) Series(interval pmodel.Interval, selectors ...prom.Selector) ([]prom.Series, error) {
func (c *fakePromClient) Series(_ context.Context, interval pmodel.Interval, selectors ...prom.Selector) ([]prom.Series, error) {
if (interval.Start != 0 && interval.Start < c.acceptibleInterval.Start) || (interval.End != 0 && interval.End > c.acceptibleInterval.End) {
return nil, fmt.Errorf("interval [%v, %v] for query is outside range [%v, %v]", interval.Start, interval.End, c.acceptibleInterval.Start, c.acceptibleInterval.End)
}
@ -67,7 +68,7 @@ func (c *fakePromClient) Series(interval pmodel.Interval, selectors ...prom.Sele
return res, nil
}
func (c *fakePromClient) Query(t pmodel.Time, query prom.Selector) (prom.QueryResult, error) {
func (c *fakePromClient) Query(_ context.Context, t pmodel.Time, query prom.Selector) (prom.QueryResult, error) {
if t < c.acceptibleInterval.Start || t > c.acceptibleInterval.End {
return prom.QueryResult{}, fmt.Errorf("time %v for query is outside range [%v, %v]", t, c.acceptibleInterval.Start, c.acceptibleInterval.End)
}
@ -81,10 +82,13 @@ func (c *fakePromClient) Query(t pmodel.Time, query prom.Selector) (prom.QueryRe
}
return prom.QueryResult{
Type: pmodel.ValVector,
Type: pmodel.ValVector,
Vector: &pmodel.Vector{},
}, nil
}
func (c *fakePromClient) QueryRange(_ context.Context, r prom.Range, query prom.Selector) (prom.QueryResult, error) {
return prom.QueryResult{}, nil
}
func setupPrometheusProvider(t *testing.T) (provider.CustomMetricsProvider, *fakePromClient) {
fakeProm := &fakePromClient{}
@ -95,32 +99,32 @@ func setupPrometheusProvider(t *testing.T) (provider.CustomMetricsProvider, *fak
containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))
namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*"))
fakeProm.series = map[prom.Selector][]prom.Series{
containerSel: []prom.Series{
containerSel: {
{
Name: "container_actually_gauge_seconds_total",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_actually_gauge_seconds_total",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
{
Name: "container_some_usage",
Labels: map[string]string{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
Name: "container_some_usage",
Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"},
},
},
namespacedSel: []prom.Series{
namespacedSel: {
{
Name: "ingress_hits_total",
Labels: map[string]string{"ingress": "someingress", "service": "somesvc", "pod": "backend1", "namespace": "somens"},
Name: "ingress_hits_total",
Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend1", "namespace": "somens"},
},
{
Name: "ingress_hits_total",
Labels: map[string]string{"ingress": "someingress", "service": "somesvc", "pod": "backend2", "namespace": "somens"},
Name: "ingress_hits_total",
Labels: pmodel.LabelSet{"ingress": "someingress", "service": "somesvc", "pod": "backend2", "namespace": "somens"},
},
{
Name: "service_proxy_packets",
Labels: map[string]string{"service": "somesvc", "namespace": "somens"},
Name: "service_proxy_packets",
Labels: pmodel.LabelSet{"service": "somesvc", "namespace": "somens"},
},
{
Name: "work_queue_wait_seconds_total",
Labels: map[string]string{"deployment": "somedep", "namespace": "somens"},
Name: "work_queue_wait_seconds_total",
Labels: pmodel.LabelSet{"deployment": "somedep", "namespace": "somens"},
},
},
}
@ -148,16 +152,16 @@ func TestListAllMetrics(t *testing.T) {
sort.Sort(metricInfoSorter(actualMetrics))
expectedMetrics := []provider.MetricInfo{
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
provider.MetricInfo{schema.GroupResource{Resource: "services"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "ingress_hits"},
provider.MetricInfo{schema.GroupResource{Resource: "services"}, true, "service_proxy_packets"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"},
provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"},
provider.MetricInfo{schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"},
{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"},
{schema.GroupResource{Resource: "pods"}, true, "some_usage"},
{schema.GroupResource{Resource: "services"}, true, "ingress_hits"},
{schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"},
{schema.GroupResource{Resource: "pods"}, true, "ingress_hits"},
{schema.GroupResource{Resource: "namespaces"}, false, "ingress_hits"},
{schema.GroupResource{Resource: "services"}, true, "service_proxy_packets"},
{schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"},
{schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"},
{schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"},
}
sort.Sort(metricInfoSorter(expectedMetrics))