diff --git a/.gitignore b/.gitignore index 69d2ba91..903d1ff3 100644 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ *~ vendor _output +deploy/adapter diff --git a/.travis.yml b/.travis.yml index 2c4f066c..751cf45f 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,17 +1,14 @@ language: go go: -- 1.8 +- '1.10' # blech, Travis downloads with capitals in DirectXMan12, which confuses go go_import_path: github.com/directxman12/k8s-prometheus-adapter -addons: - apt: - sources: - - sourceline: 'ppa:masterminds/glide' - packages: - - glide +before_install: +- curl -L -s https://github.com/golang/dep/releases/download/v0.4.1/dep-linux-amd64 -o $GOPATH/bin/dep +- chmod +x $GOPATH/bin/dep install: - make -B vendor @@ -20,7 +17,7 @@ script: make verify cache: directories: - - ~/.glide + - ~/gopath/pkg/dep/ sudo: required services: diff --git a/Gopkg.lock b/Gopkg.lock new file mode 100644 index 00000000..0d38a772 --- /dev/null +++ b/Gopkg.lock @@ -0,0 +1,807 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + branch = "default" + name = "bitbucket.org/ww/goautoneg" + packages = ["."] + revision = "75cd24fc2f2c2a2088577d12123ddee5f54e0675" + +[[projects]] + name = "github.com/NYTimes/gziphandler" + packages = ["."] + revision = "2600fb119af974220d3916a5916d6e31176aac1b" + version = "v1.0.1" + +[[projects]] + name = "github.com/PuerkitoBio/purell" + packages = ["."] + revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/PuerkitoBio/urlesc" + packages = ["."] + revision = "de5bf2ad457846296e2031421a34e2568e304e35" + +[[projects]] + branch = "master" + name = "github.com/beorn7/perks" + packages = ["quantile"] + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" + +[[projects]] + name = "github.com/coreos/etcd" + packages = [ + "auth/authpb", + "client", + "clientv3", + "etcdserver/api/v3rpc/rpctypes", + "etcdserver/etcdserverpb", + "mvcc/mvccpb", + "pkg/pathutil", + "pkg/srv", + "pkg/tlsutil", + "pkg/transport", + "pkg/types", + "version" + ] + revision = "33245c6b5b49130ca99280408fadfab01aac0e48" + version = "v3.3.8" + +[[projects]] + name = "github.com/coreos/go-semver" + packages = ["semver"] + revision = "8ab6407b697782a06568d4b7f1db25550ec2e4c6" + version = "v0.2.0" + +[[projects]] + name = "github.com/coreos/go-systemd" + packages = ["daemon"] + revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab" + version = "v17" + +[[projects]] + name = "github.com/davecgh/go-spew" + packages = ["spew"] + revision = "346938d642f2ec3594ed81d874461961cd0faa76" + version = "v1.1.0" + +[[projects]] + name = "github.com/elazarl/go-bindata-assetfs" + packages = ["."] + revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43" + version = "v1.0.0" + +[[projects]] + name = "github.com/emicklei/go-restful" + packages = [ + ".", + "log" + ] + revision = "3658237ded108b4134956c1b3050349d93e7b895" + version = "v2.7.1" + +[[projects]] + name = "github.com/emicklei/go-restful-swagger12" + packages = ["."] + revision = "dcef7f55730566d41eae5db10e7d6981829720f6" + version = "1.0.1" + +[[projects]] + name = "github.com/evanphx/json-patch" + packages = ["."] + revision = "afac545df32f2287a079e2dfb7ba2745a643747e" + version = "v3.0.0" + +[[projects]] + name = "github.com/ghodss/yaml" + packages = ["."] + revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonpointer" + packages = ["."] + revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/jsonreference" + packages = ["."] + revision = "3fb327e6747da3043567ee86abd02bb6376b6be2" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/spec" + packages = ["."] + revision = "bcff419492eeeb01f76e77d2ebc714dc97b607f5" + +[[projects]] + branch = "master" + name = "github.com/go-openapi/swag" + packages = ["."] + revision = "811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5" + +[[projects]] + name = "github.com/gogo/protobuf" + packages = [ + "gogoproto", + "proto", + "protoc-gen-gogo/descriptor", + "sortkeys" + ] + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/golang/glog" + packages = ["."] + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" + +[[projects]] + name = "github.com/golang/protobuf" + packages = [ + "proto", + "ptypes", + "ptypes/any", + "ptypes/duration", + "ptypes/timestamp" + ] + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" + +[[projects]] + branch = "master" + name = "github.com/google/btree" + packages = ["."] + revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" + +[[projects]] + branch = "master" + name = "github.com/google/gofuzz" + packages = ["."] + revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" + +[[projects]] + name = "github.com/googleapis/gnostic" + packages = [ + "OpenAPIv2", + "compiler", + "extensions" + ] + revision = "7c663266750e7d82587642f65e60bc4083f1f84e" + version = "v0.2.0" + +[[projects]] + branch = "master" + name = "github.com/gregjones/httpcache" + packages = [ + ".", + "diskcache" + ] + revision = "9cad4c3443a7200dd6400aef47183728de563a38" + +[[projects]] + branch = "master" + name = "github.com/hashicorp/golang-lru" + packages = [ + ".", + "simplelru" + ] + revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" + +[[projects]] + name = "github.com/imdario/mergo" + packages = ["."] + revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" + version = "v0.3.5" + +[[projects]] + name = "github.com/inconshreveable/mousetrap" + packages = ["."] + revision = "76626ae9c91c4f2a10f34cad8ce83ea42c93bb75" + version = "v1.0" + +[[projects]] + name = "github.com/json-iterator/go" + packages = ["."] + revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4" + version = "1.1.3" + +[[projects]] + branch = "master" + name = "github.com/kubernetes-incubator/custom-metrics-apiserver" + packages = [ + "pkg/apiserver", + "pkg/apiserver/installer", + "pkg/cmd/server", + "pkg/dynamicmapper", + "pkg/provider", + "pkg/registry/custom_metrics", + "pkg/registry/external_metrics" + ] + revision = "d8f23423aa1d0ff2bc9656da863d721725b3c68a" + +[[projects]] + branch = "master" + name = "github.com/mailru/easyjson" + packages = [ + "buffer", + "jlexer", + "jwriter" + ] + revision = "3fdea8d05856a0c8df22ed4bc71b3219245e4485" + +[[projects]] + name = "github.com/matttproud/golang_protobuf_extensions" + packages = ["pbutil"] + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" + +[[projects]] + name = "github.com/modern-go/concurrent" + packages = ["."] + revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" + version = "1.0.3" + +[[projects]] + name = "github.com/modern-go/reflect2" + packages = ["."] + revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f" + version = "1.0.0" + +[[projects]] + name = "github.com/pborman/uuid" + packages = ["."] + revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" + version = "v1.1" + +[[projects]] + branch = "master" + name = "github.com/petar/GoLLRB" + packages = ["llrb"] + revision = "53be0d36a84c2a886ca057d34b6aa4468df9ccb4" + +[[projects]] + name = "github.com/peterbourgon/diskv" + packages = ["."] + revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" + version = "v2.0.1" + +[[projects]] + name = "github.com/pmezard/go-difflib" + packages = ["difflib"] + revision = "792786c7400a136282c1664665ae0a8db921c6c2" + version = "v1.0.0" + +[[projects]] + name = "github.com/prometheus/client_golang" + packages = ["prometheus"] + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" + +[[projects]] + branch = "master" + name = "github.com/prometheus/client_model" + packages = ["go"] + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" + +[[projects]] + branch = "master" + name = "github.com/prometheus/common" + packages = [ + "expfmt", + "internal/bitbucket.org/ww/goautoneg", + "model" + ] + revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" + +[[projects]] + branch = "master" + name = "github.com/prometheus/procfs" + packages = [ + ".", + "internal/util", + "nfs", + "xfs" + ] + revision = "7d6f385de8bea29190f15ba9931442a0eaef9af7" + +[[projects]] + name = "github.com/spf13/cobra" + packages = ["."] + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" + +[[projects]] + name = "github.com/spf13/pflag" + packages = ["."] + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" + +[[projects]] + name = "github.com/stretchr/testify" + packages = [ + "assert", + "require" + ] + revision = "f35b8ab0b5a2cef36673838d662e249dd9c94686" + version = "v1.2.2" + +[[projects]] + name = "github.com/ugorji/go" + packages = ["codec"] + revision = "b4c50a2b199d93b13dc15e78929cfb23bfdf21ab" + version = "v1.1.1" + +[[projects]] + branch = "master" + name = "golang.org/x/crypto" + packages = ["ssh/terminal"] + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" + +[[projects]] + branch = "master" + name = "golang.org/x/net" + packages = [ + "context", + "http/httpguts", + "http2", + "http2/hpack", + "idna", + "internal/timeseries", + "trace", + "websocket" + ] + revision = "afe8f62b1d6bbd81f31868121a50b06d8188e1f9" + +[[projects]] + branch = "master" + name = "golang.org/x/sys" + packages = [ + "unix", + "windows" + ] + revision = "63fc586f45fe72d95d5240a5d5eb95e6503907d3" + +[[projects]] + name = "golang.org/x/text" + packages = [ + "collate", + "collate/build", + "internal/colltab", + "internal/gen", + "internal/tag", + "internal/triegen", + "internal/ucd", + "language", + "secure/bidirule", + "transform", + "unicode/bidi", + "unicode/cldr", + "unicode/norm", + "unicode/rangetable", + "width" + ] + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" + +[[projects]] + branch = "master" + name = "golang.org/x/time" + packages = ["rate"] + revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" + +[[projects]] + branch = "master" + name = "google.golang.org/genproto" + packages = ["googleapis/rpc/status"] + revision = "80063a038e333bbe006c878e4c5ce4c74d055498" + +[[projects]] + name = "google.golang.org/grpc" + packages = [ + ".", + "balancer", + "balancer/base", + "balancer/roundrobin", + "codes", + "connectivity", + "credentials", + "encoding", + "encoding/proto", + "grpclog", + "health/grpc_health_v1", + "internal", + "internal/backoff", + "internal/channelz", + "internal/grpcrand", + "keepalive", + "metadata", + "naming", + "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", + "stats", + "status", + "tap", + "transport" + ] + revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + version = "v1.13.0" + +[[projects]] + name = "gopkg.in/inf.v0" + packages = ["."] + revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" + version = "v0.9.1" + +[[projects]] + name = "gopkg.in/natefinch/lumberjack.v2" + packages = ["."] + revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" + version = "v2.1" + +[[projects]] + name = "gopkg.in/yaml.v2" + packages = ["."] + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" + +[[projects]] + name = "k8s.io/api" + packages = [ + "admission/v1beta1", + "admissionregistration/v1alpha1", + "admissionregistration/v1beta1", + "apps/v1", + "apps/v1beta1", + "apps/v1beta2", + "authentication/v1", + "authentication/v1beta1", + "authorization/v1", + "authorization/v1beta1", + "autoscaling/v1", + "autoscaling/v2beta1", + "batch/v1", + "batch/v1beta1", + "batch/v2alpha1", + "certificates/v1beta1", + "core/v1", + "events/v1beta1", + "extensions/v1beta1", + "networking/v1", + "policy/v1beta1", + "rbac/v1", + "rbac/v1alpha1", + "rbac/v1beta1", + "scheduling/v1alpha1", + "scheduling/v1beta1", + "settings/v1alpha1", + "storage/v1", + "storage/v1alpha1", + "storage/v1beta1" + ] + revision = "91b2d7a92a8930454bf5020e0595b8ea0f2a5047" + version = "kubernetes-1.11.0-rc.1" + +[[projects]] + name = "k8s.io/apimachinery" + packages = [ + "pkg/api/equality", + "pkg/api/errors", + "pkg/api/meta", + "pkg/api/resource", + "pkg/api/validation", + "pkg/api/validation/path", + "pkg/apis/meta/internalversion", + "pkg/apis/meta/v1", + "pkg/apis/meta/v1/unstructured", + "pkg/apis/meta/v1/validation", + "pkg/apis/meta/v1beta1", + "pkg/conversion", + "pkg/conversion/queryparams", + "pkg/fields", + "pkg/labels", + "pkg/runtime", + "pkg/runtime/schema", + "pkg/runtime/serializer", + "pkg/runtime/serializer/json", + "pkg/runtime/serializer/protobuf", + "pkg/runtime/serializer/recognizer", + "pkg/runtime/serializer/streaming", + "pkg/runtime/serializer/versioning", + "pkg/selection", + "pkg/types", + "pkg/util/cache", + "pkg/util/clock", + "pkg/util/diff", + "pkg/util/errors", + "pkg/util/framer", + "pkg/util/intstr", + "pkg/util/json", + "pkg/util/mergepatch", + "pkg/util/net", + "pkg/util/rand", + "pkg/util/runtime", + "pkg/util/sets", + "pkg/util/strategicpatch", + "pkg/util/uuid", + "pkg/util/validation", + "pkg/util/validation/field", + "pkg/util/wait", + "pkg/util/waitgroup", + "pkg/util/yaml", + "pkg/version", + "pkg/watch", + "third_party/forked/golang/json", + "third_party/forked/golang/reflect" + ] + revision = "fda675fbe85280c4550452dae2a5ebf74e4a59b7" + version = "kubernetes-1.11.0-rc.1" + +[[projects]] + name = "k8s.io/apiserver" + packages = [ + "pkg/admission", + "pkg/admission/configuration", + "pkg/admission/initializer", + "pkg/admission/metrics", + "pkg/admission/plugin/initialization", + "pkg/admission/plugin/namespace/lifecycle", + "pkg/admission/plugin/webhook/config", + "pkg/admission/plugin/webhook/config/apis/webhookadmission", + "pkg/admission/plugin/webhook/config/apis/webhookadmission/v1alpha1", + "pkg/admission/plugin/webhook/errors", + "pkg/admission/plugin/webhook/generic", + "pkg/admission/plugin/webhook/mutating", + "pkg/admission/plugin/webhook/namespace", + "pkg/admission/plugin/webhook/request", + "pkg/admission/plugin/webhook/rules", + "pkg/admission/plugin/webhook/validating", + "pkg/apis/apiserver", + "pkg/apis/apiserver/install", + "pkg/apis/apiserver/v1alpha1", + "pkg/apis/audit", + "pkg/apis/audit/install", + "pkg/apis/audit/v1alpha1", + "pkg/apis/audit/v1beta1", + "pkg/apis/audit/validation", + "pkg/audit", + "pkg/audit/policy", + "pkg/authentication/authenticator", + "pkg/authentication/authenticatorfactory", + "pkg/authentication/group", + "pkg/authentication/request/anonymous", + "pkg/authentication/request/bearertoken", + "pkg/authentication/request/headerrequest", + "pkg/authentication/request/union", + "pkg/authentication/request/websocket", + "pkg/authentication/request/x509", + "pkg/authentication/serviceaccount", + "pkg/authentication/token/tokenfile", + "pkg/authentication/user", + "pkg/authorization/authorizer", + "pkg/authorization/authorizerfactory", + "pkg/authorization/union", + "pkg/endpoints", + "pkg/endpoints/discovery", + "pkg/endpoints/filters", + "pkg/endpoints/handlers", + "pkg/endpoints/handlers/negotiation", + "pkg/endpoints/handlers/responsewriters", + "pkg/endpoints/metrics", + "pkg/endpoints/openapi", + "pkg/endpoints/request", + "pkg/features", + "pkg/registry/generic", + "pkg/registry/generic/registry", + "pkg/registry/rest", + "pkg/server", + "pkg/server/filters", + "pkg/server/healthz", + "pkg/server/httplog", + "pkg/server/mux", + "pkg/server/options", + "pkg/server/resourceconfig", + "pkg/server/routes", + "pkg/server/routes/data/swagger", + "pkg/server/storage", + "pkg/storage", + "pkg/storage/errors", + "pkg/storage/etcd", + "pkg/storage/etcd/metrics", + "pkg/storage/etcd/util", + "pkg/storage/etcd3", + "pkg/storage/etcd3/preflight", + "pkg/storage/names", + "pkg/storage/storagebackend", + "pkg/storage/storagebackend/factory", + "pkg/storage/value", + "pkg/util/feature", + "pkg/util/flag", + "pkg/util/flushwriter", + "pkg/util/logs", + "pkg/util/openapi", + "pkg/util/trace", + "pkg/util/webhook", + "pkg/util/wsstream", + "plugin/pkg/audit/buffered", + "plugin/pkg/audit/log", + "plugin/pkg/audit/truncate", + "plugin/pkg/audit/webhook", + "plugin/pkg/authenticator/token/webhook", + "plugin/pkg/authorizer/webhook" + ] + revision = "44b612291bb7545430c499a3882c610c727f37b0" + version = "kubernetes-1.11.0-rc.1" + +[[projects]] + name = "k8s.io/client-go" + packages = [ + "discovery", + "dynamic", + "dynamic/fake", + "informers", + "informers/admissionregistration", + "informers/admissionregistration/v1alpha1", + "informers/admissionregistration/v1beta1", + "informers/apps", + "informers/apps/v1", + "informers/apps/v1beta1", + "informers/apps/v1beta2", + "informers/autoscaling", + "informers/autoscaling/v1", + "informers/autoscaling/v2beta1", + "informers/batch", + "informers/batch/v1", + "informers/batch/v1beta1", + "informers/batch/v2alpha1", + "informers/certificates", + "informers/certificates/v1beta1", + "informers/core", + "informers/core/v1", + "informers/events", + "informers/events/v1beta1", + "informers/extensions", + "informers/extensions/v1beta1", + "informers/internalinterfaces", + "informers/networking", + "informers/networking/v1", + "informers/policy", + "informers/policy/v1beta1", + "informers/rbac", + "informers/rbac/v1", + "informers/rbac/v1alpha1", + "informers/rbac/v1beta1", + "informers/scheduling", + "informers/scheduling/v1alpha1", + "informers/scheduling/v1beta1", + "informers/settings", + "informers/settings/v1alpha1", + "informers/storage", + "informers/storage/v1", + "informers/storage/v1alpha1", + "informers/storage/v1beta1", + "kubernetes", + "kubernetes/scheme", + "kubernetes/typed/admissionregistration/v1alpha1", + "kubernetes/typed/admissionregistration/v1beta1", + "kubernetes/typed/apps/v1", + "kubernetes/typed/apps/v1beta1", + "kubernetes/typed/apps/v1beta2", + "kubernetes/typed/authentication/v1", + "kubernetes/typed/authentication/v1beta1", + "kubernetes/typed/authorization/v1", + "kubernetes/typed/authorization/v1beta1", + "kubernetes/typed/autoscaling/v1", + "kubernetes/typed/autoscaling/v2beta1", + "kubernetes/typed/batch/v1", + "kubernetes/typed/batch/v1beta1", + "kubernetes/typed/batch/v2alpha1", + "kubernetes/typed/certificates/v1beta1", + "kubernetes/typed/core/v1", + "kubernetes/typed/events/v1beta1", + "kubernetes/typed/extensions/v1beta1", + "kubernetes/typed/networking/v1", + "kubernetes/typed/policy/v1beta1", + "kubernetes/typed/rbac/v1", + "kubernetes/typed/rbac/v1alpha1", + "kubernetes/typed/rbac/v1beta1", + "kubernetes/typed/scheduling/v1alpha1", + "kubernetes/typed/scheduling/v1beta1", + "kubernetes/typed/settings/v1alpha1", + "kubernetes/typed/storage/v1", + "kubernetes/typed/storage/v1alpha1", + "kubernetes/typed/storage/v1beta1", + "listers/admissionregistration/v1alpha1", + "listers/admissionregistration/v1beta1", + "listers/apps/v1", + "listers/apps/v1beta1", + "listers/apps/v1beta2", + "listers/autoscaling/v1", + "listers/autoscaling/v2beta1", + "listers/batch/v1", + "listers/batch/v1beta1", + "listers/batch/v2alpha1", + "listers/certificates/v1beta1", + "listers/core/v1", + "listers/events/v1beta1", + "listers/extensions/v1beta1", + "listers/networking/v1", + "listers/policy/v1beta1", + "listers/rbac/v1", + "listers/rbac/v1alpha1", + "listers/rbac/v1beta1", + "listers/scheduling/v1alpha1", + "listers/scheduling/v1beta1", + "listers/settings/v1alpha1", + "listers/storage/v1", + "listers/storage/v1alpha1", + "listers/storage/v1beta1", + "pkg/apis/clientauthentication", + "pkg/apis/clientauthentication/v1alpha1", + "pkg/apis/clientauthentication/v1beta1", + "pkg/version", + "plugin/pkg/client/auth/exec", + "rest", + "rest/watch", + "restmapper", + "testing", + "tools/auth", + "tools/cache", + "tools/clientcmd", + "tools/clientcmd/api", + "tools/clientcmd/api/latest", + "tools/clientcmd/api/v1", + "tools/metrics", + "tools/pager", + "tools/reference", + "transport", + "util/buffer", + "util/cert", + "util/connrotation", + "util/flowcontrol", + "util/homedir", + "util/integer", + "util/retry" + ] + revision = "4cacfee698b01630072bc41e3384280562a97d95" + version = "kubernetes-1.11.0-rc.1" + +[[projects]] + branch = "master" + name = "k8s.io/kube-openapi" + packages = [ + "pkg/builder", + "pkg/common", + "pkg/handler", + "pkg/util", + "pkg/util/proto" + ] + revision = "91cfa479c814065e420cee7ed227db0f63a5854e" + +[[projects]] + name = "k8s.io/metrics" + packages = [ + "pkg/apis/custom_metrics", + "pkg/apis/custom_metrics/install", + "pkg/apis/custom_metrics/v1beta1", + "pkg/apis/external_metrics", + "pkg/apis/external_metrics/install", + "pkg/apis/external_metrics/v1beta1" + ] + revision = "89f8a18a5efb0c0162a32c75db752bc53ed7f8ee" + version = "kubernetes-1.11.0-rc.1" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "922da691d7be0fa3bde2ab628c629fea6718792cb234a2e5c661a193f0545d6f" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml new file mode 100644 index 00000000..3239762d --- /dev/null +++ b/Gopkg.toml @@ -0,0 +1,82 @@ +# Gopkg.toml example +# +# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md +# for detailed Gopkg.toml documentation. +# +# required = ["github.com/user/thing/cmd/thing"] +# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] +# +# [[constraint]] +# name = "github.com/user/project" +# version = "1.0.0" +# +# [[constraint]] +# name = "github.com/user/project2" +# branch = "dev" +# source = "github.com/myfork/project2" +# +# [[override]] +# name = "github.com/x/y" +# version = "2.4.0" +# +# [prune] +# non-go = false +# go-tests = true +# unused-packages = true + + +# Utility library deps +[[constraint]] + branch = "master" + name = "github.com/golang/glog" + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + branch = "master" + name = "github.com/prometheus/common" + +[[constraint]] + name = "github.com/spf13/cobra" + version = "0.0.3" + +[[constraint]] + name = "gopkg.in/yaml.v2" + version = "2.2.1" + +# Kubernetes incubator deps +[[constraint]] + version = "kubernetes-1.11.0-rc.1" + name = "github.com/kubernetes-incubator/custom-metrics-apiserver" + +# Core Kubernetes deps +[[constraint]] + name = "k8s.io/api" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/apimachinery" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/apiserver" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/client-go" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/metrics" + version = "kubernetes-1.11.0-rc.1" + +# Test deps +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.2" + +[prune] + go-tests = true + unused-packages = true diff --git a/Makefile b/Makefile index 20d5e122..0384ac25 100644 --- a/Makefile +++ b/Makefile @@ -8,7 +8,7 @@ OUT_DIR?=./_output VENDOR_DOCKERIZED=0 VERSION?=latest -GOIMAGE=golang:1.8 +GOIMAGE=golang:1.10 ifeq ($(ARCH),amd64) BASEIMAGE?=busybox @@ -24,25 +24,34 @@ ifeq ($(ARCH),ppc64le) endif ifeq ($(ARCH),s390x) BASEIMAGE?=s390x/busybox - GOIMAGE=s390x/golang:1.8 + GOIMAGE=s390x/golang:1.10 endif -.PHONY: all build docker-build push-% push test verify-gofmt gofmt verify +.PHONY: all docker-build push-% push test verify-gofmt gofmt verify build-local-image -all: build -build: vendor - CGO_ENABLED=0 GOARCH=$(ARCH) go build -a -tags netgo -o $(OUT_DIR)/$(ARCH)/adapter github.com/directxman12/k8s-prometheus-adapter/cmd/adapter +all: $(OUT_DIR)/$(ARCH)/adapter +src_deps=$(shell find pkg cmd -type f -name "*.go") +$(OUT_DIR)/%/adapter: vendor $(src_deps) + CGO_ENABLED=0 GOARCH=$* go build -tags netgo -o $(OUT_DIR)/$(ARCH)/adapter github.com/directxman12/k8s-prometheus-adapter/cmd/adapter + docker-build: vendor cp deploy/Dockerfile $(TEMP_DIR) cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|$(BASEIMAGE)|g" Dockerfile docker run -it -v $(TEMP_DIR):/build -v $(shell pwd):/go/src/github.com/directxman12/k8s-prometheus-adapter -e GOARCH=$(ARCH) $(GOIMAGE) /bin/bash -c "\ - CGO_ENABLED=0 go build -a -tags netgo -o /build/adapter github.com/directxman12/k8s-prometheus-adapter/cmd/adapter" + CGO_ENABLED=0 go build -tags netgo -o /build/adapter github.com/directxman12/k8s-prometheus-adapter/cmd/adapter" docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR) rm -rf $(TEMP_DIR) +build-local-image: $(OUT_DIR)/$(ARCH)/adapter + cp deploy/Dockerfile $(TEMP_DIR) + cp $(OUT_DIR)/$(ARCH)/adapter $(TEMP_DIR) + cd $(TEMP_DIR) && sed -i "s|BASEIMAGE|scratch|g" Dockerfile + docker build -t $(REGISTRY)/$(IMAGE)-$(ARCH):$(VERSION) $(TEMP_DIR) + rm -rf $(TEMP_DIR) + push-%: $(MAKE) ARCH=$* docker-build docker push $(REGISTRY)/$(IMAGE)-$*:$(VERSION) @@ -54,13 +63,13 @@ push: ./manifest-tool $(addprefix push-,$(ALL_ARCH)) curl -sSL https://github.com/estesp/manifest-tool/releases/download/v0.5.0/manifest-tool-linux-amd64 > manifest-tool chmod +x manifest-tool -vendor: glide.lock +vendor: Gopkg.lock ifeq ($(VENDOR_DOCKERIZED),1) - docker run -it -v $(shell pwd):/go/src/github.com/directxman12/k8s-prometheus-adapter -w /go/src/github.com/directxman12/k8s-prometheus-adapter golang:1.8 /bin/bash -c "\ - curl https://glide.sh/get | sh \ - && glide install -v" + docker run -it -v $(shell pwd):/go/src/github.com/directxman12/k8s-prometheus-adapter -w /go/src/github.com/directxman12/k8s-prometheus-adapter golang:1.10 /bin/bash -c "\ + curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh \ + && dep ensure -vendor-only" else - glide install -v + dep ensure -vendor-only endif test: vendor diff --git a/README.md b/README.md index d58421e1..73ac7150 100644 --- a/README.md +++ b/README.md @@ -29,13 +29,14 @@ adapter talks to Prometheus and the main Kubernetes cluster: - `--metrics-relist-interval=`: This is the interval at which to update the cache of available metrics from Prometheus. -- `--rate-interval=`: This is the duration used when requesting - rate metrics from Prometheus. It *must* be larger than your Prometheus - collection interval. - - `--prometheus-url=`: This is the URL used to connect to Prometheus. It will eventually contain query parameters to configure the connection. +- `--config=` (`-c`): This configures how the adapter discovers available + Prometheus metrics and the associated Kubernetes resources, and how it presents those + metrics in the custom metrics API. More information about this file can be found in + [docs/config.md](docs/config.md). + Presentation ------------ @@ -43,18 +44,14 @@ The adapter gathers the names of available metrics from Prometheus a regular interval (see [Configuration](#configuration) above), and then only exposes metrics that follow specific forms. -In general: +The rules governing this discovery are specified in a [configuration file](docs/config.md). +If you were relying on the implicit rules from the previous version of the adapter, +you can use the included `config-gen` tool to generate a configuration that matches +the old implicit ruleset: -- Metrics must have the `namespace` label to be considered. - -- For each label on a metric, if that label name corresponds to - a Kubernetes resource (like `pod` or `service`), the metric will be - associated with that resource. - -- Metrics ending in `_total` are assumed to be cumulative, and will be - exposed without the suffix as a rate metric. - -Detailed information can be found under [docs/format.md](docs/format.md). +```shell +$ go run cmd/config-gen main.go [--rate-interval=] [--label-prefix=] +``` Example ------- @@ -65,7 +62,8 @@ Additionally, [@luxas](https://github.com/luxas) has an excellent example deployment of Prometheus, this adapter, and a demo pod which serves a metric `http_requests_total`, which becomes the custom metrics API metric `pods/http_requests`. It also autoscales on that metric using the -`autoscaling/v2beta1` HorizontalPodAutoscaler. +`autoscaling/v2beta1` HorizontalPodAutoscaler. Note that @luxas's tutorial +uses a slightly older version of the adapter. It can be found at https://github.com/luxas/kubeadm-workshop. Pay special attention to: diff --git a/cmd/adapter/app/start.go b/cmd/adapter/app/start.go index 7c7a8b60..9859365d 100644 --- a/cmd/adapter/app/start.go +++ b/cmd/adapter/app/start.go @@ -24,7 +24,6 @@ import ( "time" "github.com/spf13/cobra" - apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/discovery" "k8s.io/client-go/dynamic" "k8s.io/client-go/rest" @@ -32,6 +31,7 @@ import ( prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" mprom "github.com/directxman12/k8s-prometheus-adapter/pkg/client/metrics" + adaptercfg "github.com/directxman12/k8s-prometheus-adapter/pkg/config" cmprov "github.com/directxman12/k8s-prometheus-adapter/pkg/custom-provider" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/cmd/server" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/dynamicmapper" @@ -43,9 +43,7 @@ func NewCommandStartPrometheusAdapterServer(out, errOut io.Writer, stopCh <-chan o := PrometheusAdapterServerOptions{ CustomMetricsAdapterServerOptions: baseOpts, MetricsRelistInterval: 10 * time.Minute, - RateInterval: 5 * time.Minute, PrometheusURL: "https://localhost", - DiscoveryInterval: 10 * time.Minute, } cmd := &cobra.Command{ @@ -76,19 +74,20 @@ func NewCommandStartPrometheusAdapterServer(out, errOut io.Writer, stopCh <-chan "any described objets") flags.DurationVar(&o.MetricsRelistInterval, "metrics-relist-interval", o.MetricsRelistInterval, ""+ "interval at which to re-list the set of all available metrics from Prometheus") - flags.DurationVar(&o.RateInterval, "rate-interval", o.RateInterval, ""+ - "period of time used to calculate rate metrics from cumulative metrics") flags.DurationVar(&o.DiscoveryInterval, "discovery-interval", o.DiscoveryInterval, ""+ "interval at which to refresh API discovery information") flags.StringVar(&o.PrometheusURL, "prometheus-url", o.PrometheusURL, - "URL for connecting to Prometheus. Query parameters are used to configure the connection") + "URL for connecting to Prometheus.") flags.BoolVar(&o.PrometheusAuthInCluster, "prometheus-auth-incluster", o.PrometheusAuthInCluster, "use auth details from the in-cluster kubeconfig when connecting to prometheus.") flags.StringVar(&o.PrometheusAuthConf, "prometheus-auth-config", o.PrometheusAuthConf, "kubeconfig file used to configure auth when connecting to Prometheus.") - flags.StringVar(&o.LabelPrefix, "label-prefix", o.LabelPrefix, - "Prefix to expect on labels referring to pod resources. For example, if the prefix is "+ - "'kube_', any series with the 'kube_pod' label would be considered a pod metric") + flags.StringVar(&o.AdapterConfigFile, "config", o.AdapterConfigFile, + "Configuration file containing details of how to transform between Prometheus metrics "+ + "and custom metrics API resources") + + cmd.MarkFlagRequired("config") + return cmd } @@ -128,6 +127,15 @@ func makeHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, er } func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct{}) error { + if o.AdapterConfigFile == "" { + return fmt.Errorf("no discovery configuration file specified") + } + + metricsConfig, err := adaptercfg.FromFile(o.AdapterConfigFile) + if err != nil { + return fmt.Errorf("unable to load metrics discovery configuration: %v", err) + } + config, err := o.Config() if err != nil { return err @@ -153,12 +161,12 @@ func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-c return fmt.Errorf("unable to construct discovery client for dynamic client: %v", err) } - dynamicMapper, err := dynamicmapper.NewRESTMapper(discoveryClient, apimeta.InterfacesForUnstructured, o.DiscoveryInterval) + dynamicMapper, err := dynamicmapper.NewRESTMapper(discoveryClient, o.DiscoveryInterval) if err != nil { return fmt.Errorf("unable to construct dynamic discovery mapper: %v", err) } - clientPool := dynamic.NewClientPool(clientConfig, dynamicMapper, dynamic.LegacyAPIPathResolverFunc) + dynamicClient, err := dynamic.NewForConfig(clientConfig) if err != nil { return fmt.Errorf("unable to construct lister client to initialize provider: %v", err) } @@ -176,9 +184,15 @@ func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-c instrumentedGenericPromClient := mprom.InstrumentGenericAPIClient(genericPromClient, baseURL.String()) promClient := prom.NewClientForAPI(instrumentedGenericPromClient) - cmProvider := cmprov.NewPrometheusProvider(dynamicMapper, clientPool, promClient, o.LabelPrefix, o.MetricsRelistInterval, o.RateInterval, stopCh) + namers, err := cmprov.NamersFromConfig(metricsConfig, dynamicMapper) + if err != nil { + return fmt.Errorf("unable to construct naming scheme from metrics rules: %v", err) + } - server, err := config.Complete().New("prometheus-custom-metrics-adapter", cmProvider) + cmProvider, runner := cmprov.NewPrometheusProvider(dynamicMapper, dynamicClient, promClient, namers, o.MetricsRelistInterval) + runner.RunUntil(stopCh) + + server, err := config.Complete().New("prometheus-custom-metrics-adapter", cmProvider, nil) if err != nil { return err } @@ -192,8 +206,6 @@ type PrometheusAdapterServerOptions struct { RemoteKubeConfigFile string // MetricsRelistInterval is the interval at which to relist the set of available metrics MetricsRelistInterval time.Duration - // RateInterval is the period of time used to calculate rate metrics - RateInterval time.Duration // DiscoveryInterval is the interval at which discovery information is refreshed DiscoveryInterval time.Duration // PrometheusURL is the URL describing how to connect to Prometheus. Query parameters configure connection options. @@ -202,7 +214,6 @@ type PrometheusAdapterServerOptions struct { PrometheusAuthInCluster bool // PrometheusAuthConf is the kubeconfig file that contains auth details used to connect to Prometheus PrometheusAuthConf string - // LabelPrefix is the prefix to expect on labels for Kubernetes resources - // (e.g. if the prefix is "kube_", we'd expect a "kube_pod" label for pod metrics). - LabelPrefix string + // AdapterConfigFile points to the file containing the metrics discovery configuration. + AdapterConfigFile string } diff --git a/cmd/config-gen/main.go b/cmd/config-gen/main.go new file mode 100644 index 00000000..b3d8f7e1 --- /dev/null +++ b/cmd/config-gen/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + yaml "gopkg.in/yaml.v2" + + "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" +) + +func main() { + var labelPrefix string + var rateInterval time.Duration + + cmd := &cobra.Command{ + Short: "Generate a config matching the legacy discovery rules", + Long: `Generate a config that produces the same functionality +as the legacy discovery rules. This includes discovering metrics and associating +resources according to the Kubernetes instrumention conventions and the cAdvisor +conventions, and auto-converting cumulative metrics into rate metrics.`, + RunE: func(c *cobra.Command, args []string) error { + cfg := utils.DefaultConfig(rateInterval, labelPrefix) + enc := yaml.NewEncoder(os.Stdout) + if err := enc.Encode(cfg); err != nil { + return err + } + return enc.Close() + }, + } + + cmd.Flags().StringVar(&labelPrefix, "label-prefix", "", + "Prefix to expect on labels referring to pod resources. For example, if the prefix is "+ + "'kube_', any series with the 'kube_pod' label would be considered a pod metric") + cmd.Flags().DurationVar(&rateInterval, "rate-interval", 5*time.Minute, + "Period of time used to calculate rate metrics from cumulative metrics") + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Unable to generate config: %v\n", err) + os.Exit(1) + } +} diff --git a/cmd/config-gen/utils/default.go b/cmd/config-gen/utils/default.go new file mode 100644 index 00000000..d8873d96 --- /dev/null +++ b/cmd/config-gen/utils/default.go @@ -0,0 +1,93 @@ +package utils + +import ( + "fmt" + "time" + + prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + . "github.com/directxman12/k8s-prometheus-adapter/pkg/config" + pmodel "github.com/prometheus/common/model" +) + +// DefaultConfig returns a configuration equivalent to the former +// pre-advanced-config settings. This means that "normal" series labels +// will be of the form `<<.Resource>>`, cadvisor series will be +// of the form `container_`, and have the label `pod_name`. Any series ending +// in total will be treated as a rate metric. +func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDiscoveryConfig { + return &MetricsDiscoveryConfig{ + Rules: []DiscoveryRule{ + // container seconds rate metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))), + Resources: ResourceMapping{ + Overrides: map[string]GroupResource{ + "namespace": {Resource: "namespace"}, + "pod_name": {Resource: "pod"}, + }, + }, + Name: NameMapping{Matches: "^container_(.*)_seconds_total$"}, + MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()), + }, + + // container rate metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))), + SeriesFilters: []RegexFilter{{IsNot: "^container_.*_seconds_total$"}}, + Resources: ResourceMapping{ + Overrides: map[string]GroupResource{ + "namespace": {Resource: "namespace"}, + "pod_name": {Resource: "pod"}, + }, + }, + Name: NameMapping{Matches: "^container_(.*)_total$"}, + MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()), + }, + + // container non-cumulative metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))), + SeriesFilters: []RegexFilter{{IsNot: "^container_.*_total$"}}, + Resources: ResourceMapping{ + Overrides: map[string]GroupResource{ + "namespace": {Resource: "namespace"}, + "pod_name": {Resource: "pod"}, + }, + }, + Name: NameMapping{Matches: "^container_(.*)$"}, + MetricsQuery: `sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>)`, + }, + + // normal non-cumulative metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))), + SeriesFilters: []RegexFilter{{IsNot: ".*_total$"}}, + Resources: ResourceMapping{ + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), + }, + MetricsQuery: "sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)", + }, + + // normal rate metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))), + SeriesFilters: []RegexFilter{{IsNot: ".*_seconds_total"}}, + Name: NameMapping{Matches: "^(.*)_total$"}, + Resources: ResourceMapping{ + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), + }, + MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()), + }, + + // seconds rate metrics + { + SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))), + Name: NameMapping{Matches: "^(.*)_seconds_total$"}, + Resources: ResourceMapping{ + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), + }, + MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()), + }, + }, + } +} diff --git a/deploy/manifests/custom-metrics-apiserver-deployment.yaml b/deploy/manifests/custom-metrics-apiserver-deployment.yaml index 848d4ec3..5ca0e55f 100644 --- a/deploy/manifests/custom-metrics-apiserver-deployment.yaml +++ b/deploy/manifests/custom-metrics-apiserver-deployment.yaml @@ -28,15 +28,21 @@ spec: - --logtostderr=true - --prometheus-url=http://prometheus.prom.svc:9090/ - --metrics-relist-interval=30s - - --rate-interval=5m - --v=10 + - --config=/default-config.yaml ports: - containerPort: 6443 volumeMounts: - mountPath: /var/run/serving-cert name: volume-serving-cert readOnly: true + - mountPath: /etc/adapter/ + name: config + readOnly: true volumes: - name: volume-serving-cert secret: secretName: cm-adapter-serving-certs + - name: config + configMap: + name: adapter-config diff --git a/deploy/manifests/custom-metrics-config-map.yaml b/deploy/manifests/custom-metrics-config-map.yaml new file mode 100644 index 00000000..04ee0c67 --- /dev/null +++ b/deploy/manifests/custom-metrics-config-map.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: adapter-config + namespace: custom-metrics +data: + config.yaml: | + rules: + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) diff --git a/docs/config.md b/docs/config.md new file mode 100644 index 00000000..43d5dee2 --- /dev/null +++ b/docs/config.md @@ -0,0 +1,208 @@ +Metrics Discovery and Presentation Configuration +================================================ + +The adapter determines which metrics to expose, and how to expose them, +through a set of "discovery" rules. Each rule is executed independently +(so make sure that your rules are mutually exclusive), and specifies each +of the steps the adapter needs to take to expose a metric in the API. + +Each rule can be broken down into roughly four parts: + +- *Discovery*, which specifies how the adapter should find all Prometheus + metrics for this rule. + +- *Association*, which specifies how the adapter should determine which + Kubernetes resources a particular metric is associated with. + +- *Naming*, which specifies how the adapter should expose the metric in + the custom metrics API. + +- *Querying*, which specifies how a request for a particular metric on one + or more Kubernetes objects should be turned into a query to Prometheus. + +A more comprehensive configuration file can be found in +[sample-config.yaml](sample-config.yaml), but a basic config with one rule +might look like: + +```yaml +rules: +# this rule matches cumulative cAdvisor metrics measured in seconds +- seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + resources: + # skip specifying generic resource<->label mappings, and just + # attach only pod and namespace resources by mapping label names to group-resources + overrides: + namespace: {resource: "namespace"}, + pod_name: {resource: "pod"}, + # specify that the `container_` and `_seconds_total` suffixes should be removed. + # this also introduces an implicit filter on metric family names + name: + # we use the value of the capture group implicitly as the API name + # we could also explicitly write `as: "$1"` + matches: "^container_(.*)_seconds_total$" + # specify how to construct a query to fetch samples for a given series + # This is a Go template where the `.Series` and `.LabelMatchers` string values + # are available, and the delimiters are `<<` and `>>` to avoid conflicts with + # the prometheus query language + metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[2m])) by (<<.GroupBy>>)" +``` + +Discovery +--------- + +Discovery governs the process of finding the metrics that you want to +expose in the custom metrics API. There are two fields that factor into +discovery: `seriesQuery` and `seriesFilters`. + +`seriesQuery` specifies Prometheus series query (as passed to the +`/api/v1/series` endpoint in Prometheus) to use to find some set of +Prometheus series. The adapter will strip the label values from this +series, and then use the resulting metric-name-label-names combinations +later on. + +In many cases, `seriesQuery` will be sufficient to narrow down the list of +Prometheus series. However, sometimes (especially if two rules might +otherwise overlap), it's useful to do additional filtering on metric +names. In this case, `seriesFilters` can be used. After the list of +series is returned from `seriesQuery`, each series has its metric name +filtered through any specified filters. + +Filters may be either: + +- `is: `, which matches any series whose name matches the specified + regex. + +- `isNot: `, which matches any series whose name does not match the + specified regex. + +For example: + +```yaml +# match all cAdvisor metrics that aren't measured in seconds +seriesQuery: '{__name__=~"^container_.*_total",container_name!="POD",namespace!="",pod_name!=""}' +seriesFilters: + isNot: "^container_.*_seconds_total" +``` + +Association +----------- + +Association governs the process of figuring out which Kubernetes resources +a particular metric could be attached to. The `resources` field controls +this process. + +There are two ways to associate resources with a particular metric. In +both cases, the value of the label becomes the name of the particular +object. + +One way is to specify that any label name that matches some particular +pattern refers to some group-resource based on the label name. This can +be done using the `template` field. The pattern is specified as a Go +template, with the `Group` and `Resource` fields representing group and +resource. You don't necessarily have to use the `Group` field (in which +case the group is guessed by the system). For instance: + +```yaml +# any label `kube__` becomes . in Kubernetes +resources: + template: "kube_<<.Group>>_<<.Resource>>" +``` + +The other way is to specify that some particular label represents some +particular Kubernetes resource. This can be done using the `overrides` +field. Each override maps a Prometheus label to a Kubernetes +group-resource. For instance: + +```yaml +# the microservice label corresponds to the apps.deployment resource +resource: + overrides: + microservice: {group: "apps", resource: "deployment"} +``` + +These two can be combined, so you can specify both a template and some +individual overrides. + +Naming +------ + +Naming governs the process of converting a Prometheus metric name into +a metric in the custom metrics API, and vice version. It's controlled by +the `name` field. + +Naming is controlled by specifying a pattern to extract an API name from +a Prometheus name, and potentially a transformation on that extracted +value. + +The pattern is specified in the `matches` field, and is just a regular +expression. If not specified, it defaults to `.*`. + +The transformation is specified by the `as` field. You can use any +capture groups defined in the `matches` field. If the `matches` field +doesn't contain capture groups, the `as` field defaults to `$0`. If it +contains a single capture group, the `as` field defautls to `$1`. +Otherwise, it's an error not to specify the as field. + +For example: + +```yaml +# match turn any name _total to _per_second +# e.g. http_requests_total becomes http_requests_per_second +name: + matches: "^(.*)_total$" + as: "<<1}_per_second" +``` + +Querying +-------- + +Querying governs the process of actually fetching values for a particular +metric. It's controlled by the `metricsQuery` field. + +The `metricsQuery` field is a Go template that gets turned into +a Prometheus query, using input from a particular call to the custom +metrics API. A given call to the custom metrics API is distilled down to +a metric name, a group-resource, and one or more objects of that +group-resource. These get turned into the following fields in the +template: + +- `Series`: the metric name +- `LabelMatchers`: a comma-separated list of label matchers matching the + given objects. Currently, this is the label for the particular + group-resource, plus the label for namespace, if the group-resource is + namespaced. +- `GroupBy`: a comma-separated list of labels to group by. Currently, + this contains the group-resoure label used in `LabelMarchers`. + +For instance, suppose we had a series `http_requests_total` (exposed as +`http_requests_per_second` in the API) with labels `service`, `pod`, +`ingress`, `namespace`, and `verb`. The first four correspond to +Kubernetes resources. Then, if someone requested the metric +`pods/http_request_per_second` for the pods `pod1` and `pod2` in the +`somens` namespace, we'd have: + +- `Series: "http_requests_total" +- `LabelMatchers: "pod=~\"pod1|pod2",namespace="somens"` +- `GroupBy`: `pod` + +Additionally, there are two advanced fields that are "raw" forms of other +fields: + +- `LabelValuesByName`: a map mapping the labels and values from the + `LabelMatchers` field. The values are pre-joined by `|` + (for used with the `=~` matcher in Prometheus). +- `GroupBySlice`: the slice form of `GroupBy`. + +In general, you'll probably want to use the `Series`, `LabelMatchers`, and +`GroupBy` fields. The other two are for advanced usage. + +The query is expected to return one value for each object requested. The +adapter will use the labels on the returned series to associate a given +series back to its corresponding object. + +For example: + +```yaml +# convert cumulative cAdvisor metrics into rates calculated over 2 minutes +metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[2m])) by (<<.GroupBy>>)" +``` diff --git a/docs/sample-config.yaml b/docs/sample-config.yaml new file mode 100644 index 00000000..3aa5be6d --- /dev/null +++ b/docs/sample-config.yaml @@ -0,0 +1,69 @@ +rules: +# Each rule represents a some naming and discovery logic. +# Each rule is executed independently of the others, so +# take care to avoid overlap. As an optimization, rules +# with the same `seriesQuery` but different +# `name` or `seriesFilters` will use only one query to +# Prometheus for discovery. + +# some of these rules are taken from the "default" configuration, which +# can be found in pkg/config/default.go + +# this rule matches cumulative cAdvisor metrics measured in seconds +- seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + resources: + # skip specifying generic resource<->label mappings, and just + # attach only pod and namespace resources by mapping label names to group-resources + overrides: + namespace: {resource: "namespace"}, + pod_name: {resource: "pod"}, + # specify that the `container_` and `_seconds_total` suffixes should be removed. + # this also introduces an implicit filter on metric family names + name: + # we use the value of the capture group implicitly as the API name + # we could also explicitly write `as: "$1"` + matches: "^container_(.*)_seconds_total$" + # specify how to construct a query to fetch samples for a given series + # This is a Go template where the `.Series` and `.LabelMatchers` string values + # are available, and the delimiters are `<<` and `>>` to avoid conflicts with + # the prometheus query language + metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[2m])) by (<<.GroupBy>>)" + +# this rule matches cumulative cAdvisor metrics not measured in seconds +- seriesQuery: '{__name__=~"^container_.*_total",container_name!="POD",namespace!="",pod_name!=""}' + resources: + overrides: + namespace: {resource: "namespace"}, + pod_name: {resource: "pod"}, + seriesFilters: + # since this is a superset of the query above, we introduce an additional filter here + - isNot: "^container_.*_seconds_total$" + name: {matches: "^container_(.*)_total$"} + metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[2m])) by (<<.GroupBy>>)" + +# this rule matches cumulative non-cAdvisor metrics +- seriesQuery: '{namespace!="",__name__!="^container_.*"}' + name: {matches: "^(.*)_total$"} + resources: + # specify an a generic mapping between resources and labels. This + # is a template, like the `metricsQuery` template, except with the `.Group` + # and `.Resource` strings available. It will also be used to match labels, + # so avoid using template functions which truncate the group or resource. + # Group will be converted to a form acceptible for use as a label automatically. + template: "<<.Resource>>" + # if we wanted to, we could also specify overrides here + metricsQuery: "sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[2m])) by (<<.GroupBy>>)" + +# this rule matches only a single metric, explicitly naming it something else +# It's series query *must* return only a single metric family +- seriesQuery: 'cheddar{sharp="true"}' + # this metric will appear as "cheesy_goodness" in the custom metrics API + name: {as: "cheesy_goodness"} + resources: + overrides: + # this should still resolve in our cluster + brand: {group: "cheese.io", resource: "brand"} + metricQuery: 'count(cheddar{sharp="true"})' + +# TODO: should we be able to map to a constant instance of a resource +# (e.g. `resources: {constant: [{resource: "namespace", name: "kube-system"}}]`)? diff --git a/glide.lock b/glide.lock deleted file mode 100644 index 94bf85b9..00000000 --- a/glide.lock +++ /dev/null @@ -1,571 +0,0 @@ -hash: 1d2ad16816cec54a2561278dfd05ae9725247ea9b72db5c7d330f0074d069fad -updated: 2017-09-28T15:22:28.593829441-04:00 -imports: -- name: bitbucket.org/ww/goautoneg - version: 75cd24fc2f2c2a2088577d12123ddee5f54e0675 -- name: github.com/beorn7/perks - version: 3ac7bf7a47d159a033b107610db8a1b6575507a4 - subpackages: - - quantile -- name: github.com/coreos/etcd - version: 0520cb9304cb2385f7e72b8bc02d6e4d3257158a - subpackages: - - alarm - - auth - - auth/authpb - - client - - clientv3 - - compactor - - discovery - - error - - etcdserver - - etcdserver/api - - etcdserver/api/v2http - - etcdserver/api/v2http/httptypes - - etcdserver/api/v3rpc - - etcdserver/api/v3rpc/rpctypes - - etcdserver/auth - - etcdserver/etcdserverpb - - etcdserver/membership - - etcdserver/stats - - integration - - lease - - lease/leasehttp - - lease/leasepb - - mvcc - - mvcc/backend - - mvcc/mvccpb - - pkg/adt - - pkg/contention - - pkg/cpuutil - - pkg/crc - - pkg/fileutil - - pkg/httputil - - pkg/idutil - - pkg/ioutil - - pkg/logutil - - pkg/monotime - - pkg/netutil - - pkg/pathutil - - pkg/pbutil - - pkg/runtime - - pkg/schedule - - pkg/testutil - - pkg/tlsutil - - pkg/transport - - pkg/types - - pkg/wait - - proxy/grpcproxy - - proxy/grpcproxy/cache - - raft - - raft/raftpb - - rafthttp - - snap - - snap/snappb - - store - - version - - wal - - wal/walpb -- name: github.com/coreos/go-systemd - version: 48702e0da86bd25e76cfef347e2adeb434a0d0a6 - subpackages: - - daemon - - journal -- name: github.com/coreos/pkg - version: fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8 - subpackages: - - capnslog - - health - - httputil - - timeutil -- name: github.com/davecgh/go-spew - version: 782f4967f2dc4564575ca782fe2d04090b5faca8 - subpackages: - - spew -- name: github.com/elazarl/go-bindata-assetfs - version: 3dcc96556217539f50599357fb481ac0dc7439b9 -- name: github.com/emicklei/go-restful - version: ff4f55a206334ef123e4f79bbf348980da81ca46 - subpackages: - - log -- name: github.com/emicklei/go-restful-swagger12 - version: dcef7f55730566d41eae5db10e7d6981829720f6 -- name: github.com/evanphx/json-patch - version: 944e07253867aacae43c04b2e6a239005443f33a -- name: github.com/ghodss/yaml - version: 73d445a93680fa1a78ae23a5839bad48f32ba1ee -- name: github.com/go-openapi/jsonpointer - version: 46af16f9f7b149af66e5d1bd010e3574dc06de98 -- name: github.com/go-openapi/jsonreference - version: 13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272 -- name: github.com/go-openapi/spec - version: 6aced65f8501fe1217321abf0749d354824ba2ff -- name: github.com/go-openapi/swag - version: 1d0bd113de87027671077d3c71eb3ac5d7dbba72 -- name: github.com/gogo/protobuf - version: c0656edd0d9eab7c66d1eb0c568f9039345796f7 - subpackages: - - proto - - sortkeys -- name: github.com/golang/glog - version: 44145f04b68cf362d9c4df2182967c2275eaefed -- name: github.com/golang/protobuf - version: 4bd1920723d7b7c925de087aa32e2187708897f7 - subpackages: - - jsonpb - - proto - - ptypes - - ptypes/any - - ptypes/duration - - ptypes/timestamp -- name: github.com/google/btree - version: 7d79101e329e5a3adf994758c578dab82b90c017 -- name: github.com/google/gofuzz - version: 44d81051d367757e1c7c6a5a86423ece9afcf63c -- name: github.com/googleapis/gnostic - version: 0c5108395e2debce0d731cf0287ddf7242066aba - subpackages: - - OpenAPIv2 - - compiler - - extensions -- name: github.com/gregjones/httpcache - version: 787624de3eb7bd915c329cba748687a3b22666a6 - subpackages: - - diskcache -- name: github.com/grpc-ecosystem/go-grpc-prometheus - version: 2500245aa6110c562d17020fb31a2c133d737799 -- name: github.com/grpc-ecosystem/grpc-gateway - version: 84398b94e188ee336f307779b57b3aa91af7063c - subpackages: - - runtime - - runtime/internal - - utilities -- name: github.com/hashicorp/golang-lru - version: a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4 - subpackages: - - simplelru -- name: github.com/howeyc/gopass - version: bf9dde6d0d2c004a008c27aaee91170c786f6db8 -- name: github.com/imdario/mergo - version: 6633656539c1639d9d78127b7d47c622b5d7b6dc -- name: github.com/inconshreveable/mousetrap - version: 76626ae9c91c4f2a10f34cad8ce83ea42c93bb75 -- name: github.com/json-iterator/go - version: 36b14963da70d11297d313183d7e6388c8510e1e -- name: github.com/juju/ratelimit - version: 5b9ff866471762aa2ab2dced63c9fb6f53921342 -- name: github.com/kubernetes-incubator/custom-metrics-apiserver - version: fae01650d93f5de6151a024e36323344e14427aa - subpackages: - - pkg/apiserver - - pkg/apiserver/installer - - pkg/cmd/server - - pkg/dynamicmapper - - pkg/provider - - pkg/registry/custom_metrics -- name: github.com/mailru/easyjson - version: d5b7844b561a7bc640052f1b935f7b800330d7e0 - subpackages: - - buffer - - jlexer - - jwriter -- name: github.com/matttproud/golang_protobuf_extensions - version: fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a - subpackages: - - pbutil -- name: github.com/mxk/go-flowrate - version: cca7078d478f8520f85629ad7c68962d31ed7682 - subpackages: - - flowrate -- name: github.com/NYTimes/gziphandler - version: 56545f4a5d46df9a6648819d1664c3a03a13ffdb -- name: github.com/pborman/uuid - version: ca53cad383cad2479bbba7f7a1a05797ec1386e4 -- name: github.com/peterbourgon/diskv - version: 5f041e8faa004a95c88a202771f4cc3e991971e6 -- name: github.com/pkg/errors - version: a22138067af1c4942683050411a841ade67fe1eb -- name: github.com/prometheus/client_golang - version: e7e903064f5e9eb5da98208bae10b475d4db0f8c - subpackages: - - prometheus -- name: github.com/prometheus/client_model - version: fa8ad6fec33561be4280a8f0514318c79d7f6cb6 - subpackages: - - go -- name: github.com/prometheus/common - version: 13ba4ddd0caa9c28ca7b7bffe1dfa9ed8d5ef207 - subpackages: - - expfmt - - internal/bitbucket.org/ww/goautoneg - - model -- name: github.com/prometheus/procfs - version: 65c1f6f8f0fc1e2185eb9863a3bc751496404259 - subpackages: - - xfs -- name: github.com/PuerkitoBio/purell - version: 8a290539e2e8629dbc4e6bad948158f790ec31f4 -- name: github.com/PuerkitoBio/urlesc - version: 5bd2802263f21d8788851d5305584c82a5c75d7e -- name: github.com/spf13/cobra - version: f62e98d28ab7ad31d707ba837a966378465c7b57 -- name: github.com/spf13/pflag - version: 9ff6c6923cfffbcd502984b8e0c80539a94968b7 -- name: github.com/stretchr/testify - version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0 - subpackages: - - assert - - require -- name: github.com/ugorji/go - version: ded73eae5db7e7a0ef6f55aace87a2873c5d2b74 - subpackages: - - codec -- name: golang.org/x/crypto - version: 81e90905daefcd6fd217b62423c0908922eadb30 - subpackages: - - bcrypt - - blowfish - - nacl/secretbox - - poly1305 - - salsa20/salsa - - ssh/terminal -- name: golang.org/x/net - version: 1c05540f6879653db88113bc4a2b70aec4bd491f - subpackages: - - context - - html - - html/atom - - http2 - - http2/hpack - - idna - - internal/timeseries - - lex/httplex - - trace - - websocket -- name: golang.org/x/sys - version: 7ddbeae9ae08c6a06a59597f0c9edbc5ff2444ce - subpackages: - - unix - - windows -- name: golang.org/x/text - version: b19bf474d317b857955b12035d2c5acb57ce8b01 - subpackages: - - cases - - internal - - internal/tag - - language - - runes - - secure/bidirule - - secure/precis - - transform - - unicode/bidi - - unicode/norm - - width -- name: google.golang.org/genproto - version: 09f6ed296fc66555a25fe4ce95173148778dfa85 - subpackages: - - googleapis/rpc/status -- name: google.golang.org/grpc - version: d2e1b51f33ff8c5e4a15560ff049d200e83726c5 - subpackages: - - codes - - credentials - - grpclb/grpc_lb_v1 - - grpclog - - internal - - keepalive - - metadata - - naming - - peer - - stats - - status - - tap - - transport -- name: gopkg.in/inf.v0 - version: 3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4 -- name: gopkg.in/natefinch/lumberjack.v2 - version: 20b71e5b60d756d3d2f80def009790325acc2b23 -- name: gopkg.in/yaml.v2 - version: 53feefa2559fb8dfa8d81baad31be332c97d6c77 -- name: k8s.io/api - version: cadaf100c0a3dd6b254f320d6d651df079ec8e0a - subpackages: - - admissionregistration/v1alpha1 - - apps/v1beta1 - - apps/v1beta2 - - authentication/v1 - - authentication/v1beta1 - - authorization/v1 - - authorization/v1beta1 - - autoscaling/v1 - - autoscaling/v2beta1 - - batch/v1 - - batch/v1beta1 - - batch/v2alpha1 - - certificates/v1beta1 - - core/v1 - - extensions/v1beta1 - - networking/v1 - - policy/v1beta1 - - rbac/v1 - - rbac/v1alpha1 - - rbac/v1beta1 - - scheduling/v1alpha1 - - settings/v1alpha1 - - storage/v1 - - storage/v1beta1 -- name: k8s.io/apimachinery - version: 3b05bbfa0a45413bfa184edbf9af617e277962fb - subpackages: - - pkg/api/equality - - pkg/api/errors - - pkg/api/meta - - pkg/api/resource - - pkg/api/validation - - pkg/api/validation/path - - pkg/apimachinery - - pkg/apimachinery/announced - - pkg/apimachinery/registered - - pkg/apis/meta/internalversion - - pkg/apis/meta/v1 - - pkg/apis/meta/v1/unstructured - - pkg/apis/meta/v1/validation - - pkg/apis/meta/v1alpha1 - - pkg/conversion - - pkg/conversion/queryparams - - pkg/conversion/unstructured - - pkg/fields - - pkg/labels - - pkg/runtime - - pkg/runtime/schema - - pkg/runtime/serializer - - pkg/runtime/serializer/json - - pkg/runtime/serializer/protobuf - - pkg/runtime/serializer/recognizer - - pkg/runtime/serializer/streaming - - pkg/runtime/serializer/versioning - - pkg/selection - - pkg/types - - pkg/util/cache - - pkg/util/clock - - pkg/util/diff - - pkg/util/errors - - pkg/util/framer - - pkg/util/httpstream - - pkg/util/intstr - - pkg/util/json - - pkg/util/mergepatch - - pkg/util/net - - pkg/util/proxy - - pkg/util/rand - - pkg/util/runtime - - pkg/util/sets - - pkg/util/strategicpatch - - pkg/util/uuid - - pkg/util/validation - - pkg/util/validation/field - - pkg/util/wait - - pkg/util/yaml - - pkg/version - - pkg/watch - - third_party/forked/golang/json - - third_party/forked/golang/netutil - - third_party/forked/golang/reflect -- name: k8s.io/apiserver - version: c1e53d745d0fe45bf7d5d44697e6eface25fceca - subpackages: - - pkg/admission - - pkg/admission/initializer - - pkg/admission/plugin/namespace/lifecycle - - pkg/apis/apiserver - - pkg/apis/apiserver/install - - pkg/apis/apiserver/v1alpha1 - - pkg/apis/audit - - pkg/apis/audit/install - - pkg/apis/audit/v1alpha1 - - pkg/apis/audit/v1beta1 - - pkg/apis/audit/validation - - pkg/audit - - pkg/audit/policy - - pkg/authentication/authenticator - - pkg/authentication/authenticatorfactory - - pkg/authentication/group - - pkg/authentication/request/anonymous - - pkg/authentication/request/bearertoken - - pkg/authentication/request/headerrequest - - pkg/authentication/request/union - - pkg/authentication/request/websocket - - pkg/authentication/request/x509 - - pkg/authentication/serviceaccount - - pkg/authentication/token/tokenfile - - pkg/authentication/user - - pkg/authorization/authorizer - - pkg/authorization/authorizerfactory - - pkg/authorization/union - - pkg/endpoints - - pkg/endpoints/discovery - - pkg/endpoints/filters - - pkg/endpoints/handlers - - pkg/endpoints/handlers/negotiation - - pkg/endpoints/handlers/responsewriters - - pkg/endpoints/metrics - - pkg/endpoints/openapi - - pkg/endpoints/request - - pkg/features - - pkg/registry/generic - - pkg/registry/generic/registry - - pkg/registry/rest - - pkg/server - - pkg/server/filters - - pkg/server/healthz - - pkg/server/httplog - - pkg/server/mux - - pkg/server/options - - pkg/server/routes - - pkg/server/routes/data/swagger - - pkg/server/storage - - pkg/storage - - pkg/storage/errors - - pkg/storage/etcd - - pkg/storage/etcd/metrics - - pkg/storage/etcd/util - - pkg/storage/etcd3 - - pkg/storage/etcd3/preflight - - pkg/storage/names - - pkg/storage/storagebackend - - pkg/storage/storagebackend/factory - - pkg/storage/value - - pkg/util/feature - - pkg/util/flag - - pkg/util/flushwriter - - pkg/util/logs - - pkg/util/trace - - pkg/util/webhook - - pkg/util/wsstream - - plugin/pkg/audit/log - - plugin/pkg/audit/webhook - - plugin/pkg/authenticator/token/webhook - - plugin/pkg/authorizer/webhook -- name: k8s.io/client-go - version: 82aa063804cf055e16e8911250f888bc216e8b61 - subpackages: - - discovery - - dynamic - - dynamic/fake - - informers - - informers/admissionregistration - - informers/admissionregistration/v1alpha1 - - informers/apps - - informers/apps/v1beta1 - - informers/apps/v1beta2 - - informers/autoscaling - - informers/autoscaling/v1 - - informers/autoscaling/v2beta1 - - informers/batch - - informers/batch/v1 - - informers/batch/v1beta1 - - informers/batch/v2alpha1 - - informers/certificates - - informers/certificates/v1beta1 - - informers/core - - informers/core/v1 - - informers/extensions - - informers/extensions/v1beta1 - - informers/internalinterfaces - - informers/networking - - informers/networking/v1 - - informers/policy - - informers/policy/v1beta1 - - informers/rbac - - informers/rbac/v1 - - informers/rbac/v1alpha1 - - informers/rbac/v1beta1 - - informers/scheduling - - informers/scheduling/v1alpha1 - - informers/settings - - informers/settings/v1alpha1 - - informers/storage - - informers/storage/v1 - - informers/storage/v1beta1 - - kubernetes - - kubernetes/scheme - - kubernetes/typed/admissionregistration/v1alpha1 - - kubernetes/typed/apps/v1beta1 - - kubernetes/typed/apps/v1beta2 - - kubernetes/typed/authentication/v1 - - kubernetes/typed/authentication/v1beta1 - - kubernetes/typed/authorization/v1 - - kubernetes/typed/authorization/v1beta1 - - kubernetes/typed/autoscaling/v1 - - kubernetes/typed/autoscaling/v2beta1 - - kubernetes/typed/batch/v1 - - kubernetes/typed/batch/v1beta1 - - kubernetes/typed/batch/v2alpha1 - - kubernetes/typed/certificates/v1beta1 - - kubernetes/typed/core/v1 - - kubernetes/typed/extensions/v1beta1 - - kubernetes/typed/networking/v1 - - kubernetes/typed/policy/v1beta1 - - kubernetes/typed/rbac/v1 - - kubernetes/typed/rbac/v1alpha1 - - kubernetes/typed/rbac/v1beta1 - - kubernetes/typed/scheduling/v1alpha1 - - kubernetes/typed/settings/v1alpha1 - - kubernetes/typed/storage/v1 - - kubernetes/typed/storage/v1beta1 - - listers/admissionregistration/v1alpha1 - - listers/apps/v1beta1 - - listers/apps/v1beta2 - - listers/autoscaling/v1 - - listers/autoscaling/v2beta1 - - listers/batch/v1 - - listers/batch/v1beta1 - - listers/batch/v2alpha1 - - listers/certificates/v1beta1 - - listers/core/v1 - - listers/extensions/v1beta1 - - listers/networking/v1 - - listers/policy/v1beta1 - - listers/rbac/v1 - - listers/rbac/v1alpha1 - - listers/rbac/v1beta1 - - listers/scheduling/v1alpha1 - - listers/settings/v1alpha1 - - listers/storage/v1 - - listers/storage/v1beta1 - - pkg/version - - rest - - rest/watch - - testing - - tools/auth - - tools/cache - - tools/clientcmd - - tools/clientcmd/api - - tools/clientcmd/api/latest - - tools/clientcmd/api/v1 - - tools/metrics - - tools/pager - - tools/reference - - transport - - util/cert - - util/flowcontrol - - util/homedir - - util/integer -- name: k8s.io/kube-openapi - version: 868f2f29720b192240e18284659231b440f9cda5 - subpackages: - - pkg/builder - - pkg/common - - pkg/handler - - pkg/util -- name: k8s.io/metrics - version: 4c7ac522b9daf7beeb53f6766722ba78b7e5712d - subpackages: - - pkg/apis/custom_metrics - - pkg/apis/custom_metrics/install - - pkg/apis/custom_metrics/v1beta1 -testImports: -- name: github.com/pmezard/go-difflib - version: d8ed2627bdf02c080bf22230dbb337003b7aba2d - subpackages: - - difflib diff --git a/glide.yaml b/glide.yaml deleted file mode 100644 index 8b357166..00000000 --- a/glide.yaml +++ /dev/null @@ -1,23 +0,0 @@ -package: github.com/directxman12/k8s-prometheus-adapter -import: -- package: github.com/spf13/cobra -- package: k8s.io/apimachinery - subpackages: - - pkg/util/wait -- package: k8s.io/apiserver - subpackages: - - pkg/util/logs -- package: k8s.io/client-go - subpackages: - - kubernetes/typed/core/v1 - - rest - - tools/clientcmd -- package: github.com/kubernetes-incubator/custom-metrics-apiserver - subpackages: - - pkg/cmd/server - - pkg/provider -- package: github.com/stretchr/testify - version: ^1.1.4 - subpackages: - - assert - - require diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 00ee720d..6c93a0be 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" "github.com/prometheus/common/model" @@ -121,3 +122,11 @@ func (s *Series) UnmarshalJSON(data []byte) error { return nil } + +func (s *Series) String() string { + lblStrings := make([]string, 0, len(s.Labels)) + for k, v := range s.Labels { + lblStrings = append(lblStrings, fmt.Sprintf("%s=%q", k, v)) + } + return fmt.Sprintf("%s{%s}", s.Name, strings.Join(lblStrings, ",")) +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 00000000..6a31e8f1 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,75 @@ +package config + +type MetricsDiscoveryConfig struct { + // Rules specifies how to discover and map Prometheus metrics to + // custom metrics API resources. The rules are applied independently, + // and thus must be mutually exclusive. Rules will the same SeriesQuery + // will make only a single API call. + Rules []DiscoveryRule `yaml:"rules"` +} + +// DiscoveryRule describes on set of rules for transforming Prometheus metrics to/from +// custom metrics API resources. +type DiscoveryRule struct { + // SeriesQuery specifies which metrics this rule should consider via a Prometheus query + // series selector query. + SeriesQuery string `yaml:"seriesQuery"` + // SeriesFilters specifies additional regular expressions to be applied on + // the series names returned from the query. This is useful for constraints + // that can't be represented in the SeriesQuery (e.g. series matching `container_.+` + // not matching `container_.+_total`. A filter will be automatically appended to + // match the form specified in Name. + SeriesFilters []RegexFilter `yaml:"seriesFilters"` + // Resources specifies how associated Kubernetes resources should be discovered for + // the given metrics. + Resources ResourceMapping `yaml:"resources"` + // Name specifies how the metric name should be transformed between custom metric + // API resources, and Prometheus metric names. + Name NameMapping `yaml:"name"` + // MetricsQuery specifies modifications to the metrics query, such as converting + // cumulative metrics to rate metrics. It is a template where `.LabelMatchers` is + // a the comma-separated base label matchers and `.Series` is the series name, and + // `.GroupBy` is the comma-separated expected group-by label names. The delimeters + // are `<<` and `>>`. + MetricsQuery string `yaml:"metricsQuery,omitempty"` +} + +// RegexFilter is a filter that matches positively or negatively against a regex. +// Only one field may be set at a time. +type RegexFilter struct { + Is string `yaml:"is,omitempty"` + IsNot string `yaml:"isNot,omitempty"` +} + +// ResourceMapping specifies how to map Kubernetes resources to Prometheus labels +type ResourceMapping struct { + // Template specifies a golang string template for converting a Kubernetes + // group-resource to a Prometheus label. The template object contains + // the `.Group` and `.Resource` fields. The `.Group` field will have + // dots replaced with underscores, and the `.Resource` field will be + // singularized. The delimiters are `<<` and `>>`. + Template string `yaml:"template,omitempty"` + // Overrides specifies exceptions to the above template, mapping label names + // to group-resources + Overrides map[string]GroupResource `yaml:"overrides,omitempty"` +} + +// GroupResource represents a Kubernetes group-resource. +type GroupResource struct { + Group string `yaml:"group,omitempty"` + Resource string `yaml:"resource"` +} + +// NameMapping specifies how to convert Prometheus metrics +// to/from custom metrics API resources. +type NameMapping struct { + // Matches is a regular expression that is used to match + // Prometheus series names. It may be left blank, in which + // case it is equivalent to `.*`. + Matches string `yaml:"matches"` + // As is the name used in the API. Captures from Matches + // are available for use here. If not specified, it defaults + // to $0 if no capture groups are present in Matches, or $1 + // if only one is present, and will error if multiple are. + As string `yaml:"as"` +} diff --git a/pkg/config/loader.go b/pkg/config/loader.go new file mode 100644 index 00000000..ecc36a23 --- /dev/null +++ b/pkg/config/loader.go @@ -0,0 +1,32 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + + yaml "gopkg.in/yaml.v2" +) + +// FromFile loads the configuration from a particular file. +func FromFile(filename string) (*MetricsDiscoveryConfig, error) { + file, err := os.Open(filename) + defer file.Close() + if err != nil { + return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err) + } + contents, err := ioutil.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err) + } + return FromYAML(contents) +} + +// FromYAML loads the configuration from a blob of YAML. +func FromYAML(contents []byte) (*MetricsDiscoveryConfig, error) { + var cfg MetricsDiscoveryConfig + if err := yaml.Unmarshal(contents, &cfg); err != nil { + return nil, fmt.Errorf("unable to parse metrics discovery config: %v", err) + } + return &cfg, nil +} diff --git a/pkg/custom-provider/metric_namer.go b/pkg/custom-provider/metric_namer.go index 90d7cdd5..7de4f69d 100644 --- a/pkg/custom-provider/metric_namer.go +++ b/pkg/custom-provider/metric_namer.go @@ -1,367 +1,474 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package provider import ( + "bytes" "fmt" + "regexp" "strings" "sync" + "text/template" + "github.com/golang/glog" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/golang/glog" + "github.com/directxman12/k8s-prometheus-adapter/pkg/config" pmodel "github.com/prometheus/common/model" ) -// NB: container metrics sourced from cAdvisor don't consistently follow naming conventions, -// so we need to whitelist them and handle them on a case-by-case basis. Metrics ending in `_total` -// *should* be counters, but may actually be guages in this case. +var nsGroupResource = schema.GroupResource{Resource: "namespaces"} +var groupNameSanitizer = strings.NewReplacer(".", "_", "-", "_") -// SeriesType represents the kind of series backing a metric. -type SeriesType int - -const ( - CounterSeries SeriesType = iota - SecondsCounterSeries - GaugeSeries -) - -// SeriesRegistry provides conversions between Prometheus series and MetricInfo -type SeriesRegistry interface { - // Selectors produces the appropriate Prometheus selectors to match all series handlable - // by this registry, as an optimization for SetSeries. - Selectors() []prom.Selector - // SetSeries replaces the known series in this registry - SetSeries(series []prom.Series) error - // ListAllMetrics lists all metrics known to this registry - ListAllMetrics() []provider.MetricInfo - // SeriesForMetric looks up the minimum required series information to make a query for the given metric - // against the given resource (namespace may be empty for non-namespaced resources) - QueryForMetric(info provider.MetricInfo, namespace string, resourceNames ...string) (kind SeriesType, query prom.Selector, groupBy string, found bool) - // MatchValuesToNames matches result values to resource names for the given metric and value set - MatchValuesToNames(metricInfo provider.MetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) +// MetricNamer knows how to convert Prometheus series names and label names to +// metrics API resources, and vice-versa. MetricNamers should be safe to access +// concurrently. Returned group-resources are "normalized" as per the +// MetricInfo#Normalized method. Group-resources passed as arguments must +// themselves be normalized. +type MetricNamer interface { + // Selector produces the appropriate Prometheus series selector to match all + // series handlable by this namer. + Selector() prom.Selector + // FilterSeries checks to see which of the given series match any additional + // constrains beyond the series query. It's assumed that the series given + // already matche the series query. + FilterSeries(series []prom.Series) []prom.Series + // ResourcesForSeries returns the group-resources associated with the given series, + // as well as whether or not the given series has the "namespace" resource). + ResourcesForSeries(series prom.Series) (res []schema.GroupResource, namespaced bool) + // LabelForResource returns the appropriate label for the given resource. + LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) + // MetricNameForSeries returns the name (as presented in the API) for a given series. + MetricNameForSeries(series prom.Series) (string, error) + // QueryForSeries returns the query for a given series (not API metric name), with + // the given namespace name (if relevant), resource, and resource names. + QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) } -type seriesInfo struct { - // baseSeries represents the minimum information to access a particular series - baseSeries prom.Series - // kind is the type of this series - kind SeriesType - // isContainer indicates if the series is a cAdvisor container_ metric, and thus needs special handling - isContainer bool +// labelGroupResExtractor extracts schema.GroupResources from series labels. +type labelGroupResExtractor struct { + regex *regexp.Regexp + + resourceInd int + groupInd *int + mapper apimeta.RESTMapper } -// overridableSeriesRegistry is a basic SeriesRegistry -type basicSeriesRegistry struct { - mu sync.RWMutex - - // info maps metric info to information about the corresponding series - info map[provider.MetricInfo]seriesInfo - // metrics is the list of all known metrics - metrics []provider.MetricInfo - - // namer is the metricNamer responsible for converting series to metric names and information - namer metricNamer -} - -func (r *basicSeriesRegistry) Selectors() []prom.Selector { - // container-specific metrics from cAdvsior have their own form, and need special handling - // TODO: figure out how to determine which metrics on non-namespaced objects are kubernetes-related - containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", "")) - namespacedSel := prom.MatchSeries("", prom.LabelNeq(r.namer.labelPrefix+"namespace", ""), prom.NameNotMatches("^container_.*")) - - return []prom.Selector{containerSel, namespacedSel} -} - -func (r *basicSeriesRegistry) SetSeries(newSeries []prom.Series) error { - newInfo := make(map[provider.MetricInfo]seriesInfo) - for _, series := range newSeries { - if strings.HasPrefix(series.Name, "container_") { - r.namer.processContainerSeries(series, newInfo) - } else if namespaceLabel, hasNamespaceLabel := series.Labels[pmodel.LabelName(r.namer.labelPrefix+"namespace")]; hasNamespaceLabel && namespaceLabel != "" { - // we also handle namespaced metrics here as part of the resource-association logic - if err := r.namer.processNamespacedSeries(series, newInfo); err != nil { - glog.Errorf("Unable to process namespaced series %q: %v", series.Name, err) - continue - } - } else { - if err := r.namer.processRootScopedSeries(series, newInfo); err != nil { - glog.Errorf("Unable to process root-scoped series %q: %v", series.Name, err) - continue - } - } +// newLabelGroupResExtractor creates a new labelGroupResExtractor for labels whose form +// matches the given template. It does so by creating a regular expression from the template, +// so anything in the template which limits resource or group name length will cause issues. +func newLabelGroupResExtractor(labelTemplate *template.Template) (*labelGroupResExtractor, error) { + labelRegexBuff := new(bytes.Buffer) + if err := labelTemplate.Execute(labelRegexBuff, schema.GroupResource{"(?P.+?)", "(?P.+?)"}); err != nil { + return nil, fmt.Errorf("unable to convert label template to matcher: %v", err) } - - newMetrics := make([]provider.MetricInfo, 0, len(newInfo)) - for info := range newInfo { - newMetrics = append(newMetrics, info) + if labelRegexBuff.Len() == 0 { + return nil, fmt.Errorf("unable to convert label template to matcher: empty template") } - - r.mu.Lock() - defer r.mu.Unlock() - - r.info = newInfo - r.metrics = newMetrics - - return nil -} - -func (r *basicSeriesRegistry) ListAllMetrics() []provider.MetricInfo { - r.mu.RLock() - defer r.mu.RUnlock() - - return r.metrics -} - -func (r *basicSeriesRegistry) QueryForMetric(metricInfo provider.MetricInfo, namespace string, resourceNames ...string) (kind SeriesType, query prom.Selector, groupBy string, found bool) { - r.mu.RLock() - defer r.mu.RUnlock() - - if len(resourceNames) == 0 { - glog.Errorf("no resource names requested while producing a query for metric %s", metricInfo.String()) - return 0, "", "", false - } - - metricInfo, singularResource, err := metricInfo.Normalized(r.namer.mapper) + labelRegexRaw := "^" + labelRegexBuff.String() + "$" + labelRegex, err := regexp.Compile(labelRegexRaw) if err != nil { - glog.Errorf("unable to normalize group resource while producing a query: %v", err) - return 0, "", "", false - } - resourceLbl := r.namer.labelPrefix + singularResource - - // TODO: support container metrics - if info, found := r.info[metricInfo]; found { - targetValue := resourceNames[0] - matcher := prom.LabelEq - if len(resourceNames) > 1 { - targetValue = strings.Join(resourceNames, "|") - matcher = prom.LabelMatches - } - - var expressions []string - if info.isContainer { - expressions = []string{matcher("pod_name", targetValue), prom.LabelNeq("container_name", "POD")} - groupBy = "pod_name" - } else { - // TODO: copy base series labels? - expressions = []string{matcher(resourceLbl, targetValue)} - groupBy = resourceLbl - } - - if metricInfo.Namespaced { - prefix := r.namer.labelPrefix - if info.isContainer { - prefix = "" - } - expressions = append(expressions, prom.LabelEq(prefix+"namespace", namespace)) - } - - return info.kind, prom.MatchSeries(info.baseSeries.Name, expressions...), groupBy, true + return nil, fmt.Errorf("unable to convert label template to matcher: %v", err) } - glog.V(10).Infof("metric %v not registered", metricInfo) - return 0, "", "", false + var groupInd *int + var resInd *int + + for i, name := range labelRegex.SubexpNames() { + switch name { + case "group": + ind := i // copy to avoid iteration variable reference + groupInd = &ind + case "resource": + ind := i // copy to avoid iteration variable reference + resInd = &ind + } + } + + if resInd == nil { + return nil, fmt.Errorf("must include at least `{{.Resource}}` in the label template") + } + + return &labelGroupResExtractor{ + regex: labelRegex, + resourceInd: *resInd, + groupInd: groupInd, + }, nil } -func (r *basicSeriesRegistry) MatchValuesToNames(metricInfo provider.MetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) { - r.mu.RLock() - defer r.mu.RUnlock() - - metricInfo, singularResource, err := metricInfo.Normalized(r.namer.mapper) - if err != nil { - glog.Errorf("unable to normalize group resource while matching values to names: %v", err) - return nil, false - } - resourceLbl := r.namer.labelPrefix + singularResource - - if info, found := r.info[metricInfo]; found { - res := make(map[string]pmodel.SampleValue, len(values)) - for _, val := range values { - if val == nil { - // skip empty values - continue - } - - labelName := pmodel.LabelName(resourceLbl) - if info.isContainer { - labelName = pmodel.LabelName("pod_name") - } - res[string(val.Metric[labelName])] = val.Value +// GroupResourceForLabel extracts a schema.GroupResource from the given label, if possible. +// The second argument indicates whether or not a potential group-resource was found in this label. +func (e *labelGroupResExtractor) GroupResourceForLabel(lbl pmodel.LabelName) (schema.GroupResource, bool) { + matchGroups := e.regex.FindStringSubmatch(string(lbl)) + if matchGroups != nil { + group := "" + if e.groupInd != nil { + group = matchGroups[*e.groupInd] } - return res, true + return schema.GroupResource{ + Group: group, + Resource: matchGroups[e.resourceInd], + }, true } - return nil, false + return schema.GroupResource{}, false } -// metricNamer knows how to construct MetricInfo out of raw prometheus series descriptions. -type metricNamer struct { - // overrides contains the list of container metrics whose naming we want to override. - // This is used to properly convert certain cAdvisor container metrics. - overrides map[string]seriesSpec - - mapper apimeta.RESTMapper - - labelPrefix string +func (r *metricNamer) Selector() prom.Selector { + return r.seriesQuery } -// seriesSpec specifies how to produce metric info for a particular prometheus series source -type seriesSpec struct { - // metricName is the desired output API metric name - metricName string - // kind indicates whether or not this metric is cumulative, - // and thus has to be calculated as a rate when returning it - kind SeriesType +// reMatcher either positively or negatively matches a regex +type reMatcher struct { + regex *regexp.Regexp + positive bool } -// processContainerSeries performs special work to extract metric definitions -// from cAdvisor-sourced container metrics, which don't particularly follow any useful conventions consistently. -func (n *metricNamer) processContainerSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) { +func newReMatcher(cfg config.RegexFilter) (*reMatcher, error) { + if cfg.Is != "" && cfg.IsNot != "" { + return nil, fmt.Errorf("cannot have both an `is` (%q) and `isNot` (%q) expression in a single filter", cfg.Is, cfg.IsNot) + } + if cfg.Is == "" && cfg.IsNot == "" { + return nil, fmt.Errorf("must have either an `is` or `isNot` expression in a filter") + } - originalName := series.Name - - var name string - metricKind := GaugeSeries - if override, hasOverride := n.overrides[series.Name]; hasOverride { - name = override.metricName - metricKind = override.kind + var positive bool + var regexRaw string + if cfg.Is != "" { + positive = true + regexRaw = cfg.Is } else { - // chop of the "container_" prefix - series.Name = series.Name[10:] - name, metricKind = n.metricNameFromSeries(series) + positive = false + regexRaw = cfg.IsNot } - info := provider.MetricInfo{ - GroupResource: schema.GroupResource{Resource: "pods"}, - Namespaced: true, - Metric: name, - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: originalName}, - isContainer: true, - } -} - -// processNamespacedSeries adds the metric info for the given generic namespaced series to -// the map of metric info. -func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error { - // NB: all errors must occur *before* we save the series info - name, metricKind := n.metricNameFromSeries(series) - resources, err := n.groupResourcesFromSeries(series) + regex, err := regexp.Compile(regexRaw) if err != nil { - return fmt.Errorf("unable to process prometheus series %s: %v", series.Name, err) + return nil, fmt.Errorf("unable to compile series filter %q: %v", regexRaw, err) } - // we add one metric for each resource that this could describe - for _, resource := range resources { - info := provider.MetricInfo{ - GroupResource: resource, - Namespaced: true, - Metric: name, - } - - // metrics describing namespaces aren't considered to be namespaced - if resource == (schema.GroupResource{Resource: "namespaces"}) { - info.Namespaced = false - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: series.Name}, - } - } - - return nil + return &reMatcher{ + regex: regex, + positive: positive, + }, nil } -// processesRootScopedSeries adds the metric info for the given generic namespaced series to -// the map of metric info. -func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[provider.MetricInfo]seriesInfo) error { - // NB: all errors must occur *before* we save the series info - name, metricKind := n.metricNameFromSeries(series) - resources, err := n.groupResourcesFromSeries(series) - if err != nil { - return fmt.Errorf("unable to process prometheus series %s: %v", series.Name, err) - } - - // we add one metric for each resource that this could describe - for _, resource := range resources { - info := provider.MetricInfo{ - GroupResource: resource, - Namespaced: false, - Metric: name, - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: series.Name}, - } - } - - return nil +func (m *reMatcher) Matches(val string) bool { + return m.regex.MatchString(val) == m.positive } -// groupResourceFromSeries collects the possible group-resources that this series could describe by -// going through each label, checking to see if it corresponds to a known resource. For instance, -// a series `ingress_http_hits_total{pod="foo",service="bar",ingress="baz",namespace="ns"}` -// would return three GroupResources: "pods", "services", and "ingresses". -// Returned MetricInfo is equilavent to the "normalized" info produced by metricInfo.Normalized. -func (n *metricNamer) groupResourcesFromSeries(series prom.Series) ([]schema.GroupResource, error) { - var res []schema.GroupResource - for label := range series.Labels { - if !strings.HasPrefix(string(label), n.labelPrefix) { - continue - } - label = label[len(n.labelPrefix):] - // TODO: figure out a way to let people specify a fully-qualified name in label-form - gvr, err := n.mapper.ResourceFor(schema.GroupVersionResource{Resource: string(label)}) - if err != nil { - if apimeta.IsNoMatchError(err) { - continue +type metricNamer struct { + seriesQuery prom.Selector + labelTemplate *template.Template + labelResExtractor *labelGroupResExtractor + metricsQueryTemplate *template.Template + nameMatches *regexp.Regexp + nameAs string + seriesMatchers []*reMatcher + + labelResourceMu sync.RWMutex + labelToResource map[pmodel.LabelName]schema.GroupResource + resourceToLabel map[schema.GroupResource]pmodel.LabelName + mapper apimeta.RESTMapper +} + +// queryTemplateArgs are the arguments for the metrics query template. +type queryTemplateArgs struct { + Series string + LabelMatchers string + LabelValuesByName map[string][]string + GroupBy string + GroupBySlice []string +} + +func (n *metricNamer) FilterSeries(initialSeries []prom.Series) []prom.Series { + if len(n.seriesMatchers) == 0 { + return initialSeries + } + + finalSeries := make([]prom.Series, 0, len(initialSeries)) +SeriesLoop: + for _, series := range initialSeries { + for _, matcher := range n.seriesMatchers { + if !matcher.Matches(series.Name) { + continue SeriesLoop } - return nil, err } - res = append(res, gvr.GroupResource()) + finalSeries = append(finalSeries, series) } - return res, nil + return finalSeries } -// metricNameFromSeries extracts a metric name from a series name, and indicates -// whether or not that series was a counter. It also has special logic to deal with time-based -// counters, which general get converted to milli-unit rate metrics. -func (n *metricNamer) metricNameFromSeries(series prom.Series) (name string, kind SeriesType) { - kind = GaugeSeries - name = series.Name - if strings.HasSuffix(name, "_total") { - kind = CounterSeries - name = name[:len(name)-6] +func (n *metricNamer) QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) { + var exprs []string + valuesByName := map[string][]string{} - if strings.HasSuffix(name, "_seconds") { - kind = SecondsCounterSeries - name = name[:len(name)-8] + if namespace != "" { + namespaceLbl, err := n.LabelForResource(nsGroupResource) + if err != nil { + return "", err + } + exprs = append(exprs, prom.LabelEq(string(namespaceLbl), namespace)) + valuesByName[string(namespaceLbl)] = []string{namespace} + } + + resourceLbl, err := n.LabelForResource(resource) + if err != nil { + return "", err + } + matcher := prom.LabelEq + targetValue := names[0] + if len(names) > 1 { + matcher = prom.LabelMatches + targetValue = strings.Join(names, "|") + } + exprs = append(exprs, matcher(string(resourceLbl), targetValue)) + valuesByName[string(resourceLbl)] = names + + args := queryTemplateArgs{ + Series: series, + LabelMatchers: strings.Join(exprs, ","), + LabelValuesByName: valuesByName, + GroupBy: string(resourceLbl), + GroupBySlice: []string{string(resourceLbl)}, + } + queryBuff := new(bytes.Buffer) + if err := n.metricsQueryTemplate.Execute(queryBuff, args); err != nil { + return "", err + } + + if queryBuff.Len() == 0 { + return "", fmt.Errorf("empty query produced by metrics query template") + } + + return prom.Selector(queryBuff.String()), nil +} + +func (n *metricNamer) ResourcesForSeries(series prom.Series) ([]schema.GroupResource, bool) { + // use an updates map to avoid having to drop the read lock to update the cache + // until the end. Since we'll probably have few updates after the first run, + // this should mean that we rarely have to hold the write lock. + var resources []schema.GroupResource + updates := make(map[pmodel.LabelName]schema.GroupResource) + namespaced := false + + // use an anon func to get the right defer behavior + func() { + n.labelResourceMu.RLock() + defer n.labelResourceMu.RUnlock() + + for lbl := range series.Labels { + var groupRes schema.GroupResource + var ok bool + + // check if we have an override + if groupRes, ok = n.labelToResource[lbl]; ok { + resources = append(resources, groupRes) + } else if groupRes, ok = updates[lbl]; ok { + resources = append(resources, groupRes) + } else if n.labelResExtractor != nil { + // if not, check if it matches the form we expect, and if so, + // convert to a group-resource. + if groupRes, ok = n.labelResExtractor.GroupResourceForLabel(lbl); ok { + info, _, err := provider.CustomMetricInfo{GroupResource: groupRes}.Normalized(n.mapper) + if err != nil { + glog.Errorf("unable to normalize group-resource %s from label %q, skipping: %v", groupRes.String(), lbl, err) + continue + } + + groupRes = info.GroupResource + resources = append(resources, groupRes) + updates[lbl] = groupRes + } + } + + if groupRes == nsGroupResource { + namespaced = true + } + } + }() + + // update the cache for next time. This should only be called by discovery, + // so we don't really have to worry about the grap between read and write locks + // (plus, we don't care if someone else updates the cache first, since the results + // are necessarily the same, so at most we've done extra work). + if len(updates) > 0 { + n.labelResourceMu.Lock() + defer n.labelResourceMu.Unlock() + + for lbl, groupRes := range updates { + n.labelToResource[lbl] = groupRes } } - return + return resources, namespaced +} + +func (n *metricNamer) LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) { + n.labelResourceMu.RLock() + // check if we have a cached copy or override + lbl, ok := n.resourceToLabel[resource] + n.labelResourceMu.RUnlock() // release before we call makeLabelForResource + if ok { + return lbl, nil + } + + // NB: we don't actually care about the gap between releasing read lock + // and acquiring the write lock -- if we do duplicate work sometimes, so be + // it, as long as we're correct. + + // otherwise, use the template and save the result + lbl, err := n.makeLabelForResource(resource) + if err != nil { + return "", fmt.Errorf("unable to convert resource %s into label: %v", resource.String(), err) + } + return lbl, nil +} + +// makeLabelForResource constructs a label name for the given resource, and saves the result. +// It must *not* be called under an existing lock. +func (n *metricNamer) makeLabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) { + if n.labelTemplate == nil { + return "", fmt.Errorf("no generic resource label form specified for this metric") + } + buff := new(bytes.Buffer) + + singularRes, err := n.mapper.ResourceSingularizer(resource.Resource) + if err != nil { + return "", fmt.Errorf("unable to singularize resource %s: %v", resource.String(), err) + } + convResource := schema.GroupResource{ + Group: groupNameSanitizer.Replace(resource.Group), + Resource: singularRes, + } + + if err := n.labelTemplate.Execute(buff, convResource); err != nil { + return "", err + } + if buff.Len() == 0 { + return "", fmt.Errorf("empty label produced by label template") + } + lbl := pmodel.LabelName(buff.String()) + + n.labelResourceMu.Lock() + defer n.labelResourceMu.Unlock() + + n.resourceToLabel[resource] = lbl + n.labelToResource[lbl] = resource + return lbl, nil +} + +func (n *metricNamer) MetricNameForSeries(series prom.Series) (string, error) { + matches := n.nameMatches.FindStringSubmatchIndex(series.Name) + if matches == nil { + return "", fmt.Errorf("series name %q did not match expected pattern %q", series.Name, n.nameMatches.String()) + } + outNameBytes := n.nameMatches.ExpandString(nil, n.nameAs, series.Name, matches) + return string(outNameBytes), nil +} + +// NamersFromConfig produces a MetricNamer for each rule in the given config. +func NamersFromConfig(cfg *config.MetricsDiscoveryConfig, mapper apimeta.RESTMapper) ([]MetricNamer, error) { + namers := make([]MetricNamer, len(cfg.Rules)) + + for i, rule := range cfg.Rules { + var labelTemplate *template.Template + var labelResExtractor *labelGroupResExtractor + var err error + if rule.Resources.Template != "" { + labelTemplate, err = template.New("resource-label").Delims("<<", ">>").Parse(rule.Resources.Template) + if err != nil { + return nil, fmt.Errorf("unable to parse label template %q associated with series query %q: %v", rule.Resources.Template, rule.SeriesQuery, err) + } + + labelResExtractor, err = newLabelGroupResExtractor(labelTemplate) + if err != nil { + return nil, fmt.Errorf("unable to generate label format from template %q associated with series query %q: %v", rule.Resources.Template, rule.SeriesQuery, err) + } + } + + metricsQueryTemplate, err := template.New("metrics-query").Delims("<<", ">>").Parse(rule.MetricsQuery) + if err != nil { + return nil, fmt.Errorf("unable to parse metrics query template %q associated with series query %q: %v", rule.MetricsQuery, rule.SeriesQuery, err) + } + + seriesMatchers := make([]*reMatcher, len(rule.SeriesFilters)) + for i, filterRaw := range rule.SeriesFilters { + matcher, err := newReMatcher(filterRaw) + if err != nil { + return nil, fmt.Errorf("unable to generate series name filter associated with series query %q: %v", rule.SeriesQuery, err) + } + seriesMatchers[i] = matcher + } + if rule.Name.Matches != "" { + matcher, err := newReMatcher(config.RegexFilter{Is: rule.Name.Matches}) + if err != nil { + return nil, fmt.Errorf("unable to generate series name filter from name rules associated with series query %q: %v", rule.SeriesQuery, err) + } + seriesMatchers = append(seriesMatchers, matcher) + } + + var nameMatches *regexp.Regexp + if rule.Name.Matches != "" { + nameMatches, err = regexp.Compile(rule.Name.Matches) + if err != nil { + return nil, fmt.Errorf("unable to compile series name match expression %q associated with series query %q: %v", rule.Name.Matches, rule.SeriesQuery, err) + } + } else { + // this will always succeed + nameMatches = regexp.MustCompile(".*") + } + nameAs := rule.Name.As + if nameAs == "" { + // check if we have an obvious default + subexpNames := nameMatches.SubexpNames() + if len(subexpNames) == 1 { + // no capture groups, use the whole thing + nameAs = "$0" + } else if len(subexpNames) == 2 { + // one capture group, use that + nameAs = "$1" + } else { + return nil, fmt.Errorf("must specify an 'as' value for name matcher %q associated with series query %q", rule.Name.Matches, rule.SeriesQuery) + } + } + + namer := &metricNamer{ + seriesQuery: prom.Selector(rule.SeriesQuery), + labelTemplate: labelTemplate, + labelResExtractor: labelResExtractor, + metricsQueryTemplate: metricsQueryTemplate, + mapper: mapper, + nameMatches: nameMatches, + nameAs: nameAs, + seriesMatchers: seriesMatchers, + + labelToResource: make(map[pmodel.LabelName]schema.GroupResource), + resourceToLabel: make(map[schema.GroupResource]pmodel.LabelName), + } + + // invert the structure for consistency with the template + for lbl, groupRes := range rule.Resources.Overrides { + infoRaw := provider.CustomMetricInfo{ + GroupResource: schema.GroupResource{ + Group: groupRes.Group, + Resource: groupRes.Resource, + }, + } + info, _, err := infoRaw.Normalized(mapper) + if err != nil { + return nil, fmt.Errorf("unable to normalize group-resource %v: %v", groupRes, err) + } + + namer.labelToResource[pmodel.LabelName(lbl)] = info.GroupResource + namer.resourceToLabel[info.GroupResource] = pmodel.LabelName(lbl) + } + + namers[i] = namer + } + + return namers, nil } diff --git a/pkg/custom-provider/provider.go b/pkg/custom-provider/provider.go index 9a529104..b8e122cf 100644 --- a/pkg/custom-provider/provider.go +++ b/pkg/custom-provider/provider.go @@ -19,9 +19,9 @@ package provider import ( "context" "fmt" - "github.com/golang/glog" "time" + "github.com/golang/glog" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" pmodel "github.com/prometheus/common/model" apierr "k8s.io/apimachinery/pkg/api/errors" @@ -40,42 +40,40 @@ import ( prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" ) +// Runnable represents something that can be run until told to stop. +type Runnable interface { + // Run runs the runnable forever. + Run() + // RunUntil runs the runnable until the given channel is closed. + RunUntil(stopChan <-chan struct{}) +} + type prometheusProvider struct { mapper apimeta.RESTMapper - kubeClient dynamic.ClientPool + kubeClient dynamic.Interface promClient prom.Client SeriesRegistry - - rateInterval time.Duration } -func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.ClientPool, promClient prom.Client, labelPrefix string, updateInterval time.Duration, rateInterval time.Duration, stopChan <-chan struct{}) provider.CustomMetricsProvider { +func NewPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []MetricNamer, updateInterval time.Duration) (provider.CustomMetricsProvider, Runnable) { lister := &cachingMetricsLister{ updateInterval: updateInterval, promClient: promClient, + namers: namers, SeriesRegistry: &basicSeriesRegistry{ - namer: metricNamer{ - // TODO: populate the overrides list - overrides: nil, - mapper: mapper, - labelPrefix: labelPrefix, - }, + mapper: mapper, }, } - lister.RunUntil(stopChan) - return &prometheusProvider{ mapper: mapper, kubeClient: kubeClient, promClient: promClient, SeriesRegistry: lister, - - rateInterval: rateInterval, - } + }, lister } func (p *prometheusProvider) metricFor(value pmodel.SampleValue, groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error) { @@ -97,7 +95,7 @@ func (p *prometheusProvider) metricFor(value pmodel.SampleValue, groupResource s }, nil } -func (p *prometheusProvider) metricsFor(valueSet pmodel.Vector, info provider.MetricInfo, list runtime.Object) (*custom_metrics.MetricValueList, error) { +func (p *prometheusProvider) metricsFor(valueSet pmodel.Vector, info provider.CustomMetricInfo, list runtime.Object) (*custom_metrics.MetricValueList, error) { if !apimeta.IsListType(list) { return nil, apierr.NewInternalError(fmt.Errorf("result of label selector list operation was not a list")) } @@ -131,30 +129,14 @@ func (p *prometheusProvider) metricsFor(valueSet pmodel.Vector, info provider.Me }, nil } -func (p *prometheusProvider) buildQuery(info provider.MetricInfo, namespace string, names ...string) (pmodel.Vector, error) { - kind, baseQuery, groupBy, found := p.QueryForMetric(info, namespace, names...) +func (p *prometheusProvider) buildQuery(info provider.CustomMetricInfo, namespace string, names ...string) (pmodel.Vector, error) { + query, found := p.QueryForMetric(info, namespace, names...) if !found { return nil, provider.NewMetricNotFoundError(info.GroupResource, info.Metric) } - fullQuery := baseQuery - switch kind { - case CounterSeries: - fullQuery = prom.Selector(fmt.Sprintf("rate(%s[%s])", baseQuery, pmodel.Duration(p.rateInterval).String())) - case SecondsCounterSeries: - // TODO: futher modify for seconds? - fullQuery = prom.Selector(prom.Selector(fmt.Sprintf("rate(%s[%s])", baseQuery, pmodel.Duration(p.rateInterval).String()))) - } - - // NB: too small of a rate interval will return no results... - - // sum over all other dimensions of this query (e.g. if we select on route, sum across all pods, - // but if we select on pods, sum across all routes), and split by the dimension of our resource - // TODO: return/populate the by list in SeriesForMetric - fullQuery = prom.Selector(fmt.Sprintf("sum(%s) by (%s)", fullQuery, groupBy)) - // TODO: use an actual context - queryResults, err := p.promClient.Query(context.Background(), pmodel.Now(), fullQuery) + queryResults, err := p.promClient.Query(context.TODO(), pmodel.Now(), query) if err != nil { glog.Errorf("unable to fetch metrics from prometheus: %v", err) // don't leak implementation details to the user @@ -169,7 +151,7 @@ func (p *prometheusProvider) buildQuery(info provider.MetricInfo, namespace stri return *queryResults.Vector, nil } -func (p *prometheusProvider) getSingle(info provider.MetricInfo, namespace, name string) (*custom_metrics.MetricValue, error) { +func (p *prometheusProvider) getSingle(info provider.CustomMetricInfo, namespace, name string) (*custom_metrics.MetricValue, error) { queryResults, err := p.buildQuery(info, namespace, name) if err != nil { return nil, err @@ -197,24 +179,25 @@ func (p *prometheusProvider) getSingle(info provider.MetricInfo, namespace, name return p.metricFor(resultValue, info.GroupResource, "", name, info.Metric) } -func (p *prometheusProvider) getMultiple(info provider.MetricInfo, namespace string, selector labels.Selector) (*custom_metrics.MetricValueList, error) { - // construct a client to list the names of objects matching the label selector - client, err := p.kubeClient.ClientForGroupVersionResource(info.GroupResource.WithVersion("")) +func (p *prometheusProvider) getMultiple(info provider.CustomMetricInfo, namespace string, selector labels.Selector) (*custom_metrics.MetricValueList, error) { + fullResources, err := p.mapper.ResourcesFor(info.GroupResource.WithVersion("")) + if err == nil && len(fullResources) == 0 { + err = fmt.Errorf("no fully versioned resources known for group-resource %v", info.GroupResource) + } if err != nil { - glog.Errorf("unable to construct dynamic client to list matching resource names: %v", err) + glog.Errorf("unable to find preferred version to list matching resource names: %v", err) // don't leak implementation details to the user return nil, apierr.NewInternalError(fmt.Errorf("unable to list matching resources")) } - - // we can construct a this APIResource ourself, since the dynamic client only uses Name and Namespaced - apiRes := &metav1.APIResource{ - Name: info.GroupResource.Resource, - Namespaced: info.Namespaced, + var client dynamic.ResourceInterface + if namespace != "" { + client = p.kubeClient.Resource(fullResources[0]).Namespace(namespace) + } else { + client = p.kubeClient.Resource(fullResources[0]) } // actually list the objects matching the label selector - matchingObjectsRaw, err := client.Resource(apiRes, namespace). - List(metav1.ListOptions{LabelSelector: selector.String()}) + matchingObjectsRaw, err := client.List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { glog.Errorf("unable to list matching resource names: %v", err) // don't leak implementation details to the user @@ -243,7 +226,7 @@ func (p *prometheusProvider) getMultiple(info provider.MetricInfo, namespace str } func (p *prometheusProvider) GetRootScopedMetricByName(groupResource schema.GroupResource, name string, metricName string) (*custom_metrics.MetricValue, error) { - info := provider.MetricInfo{ + info := provider.CustomMetricInfo{ GroupResource: groupResource, Metric: metricName, Namespaced: false, @@ -253,7 +236,7 @@ func (p *prometheusProvider) GetRootScopedMetricByName(groupResource schema.Grou } func (p *prometheusProvider) GetRootScopedMetricBySelector(groupResource schema.GroupResource, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error) { - info := provider.MetricInfo{ + info := provider.CustomMetricInfo{ GroupResource: groupResource, Metric: metricName, Namespaced: false, @@ -262,7 +245,7 @@ func (p *prometheusProvider) GetRootScopedMetricBySelector(groupResource schema. } func (p *prometheusProvider) GetNamespacedMetricByName(groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error) { - info := provider.MetricInfo{ + info := provider.CustomMetricInfo{ GroupResource: groupResource, Metric: metricName, Namespaced: true, @@ -272,7 +255,7 @@ func (p *prometheusProvider) GetNamespacedMetricByName(groupResource schema.Grou } func (p *prometheusProvider) GetNamespacedMetricBySelector(groupResource schema.GroupResource, namespace string, selector labels.Selector, metricName string) (*custom_metrics.MetricValueList, error) { - info := provider.MetricInfo{ + info := provider.CustomMetricInfo{ GroupResource: groupResource, Metric: metricName, Namespaced: true, @@ -285,6 +268,7 @@ type cachingMetricsLister struct { promClient prom.Client updateInterval time.Duration + namers []MetricNamer } func (l *cachingMetricsLister) Run() { @@ -299,20 +283,65 @@ func (l *cachingMetricsLister) RunUntil(stopChan <-chan struct{}) { }, l.updateInterval, stopChan) } +type selectorSeries struct { + selector prom.Selector + series []prom.Series +} + func (l *cachingMetricsLister) updateMetrics() error { startTime := pmodel.Now().Add(-1 * l.updateInterval) - sels := l.Selectors() + // don't do duplicate queries when it's just the matchers that change + seriesCacheByQuery := make(map[prom.Selector][]prom.Series) - // TODO: use an actual context here - series, err := l.promClient.Series(context.Background(), pmodel.Interval{startTime, 0}, sels...) - if err != nil { - return fmt.Errorf("unable to update list of all available metrics: %v", err) + // these can take a while on large clusters, so launch in parallel + // and don't duplicate + selectors := make(map[prom.Selector]struct{}) + selectorSeriesChan := make(chan selectorSeries, len(l.namers)) + errs := make(chan error, len(l.namers)) + for _, namer := range l.namers { + sel := namer.Selector() + if _, ok := selectors[sel]; ok { + errs <- nil + selectorSeriesChan <- selectorSeries{} + continue + } + selectors[sel] = struct{}{} + go func() { + series, err := l.promClient.Series(context.TODO(), pmodel.Interval{startTime, 0}, sel) + if err != nil { + errs <- fmt.Errorf("unable to fetch metrics for query %q: %v", sel, err) + return + } + errs <- nil + selectorSeriesChan <- selectorSeries{ + selector: sel, + series: series, + } + }() } - glog.V(10).Infof("Set available metric list from Prometheus to: %v", series) + // iterate through, blocking until we've got all results + for range l.namers { + if err := <-errs; err != nil { + return fmt.Errorf("unable to update list of all metrics: %v", err) + } + if ss := <-selectorSeriesChan; ss.series != nil { + seriesCacheByQuery[ss.selector] = ss.series + } + } + close(errs) - l.SetSeries(series) + newSeries := make([][]prom.Series, len(l.namers)) + for i, namer := range l.namers { + series, cached := seriesCacheByQuery[namer.Selector()] + if !cached { + return fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", namer.Selector()) + } + newSeries[i] = namer.FilterSeries(series) + } - return nil + glog.V(10).Infof("Set available metric list from Prometheus to: %v", newSeries) + + return l.SetSeries(newSeries, l.namers) } diff --git a/pkg/custom-provider/provider_test.go b/pkg/custom-provider/provider_test.go index b5a756e0..a62bc3ac 100644 --- a/pkg/custom-provider/provider_test.go +++ b/pkg/custom-provider/provider_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" fakedyn "k8s.io/client-go/dynamic/fake" + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" pmodel "github.com/prometheus/common/model" ) @@ -86,20 +87,20 @@ func (c *fakePromClient) QueryRange(_ context.Context, r prom.Range, query prom. return prom.QueryResult{}, nil } -func setupPrometheusProvider(t *testing.T, stopCh <-chan struct{}) (provider.CustomMetricsProvider, *fakePromClient) { +func setupPrometheusProvider(t *testing.T) (provider.CustomMetricsProvider, *fakePromClient) { fakeProm := &fakePromClient{} - fakeKubeClient := &fakedyn.FakeClientPool{} + fakeKubeClient := &fakedyn.FakeDynamicClient{} - prov := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, "", fakeProviderUpdateInterval, 1*time.Minute, stopCh) + cfg := config.DefaultConfig(1*time.Minute, "") + namers, err := NamersFromConfig(cfg, restMapper()) + require.NoError(t, err) + + prov, _ := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, namers, fakeProviderUpdateInterval) containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", "")) namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*")) fakeProm.series = map[prom.Selector][]prom.Series{ containerSel: { - { - Name: "container_actually_gauge_seconds_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, { Name: "container_some_usage", Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, @@ -130,28 +131,24 @@ func setupPrometheusProvider(t *testing.T, stopCh <-chan struct{}) (provider.Cus func TestListAllMetrics(t *testing.T) { // setup - stopCh := make(chan struct{}) - defer close(stopCh) - prov, fakeProm := setupPrometheusProvider(t, stopCh) + prov, fakeProm := setupPrometheusProvider(t) // assume we have no updates require.Len(t, prov.ListAllMetrics(), 0, "assume: should have no metrics updates at the start") // set the acceptible interval (now until the next update, with a bit of wiggle room) - startTime := pmodel.Now() - endTime := startTime.Add(fakeProviderUpdateInterval + fakeProviderUpdateInterval/10) - fakeProm.acceptibleInterval = pmodel.Interval{Start: startTime, End: endTime} + startTime := pmodel.Now().Add(-1*fakeProviderUpdateInterval - fakeProviderUpdateInterval/10) + fakeProm.acceptibleInterval = pmodel.Interval{Start: startTime, End: 0} - // wait one update interval (with a bit of wiggle room) - time.Sleep(fakeProviderUpdateInterval + fakeProviderUpdateInterval/10) + // update the metrics (without actually calling RunUntil, so we can avoid timing issues) + lister := prov.(*prometheusProvider).SeriesRegistry.(*cachingMetricsLister) + require.NoError(t, lister.updateMetrics()) // list/sort the metrics actualMetrics := prov.ListAllMetrics() sort.Sort(metricInfoSorter(actualMetrics)) - expectedMetrics := []provider.MetricInfo{ - {schema.GroupResource{Resource: "pods"}, true, "actually_gauge"}, - {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + expectedMetrics := []provider.CustomMetricInfo{ {schema.GroupResource{Resource: "services"}, true, "ingress_hits"}, {schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"}, {schema.GroupResource{Resource: "pods"}, true, "ingress_hits"}, @@ -160,6 +157,8 @@ func TestListAllMetrics(t *testing.T) { {schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"}, {schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"}, {schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_usage"}, + {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, } sort.Sort(metricInfoSorter(expectedMetrics)) diff --git a/pkg/custom-provider/series_registry.go b/pkg/custom-provider/series_registry.go new file mode 100644 index 00000000..a4701cf4 --- /dev/null +++ b/pkg/custom-provider/series_registry.go @@ -0,0 +1,198 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + "sync" + + "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" + apimeta "k8s.io/apimachinery/pkg/api/meta" + + prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + "github.com/golang/glog" + pmodel "github.com/prometheus/common/model" +) + +// NB: container metrics sourced from cAdvisor don't consistently follow naming conventions, +// so we need to whitelist them and handle them on a case-by-case basis. Metrics ending in `_total` +// *should* be counters, but may actually be guages in this case. + +// SeriesType represents the kind of series backing a metric. +type SeriesType int + +const ( + CounterSeries SeriesType = iota + SecondsCounterSeries + GaugeSeries +) + +// SeriesRegistry provides conversions between Prometheus series and MetricInfo +type SeriesRegistry interface { + // SetSeries replaces the known series in this registry. + // Each slice in series should correspond to a MetricNamer in namers. + SetSeries(series [][]prom.Series, namers []MetricNamer) error + // ListAllMetrics lists all metrics known to this registry + ListAllMetrics() []provider.CustomMetricInfo + // SeriesForMetric looks up the minimum required series information to make a query for the given metric + // against the given resource (namespace may be empty for non-namespaced resources) + QueryForMetric(info provider.CustomMetricInfo, namespace string, resourceNames ...string) (query prom.Selector, found bool) + // MatchValuesToNames matches result values to resource names for the given metric and value set + MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) +} + +type seriesInfo struct { + // seriesName is the name of the corresponding Prometheus series + seriesName string + + // namer is the MetricNamer used to name this series + namer MetricNamer +} + +// overridableSeriesRegistry is a basic SeriesRegistry +type basicSeriesRegistry struct { + mu sync.RWMutex + + // info maps metric info to information about the corresponding series + info map[provider.CustomMetricInfo]seriesInfo + // metrics is the list of all known metrics + metrics []provider.CustomMetricInfo + + mapper apimeta.RESTMapper +} + +func (r *basicSeriesRegistry) SetSeries(newSeriesSlices [][]prom.Series, namers []MetricNamer) error { + if len(newSeriesSlices) != len(namers) { + return fmt.Errorf("need one set of series per namer") + } + + newInfo := make(map[provider.CustomMetricInfo]seriesInfo) + for i, newSeries := range newSeriesSlices { + namer := namers[i] + for _, series := range newSeries { + // TODO: warn if it doesn't match any resources + resources, namespaced := namer.ResourcesForSeries(series) + name, err := namer.MetricNameForSeries(series) + if err != nil { + glog.Errorf("unable to name series %q, skipping: %v", series.String(), err) + continue + } + for _, resource := range resources { + info := provider.CustomMetricInfo{ + GroupResource: resource, + Namespaced: namespaced, + Metric: name, + } + + // namespace metrics aren't counted as namespaced + if resource == nsGroupResource { + info.Namespaced = false + } + + // we don't need to re-normalize, because the metric namer should have already normalized for us + newInfo[info] = seriesInfo{ + seriesName: series.Name, + namer: namer, + } + } + } + } + + // regenerate metrics + newMetrics := make([]provider.CustomMetricInfo, 0, len(newInfo)) + for info := range newInfo { + newMetrics = append(newMetrics, info) + } + + r.mu.Lock() + defer r.mu.Unlock() + + r.info = newInfo + r.metrics = newMetrics + + return nil +} + +func (r *basicSeriesRegistry) ListAllMetrics() []provider.CustomMetricInfo { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.metrics +} + +func (r *basicSeriesRegistry) QueryForMetric(metricInfo provider.CustomMetricInfo, namespace string, resourceNames ...string) (prom.Selector, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + if len(resourceNames) == 0 { + glog.Errorf("no resource names requested while producing a query for metric %s", metricInfo.String()) + return "", false + } + + metricInfo, _, err := metricInfo.Normalized(r.mapper) + if err != nil { + glog.Errorf("unable to normalize group resource while producing a query: %v", err) + return "", false + } + + info, infoFound := r.info[metricInfo] + if !infoFound { + glog.V(10).Infof("metric %v not registered", metricInfo) + return "", false + } + + query, err := info.namer.QueryForSeries(info.seriesName, metricInfo.GroupResource, namespace, resourceNames...) + if err != nil { + glog.Errorf("unable to construct query for metric %s: %v", metricInfo.String(), err) + return "", false + } + + return query, true +} + +func (r *basicSeriesRegistry) MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + metricInfo, _, err := metricInfo.Normalized(r.mapper) + if err != nil { + glog.Errorf("unable to normalize group resource while matching values to names: %v", err) + return nil, false + } + + info, infoFound := r.info[metricInfo] + if !infoFound { + return nil, false + } + + resourceLbl, err := info.namer.LabelForResource(metricInfo.GroupResource) + if err != nil { + glog.Errorf("unable to construct resource label for metric %s: %v", metricInfo.String(), err) + return nil, false + } + + res := make(map[string]pmodel.SampleValue, len(values)) + for _, val := range values { + if val == nil { + // skip empty values + continue + } + res[string(val.Metric[resourceLbl])] = val.Value + } + + return res, true +} diff --git a/pkg/custom-provider/metric_namer_test.go b/pkg/custom-provider/series_registry_test.go similarity index 50% rename from pkg/custom-provider/metric_namer_test.go rename to pkg/custom-provider/series_registry_test.go index c01e25e1..ed5eb11f 100644 --- a/pkg/custom-provider/metric_namer_test.go +++ b/pkg/custom-provider/series_registry_test.go @@ -19,6 +19,7 @@ package provider import ( "sort" "testing" + "time" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" pmodel "github.com/prometheus/common/model" @@ -29,13 +30,14 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" ) // restMapper creates a RESTMapper with just the types we need for // these tests. func restMapper() apimeta.RESTMapper { - mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{coreapi.SchemeGroupVersion}, apimeta.InterfacesForUnstructured) + mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{coreapi.SchemeGroupVersion}) mapper.Add(coreapi.SchemeGroupVersion.WithKind("Pod"), apimeta.RESTScopeNamespace) mapper.Add(coreapi.SchemeGroupVersion.WithKind("Service"), apimeta.RESTScopeNamespace) @@ -49,122 +51,46 @@ func restMapper() apimeta.RESTMapper { return mapper } -func setupMetricNamer(t *testing.T) *metricNamer { - return &metricNamer{ - overrides: map[string]seriesSpec{ - "container_actually_gauge_seconds_total": { - metricName: "actually_gauge", - kind: GaugeSeries, - }, - }, - labelPrefix: "kube_", - mapper: restMapper(), - } +func setupMetricNamer(t testing.TB) []MetricNamer { + cfg := config.DefaultConfig(1*time.Minute, "kube_") + namers, err := NamersFromConfig(cfg, restMapper()) + require.NoError(t, err) + return namers } -func TestMetricNamerContainerSeries(t *testing.T) { - testCases := []struct { - input prom.Series - outputMetricName string - outputInfo seriesInfo - }{ - { - input: prom.Series{ - Name: "container_actually_gauge_seconds_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - outputMetricName: "actually_gauge", - outputInfo: seriesInfo{ - baseSeries: prom.Series{Name: "container_actually_gauge_seconds_total"}, - kind: GaugeSeries, - isContainer: true, - }, - }, - { - input: prom.Series{ - Name: "container_some_usage", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - outputMetricName: "some_usage", - outputInfo: seriesInfo{ - baseSeries: prom.Series{Name: "container_some_usage"}, - kind: GaugeSeries, - isContainer: true, - }, - }, - { - input: prom.Series{ - Name: "container_some_count_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - outputMetricName: "some_count", - outputInfo: seriesInfo{ - baseSeries: prom.Series{Name: "container_some_count_total"}, - kind: CounterSeries, - isContainer: true, - }, - }, - { - input: prom.Series{ - Name: "container_some_time_seconds_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - outputMetricName: "some_time", - outputInfo: seriesInfo{ - baseSeries: prom.Series{Name: "container_some_time_seconds_total"}, - kind: SecondsCounterSeries, - isContainer: true, - }, - }, - } - - assert := assert.New(t) - - namer := setupMetricNamer(t) - resMap := map[provider.MetricInfo]seriesInfo{} - - for _, test := range testCases { - namer.processContainerSeries(test.input, resMap) - metric := provider.MetricInfo{ - Metric: test.outputMetricName, - GroupResource: schema.GroupResource{Resource: "pods"}, - Namespaced: true, - } - if assert.Contains(resMap, metric) { - assert.Equal(test.outputInfo, resMap[metric]) - } - } -} - -func TestSeriesRegistry(t *testing.T) { - assert := assert.New(t) - require := require.New(t) - - namer := setupMetricNamer(t) - registry := &basicSeriesRegistry{ - namer: *namer, - } - - inputSeries := []prom.Series{ - // container series - { - Name: "container_actually_gauge_seconds_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - { - Name: "container_some_usage", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, - { - Name: "container_some_count_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, +var seriesRegistryTestSeries = [][]prom.Series{ + // container series + { { Name: "container_some_time_seconds_total", Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, }, - // namespaced series - // a series that should turn into multiple metrics + }, + { + { + Name: "container_some_count_total", + Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, + }, + }, + { + { + Name: "container_some_usage", + Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, + }, + }, + { + // guage metrics + { + Name: "node_gigawatts", + Labels: pmodel.LabelSet{"kube_node": "somenode"}, + }, + { + Name: "service_proxy_packets", + Labels: pmodel.LabelSet{"kube_service": "somesvc", "kube_namespace": "somens"}, + }, + }, + { + // cumulative --> rate metrics { Name: "ingress_hits_total", Labels: pmodel.LabelSet{"kube_ingress": "someingress", "kube_service": "somesvc", "kube_pod": "backend1", "kube_namespace": "somens"}, @@ -174,191 +100,151 @@ func TestSeriesRegistry(t *testing.T) { Labels: pmodel.LabelSet{"kube_ingress": "someingress", "kube_service": "somesvc", "kube_pod": "backend2", "kube_namespace": "somens"}, }, { - Name: "service_proxy_packets", - Labels: pmodel.LabelSet{"kube_service": "somesvc", "kube_namespace": "somens"}, + Name: "volume_claims_total", + Labels: pmodel.LabelSet{"kube_persistentvolume": "somepv"}, }, + }, + { + // cumulative seconds --> rate metrics { Name: "work_queue_wait_seconds_total", Labels: pmodel.LabelSet{"kube_deployment": "somedep", "kube_namespace": "somens"}, }, - // non-namespaced series - { - Name: "node_gigawatts", - Labels: pmodel.LabelSet{"kube_node": "somenode"}, - }, - { - Name: "volume_claims_total", - Labels: pmodel.LabelSet{"kube_persistentvolume": "somepv"}, - }, { Name: "node_fan_seconds_total", Labels: pmodel.LabelSet{"kube_node": "somenode"}, }, - // unrelated series - { - Name: "admin_coffee_liters_total", - Labels: pmodel.LabelSet{"admin": "some-admin"}, - }, - { - Name: "admin_unread_emails", - Labels: pmodel.LabelSet{"admin": "some-admin"}, - }, - { - Name: "admin_reddit_seconds_total", - Labels: pmodel.LabelSet{"kube_admin": "some-admin"}, - }, + }, +} + +func TestSeriesRegistry(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + namers := setupMetricNamer(t) + registry := &basicSeriesRegistry{ + mapper: restMapper(), } // set up the registry - require.NoError(registry.SetSeries(inputSeries)) + require.NoError(registry.SetSeries(seriesRegistryTestSeries, namers)) // make sure each metric got registered and can form queries testCases := []struct { title string - info provider.MetricInfo + info provider.CustomMetricInfo namespace string resourceNames []string - expectedKind SeriesType - expectedQuery string - expectedGroupBy string + expectedQuery string }{ // container metrics - { - title: "container metrics overrides / single resource name", - info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "actually_gauge"}, - namespace: "somens", - resourceNames: []string{"somepod"}, - - expectedKind: GaugeSeries, - expectedQuery: "container_actually_gauge_seconds_total{pod_name=\"somepod\",container_name!=\"POD\",namespace=\"somens\"}", - expectedGroupBy: "pod_name", - }, { title: "container metrics gauge / multiple resource names", - info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"}, namespace: "somens", resourceNames: []string{"somepod1", "somepod2"}, - expectedKind: GaugeSeries, - expectedQuery: "container_some_usage{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}", - expectedGroupBy: "pod_name", + expectedQuery: "sum(container_some_usage{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}) by (pod_name)", }, { title: "container metrics counter", - info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"}, namespace: "somens", resourceNames: []string{"somepod1", "somepod2"}, - expectedKind: CounterSeries, - expectedQuery: "container_some_count_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}", - expectedGroupBy: "pod_name", + expectedQuery: "sum(rate(container_some_count_total{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}[1m])) by (pod_name)", }, { title: "container metrics seconds counter", - info: provider.MetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"}, namespace: "somens", resourceNames: []string{"somepod1", "somepod2"}, - expectedKind: SecondsCounterSeries, - expectedQuery: "container_some_time_seconds_total{pod_name=~\"somepod1|somepod2\",container_name!=\"POD\",namespace=\"somens\"}", - expectedGroupBy: "pod_name", + expectedQuery: "sum(rate(container_some_time_seconds_total{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}[1m])) by (pod_name)", }, // namespaced metrics { title: "namespaced metrics counter / multidimensional (service)", - info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "ingress_hits"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "service"}, true, "ingress_hits"}, namespace: "somens", resourceNames: []string{"somesvc"}, - expectedKind: CounterSeries, - expectedQuery: "ingress_hits_total{kube_service=\"somesvc\",kube_namespace=\"somens\"}", + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_service=\"somesvc\"}[1m])) by (kube_service)", }, { title: "namespaced metrics counter / multidimensional (ingress)", - info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingress"}, true, "ingress_hits"}, + info: provider.CustomMetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingress"}, true, "ingress_hits"}, namespace: "somens", resourceNames: []string{"someingress"}, - expectedKind: CounterSeries, - expectedQuery: "ingress_hits_total{kube_ingress=\"someingress\",kube_namespace=\"somens\"}", + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_ingress=\"someingress\"}[1m])) by (kube_ingress)", }, { title: "namespaced metrics counter / multidimensional (pod)", - info: provider.MetricInfo{schema.GroupResource{Resource: "pod"}, true, "ingress_hits"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pod"}, true, "ingress_hits"}, namespace: "somens", resourceNames: []string{"somepod"}, - expectedKind: CounterSeries, - expectedQuery: "ingress_hits_total{kube_pod=\"somepod\",kube_namespace=\"somens\"}", + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_pod=\"somepod\"}[1m])) by (kube_pod)", }, { title: "namespaced metrics gauge", - info: provider.MetricInfo{schema.GroupResource{Resource: "service"}, true, "service_proxy_packets"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "service"}, true, "service_proxy_packets"}, namespace: "somens", resourceNames: []string{"somesvc"}, - expectedKind: GaugeSeries, - expectedQuery: "service_proxy_packets{kube_service=\"somesvc\",kube_namespace=\"somens\"}", + expectedQuery: "sum(service_proxy_packets{kube_namespace=\"somens\",kube_service=\"somesvc\"}) by (kube_service)", }, { title: "namespaced metrics seconds counter", - info: provider.MetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployment"}, true, "work_queue_wait"}, + info: provider.CustomMetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployment"}, true, "work_queue_wait"}, namespace: "somens", resourceNames: []string{"somedep"}, - expectedKind: SecondsCounterSeries, - expectedQuery: "work_queue_wait_seconds_total{kube_deployment=\"somedep\",kube_namespace=\"somens\"}", + expectedQuery: "sum(rate(work_queue_wait_seconds_total{kube_namespace=\"somens\",kube_deployment=\"somedep\"}[1m])) by (kube_deployment)", }, // non-namespaced series { title: "root scoped metrics gauge", - info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_gigawatts"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "node"}, false, "node_gigawatts"}, resourceNames: []string{"somenode"}, - expectedKind: GaugeSeries, - expectedQuery: "node_gigawatts{kube_node=\"somenode\"}", + expectedQuery: "sum(node_gigawatts{kube_node=\"somenode\"}) by (kube_node)", }, { title: "root scoped metrics counter", - info: provider.MetricInfo{schema.GroupResource{Resource: "persistentvolume"}, false, "volume_claims"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "persistentvolume"}, false, "volume_claims"}, resourceNames: []string{"somepv"}, - expectedKind: CounterSeries, - expectedQuery: "volume_claims_total{kube_persistentvolume=\"somepv\"}", + expectedQuery: "sum(rate(volume_claims_total{kube_persistentvolume=\"somepv\"}[1m])) by (kube_persistentvolume)", }, { title: "root scoped metrics seconds counter", - info: provider.MetricInfo{schema.GroupResource{Resource: "node"}, false, "node_fan"}, + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "node"}, false, "node_fan"}, resourceNames: []string{"somenode"}, - expectedKind: SecondsCounterSeries, - expectedQuery: "node_fan_seconds_total{kube_node=\"somenode\"}", + expectedQuery: "sum(rate(node_fan_seconds_total{kube_node=\"somenode\"}[1m])) by (kube_node)", }, } for _, testCase := range testCases { - outputKind, outputQuery, groupBy, found := registry.QueryForMetric(testCase.info, testCase.namespace, testCase.resourceNames...) + outputQuery, found := registry.QueryForMetric(testCase.info, testCase.namespace, testCase.resourceNames...) if !assert.True(found, "%s: metric %v should available", testCase.title, testCase.info) { continue } - assert.Equal(testCase.expectedKind, outputKind, "%s: metric %v should have had the right series type", testCase.title, testCase.info) assert.Equal(prom.Selector(testCase.expectedQuery), outputQuery, "%s: metric %v should have produced the correct query for %v in namespace %s", testCase.title, testCase.info, testCase.resourceNames, testCase.namespace) - - expectedGroupBy := testCase.expectedGroupBy - if expectedGroupBy == "" { - expectedGroupBy = registry.namer.labelPrefix + testCase.info.GroupResource.Resource - } - assert.Equal(expectedGroupBy, groupBy, "%s: metric %v should have produced the correct groupBy clause", testCase.title, testCase.info) } allMetrics := registry.ListAllMetrics() - expectedMetrics := []provider.MetricInfo{ - {schema.GroupResource{Resource: "pods"}, true, "actually_gauge"}, - {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + expectedMetrics := []provider.CustomMetricInfo{ {schema.GroupResource{Resource: "pods"}, true, "some_count"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_count"}, {schema.GroupResource{Resource: "pods"}, true, "some_time"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_time"}, + {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_usage"}, {schema.GroupResource{Resource: "services"}, true, "ingress_hits"}, {schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"}, {schema.GroupResource{Resource: "pods"}, true, "ingress_hits"}, @@ -379,8 +265,32 @@ func TestSeriesRegistry(t *testing.T) { assert.Equal(expectedMetrics, allMetrics, "should have listed all expected metrics") } -// metricInfoSorter is a sort.Interface for sorting provider.MetricInfos -type metricInfoSorter []provider.MetricInfo +func BenchmarkSetSeries(b *testing.B) { + namers := setupMetricNamer(b) + registry := &basicSeriesRegistry{ + mapper: restMapper(), + } + + numDuplicates := 10000 + newSeriesSlices := make([][]prom.Series, len(seriesRegistryTestSeries)) + for i, seriesSlice := range seriesRegistryTestSeries { + newSlice := make([]prom.Series, len(seriesSlice)*numDuplicates) + for j, series := range seriesSlice { + for k := 0; k < numDuplicates; k++ { + newSlice[j*numDuplicates+k] = series + } + } + newSeriesSlices[i] = newSlice + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + registry.SetSeries(newSeriesSlices, namers) + } +} + +// metricInfoSorter is a sort.Interface for sorting provider.CustomMetricInfos +type metricInfoSorter []provider.CustomMetricInfo func (s metricInfoSorter) Len() int { return len(s)