diff --git a/Gopkg.lock b/Gopkg.lock index 9b9b5ce2..4c5103ff 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -10,23 +10,26 @@ [[projects]] name = "github.com/NYTimes/gziphandler" packages = ["."] - revision = "56545f4a5d46df9a6648819d1664c3a03a13ffdb" + revision = "2600fb119af974220d3916a5916d6e31176aac1b" + version = "v1.0.1" [[projects]] name = "github.com/PuerkitoBio/purell" packages = ["."] - revision = "8a290539e2e8629dbc4e6bad948158f790ec31f4" - version = "v1.0.0" + revision = "0bcb03f4b4d0a9428594752bd2a3b9aa0a9d4bd4" + version = "v1.1.0" [[projects]] + branch = "master" name = "github.com/PuerkitoBio/urlesc" packages = ["."] - revision = "5bd2802263f21d8788851d5305584c82a5c75d7e" + revision = "de5bf2ad457846296e2031421a34e2568e304e35" [[projects]] + branch = "master" name = "github.com/beorn7/perks" packages = ["quantile"] - revision = "3ac7bf7a47d159a033b107610db8a1b6575507a4" + revision = "3a771d992973f24aa725d07868b467d1ddfceafb" [[projects]] name = "github.com/coreos/etcd" @@ -37,28 +40,27 @@ "etcdserver/api/v3rpc/rpctypes", "etcdserver/etcdserverpb", "mvcc/mvccpb", - "pkg/fileutil", "pkg/pathutil", + "pkg/srv", "pkg/tlsutil", "pkg/transport", - "pkg/types" + "pkg/types", + "version" ] - revision = "0520cb9304cb2385f7e72b8bc02d6e4d3257158a" - version = "v3.1.10" + revision = "33245c6b5b49130ca99280408fadfab01aac0e48" + version = "v3.3.8" + +[[projects]] + name = "github.com/coreos/go-semver" + packages = ["semver"] + revision = "8ab6407b697782a06568d4b7f1db25550ec2e4c6" + version = "v0.2.0" [[projects]] name = "github.com/coreos/go-systemd" - packages = [ - "daemon", - "journal" - ] - revision = "48702e0da86bd25e76cfef347e2adeb434a0d0a6" - version = "v14" - -[[projects]] - name = "github.com/coreos/pkg" - packages = ["capnslog"] - revision = "fa29b1d70f0beaddd4c7021607cc3c3be8ce94b8" + packages = ["daemon"] + revision = "39ca1b05acc7ad1220e09f133283b8859a8b71ab" + version = "v17" [[projects]] name = "github.com/davecgh/go-spew" @@ -67,20 +69,23 @@ version = "v1.1.0" [[projects]] + branch = "master" name = "github.com/directxman12/k8s-prometheus-adapter" packages = [ "cmd/adapter/app", + "cmd/config-gen/utils", "pkg/client", "pkg/client/metrics", + "pkg/config", "pkg/custom-provider" ] - revision = "8dd527d821f46b9e3961f3f7e5fdf5131d95681c" - version = "v0.2.0" + revision = "7b606a79fc2fdc4246e455e8f28503e0b4807a92" [[projects]] name = "github.com/elazarl/go-bindata-assetfs" packages = ["."] - revision = "3dcc96556217539f50599357fb481ac0dc7439b9" + revision = "30f82fa23fd844bd5bb1e5f216db87fd77b5eb43" + version = "v1.0.0" [[projects]] name = "github.com/emicklei/go-restful" @@ -88,7 +93,8 @@ ".", "log" ] - revision = "ff4f55a206334ef123e4f79bbf348980da81ca46" + revision = "3658237ded108b4134956c1b3050349d93e7b895" + version = "v2.7.1" [[projects]] name = "github.com/emicklei/go-restful-swagger12" @@ -99,67 +105,79 @@ [[projects]] name = "github.com/evanphx/json-patch" packages = ["."] - revision = "944e07253867aacae43c04b2e6a239005443f33a" + revision = "afac545df32f2287a079e2dfb7ba2745a643747e" + version = "v3.0.0" [[projects]] name = "github.com/ghodss/yaml" packages = ["."] - revision = "73d445a93680fa1a78ae23a5839bad48f32ba1ee" + revision = "0ca9ea5df5451ffdf184b4428c902747c2c11cd7" + version = "v1.0.0" [[projects]] + branch = "master" name = "github.com/go-openapi/jsonpointer" packages = ["."] - revision = "46af16f9f7b149af66e5d1bd010e3574dc06de98" + revision = "3a0015ad55fa9873f41605d3e8f28cd279c32ab2" [[projects]] + branch = "master" name = "github.com/go-openapi/jsonreference" packages = ["."] - revision = "13c6e3589ad90f49bd3e3bbe2c2cb3d7a4142272" + revision = "3fb327e6747da3043567ee86abd02bb6376b6be2" [[projects]] + branch = "master" name = "github.com/go-openapi/spec" packages = ["."] - revision = "6aced65f8501fe1217321abf0749d354824ba2ff" + revision = "bcff419492eeeb01f76e77d2ebc714dc97b607f5" [[projects]] + branch = "master" name = "github.com/go-openapi/swag" packages = ["."] - revision = "1d0bd113de87027671077d3c71eb3ac5d7dbba72" + revision = "811b1089cde9dad18d4d0c2d09fbdbf28dbd27a5" [[projects]] name = "github.com/gogo/protobuf" packages = [ + "gogoproto", "proto", + "protoc-gen-gogo/descriptor", "sortkeys" ] - revision = "c0656edd0d9eab7c66d1eb0c568f9039345796f7" + revision = "1adfc126b41513cc696b209667c8656ea7aac67c" + version = "v1.0.0" [[projects]] + branch = "master" name = "github.com/golang/glog" packages = ["."] - revision = "44145f04b68cf362d9c4df2182967c2275eaefed" + revision = "23def4e6c14b4da8ac2ed8007337bc5eb5007998" [[projects]] name = "github.com/golang/protobuf" packages = [ - "jsonpb", "proto", "ptypes", "ptypes/any", "ptypes/duration", "ptypes/timestamp" ] - revision = "4bd1920723d7b7c925de087aa32e2187708897f7" + revision = "b4deda0973fb4c70b50d226b1af49f3da59f5265" + version = "v1.1.0" [[projects]] + branch = "master" name = "github.com/google/btree" packages = ["."] - revision = "7d79101e329e5a3adf994758c578dab82b90c017" + revision = "e89373fe6b4a7413d7acd6da1725b83ef713e6e4" [[projects]] + branch = "master" name = "github.com/google/gofuzz" packages = ["."] - revision = "44d81051d367757e1c7c6a5a86423ece9afcf63c" + revision = "24818f796faf91cd76ec7bddd72458fbced7a6c1" [[projects]] name = "github.com/googleapis/gnostic" @@ -168,48 +186,32 @@ "compiler", "extensions" ] - revision = "0c5108395e2debce0d731cf0287ddf7242066aba" + revision = "7c663266750e7d82587642f65e60bc4083f1f84e" + version = "v0.2.0" [[projects]] + branch = "master" name = "github.com/gregjones/httpcache" packages = [ ".", "diskcache" ] - revision = "787624de3eb7bd915c329cba748687a3b22666a6" - -[[projects]] - name = "github.com/grpc-ecosystem/go-grpc-prometheus" - packages = ["."] - revision = "2500245aa6110c562d17020fb31a2c133d737799" - -[[projects]] - name = "github.com/grpc-ecosystem/grpc-gateway" - packages = [ - "runtime", - "runtime/internal", - "utilities" - ] - revision = "84398b94e188ee336f307779b57b3aa91af7063c" + revision = "9cad4c3443a7200dd6400aef47183728de563a38" [[projects]] + branch = "master" name = "github.com/hashicorp/golang-lru" packages = [ ".", "simplelru" ] - revision = "a0d98a5f288019575c6d1f4bb1573fef2d1fcdc4" - -[[projects]] - branch = "master" - name = "github.com/howeyc/gopass" - packages = ["."] - revision = "bf9dde6d0d2c004a008c27aaee91170c786f6db8" + revision = "0fb14efe8c47ae851c0034ed7a448854d3d34cf3" [[projects]] name = "github.com/imdario/mergo" packages = ["."] - revision = "6633656539c1639d9d78127b7d47c622b5d7b6dc" + revision = "9316a62528ac99aaecb4e47eadd6dc8aa6533d58" + version = "v0.3.5" [[projects]] name = "github.com/inconshreveable/mousetrap" @@ -220,16 +222,11 @@ [[projects]] name = "github.com/json-iterator/go" packages = ["."] - revision = "36b14963da70d11297d313183d7e6388c8510e1e" - version = "1.0.0" - -[[projects]] - name = "github.com/juju/ratelimit" - packages = ["."] - revision = "5b9ff866471762aa2ab2dced63c9fb6f53921342" - version = "1.0" + revision = "ca39e5af3ece67bbcda3d0f4f56a8e24d9f2dad4" + version = "1.1.3" [[projects]] + branch = "master" name = "github.com/kubernetes-incubator/custom-metrics-apiserver" packages = [ "pkg/apiserver", @@ -240,32 +237,41 @@ "pkg/registry/custom_metrics", "pkg/registry/external_metrics" ] - revision = "e61f72fec56ab519d74ebd396cd3fcf31b084558" + revision = "d8f23423aa1d0ff2bc9656da863d721725b3c68a" [[projects]] + branch = "master" name = "github.com/mailru/easyjson" packages = [ "buffer", "jlexer", "jwriter" ] - revision = "d5b7844b561a7bc640052f1b935f7b800330d7e0" + revision = "3fdea8d05856a0c8df22ed4bc71b3219245e4485" [[projects]] name = "github.com/matttproud/golang_protobuf_extensions" packages = ["pbutil"] - revision = "fc2b8d3a73c4867e51861bbdd5ae3c1f0869dd6a" + revision = "c12348ce28de40eed0136aa2b644d0ee0650e56c" + version = "v1.0.1" [[projects]] - branch = "master" - name = "github.com/mxk/go-flowrate" - packages = ["flowrate"] - revision = "cca7078d478f8520f85629ad7c68962d31ed7682" + name = "github.com/modern-go/concurrent" + packages = ["."] + revision = "bacd9c7ef1dd9b15be4a9909b8ac7a4e313eec94" + version = "1.0.3" + +[[projects]] + name = "github.com/modern-go/reflect2" + packages = ["."] + revision = "1df9eeb2bb81f327b96228865c5687bc2194af3f" + version = "1.0.0" [[projects]] name = "github.com/pborman/uuid" packages = ["."] - revision = "ca53cad383cad2479bbba7f7a1a05797ec1386e4" + revision = "e790cca94e6cc75c7064b1332e63811d4aae1a53" + version = "v1.1" [[projects]] branch = "master" @@ -279,11 +285,6 @@ revision = "5f041e8faa004a95c88a202771f4cc3e991971e6" version = "v2.0.1" -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "a22138067af1c4942683050411a841ade67fe1eb" - [[projects]] name = "github.com/pmezard/go-difflib" packages = ["difflib"] @@ -293,39 +294,47 @@ [[projects]] name = "github.com/prometheus/client_golang" packages = ["prometheus"] - revision = "e7e903064f5e9eb5da98208bae10b475d4db0f8c" + revision = "c5b7fccd204277076155f10851dad72b76a49317" + version = "v0.8.0" [[projects]] + branch = "master" name = "github.com/prometheus/client_model" packages = ["go"] - revision = "fa8ad6fec33561be4280a8f0514318c79d7f6cb6" + revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c" [[projects]] + branch = "master" name = "github.com/prometheus/common" packages = [ "expfmt", "internal/bitbucket.org/ww/goautoneg", "model" ] - revision = "13ba4ddd0caa9c28ca7b7bffe1dfa9ed8d5ef207" + revision = "7600349dcfe1abd18d72d3a1770870d9800a7801" [[projects]] + branch = "master" name = "github.com/prometheus/procfs" packages = [ ".", + "internal/util", + "nfs", "xfs" ] - revision = "65c1f6f8f0fc1e2185eb9863a3bc751496404259" + revision = "7d6f385de8bea29190f15ba9931442a0eaef9af7" [[projects]] name = "github.com/spf13/cobra" packages = ["."] - revision = "f62e98d28ab7ad31d707ba837a966378465c7b57" + revision = "ef82de70bb3f60c65fb8eebacbb2d122ef517385" + version = "v0.0.3" [[projects]] name = "github.com/spf13/pflag" packages = ["."] - revision = "9ff6c6923cfffbcd502984b8e0c80539a94968b7" + revision = "583c0c0531f06d5278b7d917446061adc344b5cd" + version = "v1.0.1" [[projects]] name = "github.com/stretchr/testify" @@ -339,50 +348,51 @@ [[projects]] name = "github.com/ugorji/go" packages = ["codec"] - revision = "ded73eae5db7e7a0ef6f55aace87a2873c5d2b74" + revision = "b4c50a2b199d93b13dc15e78929cfb23bfdf21ab" + version = "v1.1.1" [[projects]] + branch = "master" name = "golang.org/x/crypto" packages = ["ssh/terminal"] - revision = "81e90905daefcd6fd217b62423c0908922eadb30" + revision = "a49355c7e3f8fe157a85be2f77e6e269a0f89602" [[projects]] + branch = "master" name = "golang.org/x/net" packages = [ "context", - "html", - "html/atom", + "http/httpguts", "http2", "http2/hpack", "idna", "internal/timeseries", - "lex/httplex", "trace", "websocket" ] - revision = "1c05540f6879653db88113bc4a2b70aec4bd491f" + revision = "afe8f62b1d6bbd81f31868121a50b06d8188e1f9" [[projects]] + branch = "master" name = "golang.org/x/sys" packages = [ "unix", "windows" ] - revision = "7ddbeae9ae08c6a06a59597f0c9edbc5ff2444ce" + revision = "63fc586f45fe72d95d5240a5d5eb95e6503907d3" [[projects]] name = "golang.org/x/text" packages = [ - "cases", - "internal", + "collate", + "collate/build", + "internal/colltab", "internal/gen", "internal/tag", "internal/triegen", "internal/ucd", "language", - "runes", "secure/bidirule", - "secure/precis", "transform", "unicode/bidi", "unicode/cldr", @@ -390,54 +400,79 @@ "unicode/rangetable", "width" ] - revision = "b19bf474d317b857955b12035d2c5acb57ce8b01" + revision = "f21a4dfb5e38f5895301dc265a8def02365cc3d0" + version = "v0.3.0" [[projects]] + branch = "master" + name = "golang.org/x/time" + packages = ["rate"] + revision = "fbb02b2291d28baffd63558aa44b4b56f178d650" + +[[projects]] + branch = "master" name = "google.golang.org/genproto" packages = ["googleapis/rpc/status"] - revision = "09f6ed296fc66555a25fe4ce95173148778dfa85" + revision = "80063a038e333bbe006c878e4c5ce4c74d055498" [[projects]] name = "google.golang.org/grpc" packages = [ ".", + "balancer", + "balancer/base", + "balancer/roundrobin", "codes", + "connectivity", "credentials", - "grpclb/grpc_lb_v1", + "encoding", + "encoding/proto", "grpclog", + "health/grpc_health_v1", "internal", + "internal/backoff", + "internal/channelz", + "internal/grpcrand", "keepalive", "metadata", "naming", "peer", + "resolver", + "resolver/dns", + "resolver/passthrough", "stats", "status", "tap", "transport" ] - revision = "d2e1b51f33ff8c5e4a15560ff049d200e83726c5" - version = "v1.3.0" + revision = "168a6198bcb0ef175f7dacec0b8691fc141dc9b8" + version = "v1.13.0" [[projects]] name = "gopkg.in/inf.v0" packages = ["."] - revision = "3887ee99ecf07df5b447e9b00d9c0b2adaa9f3e4" - version = "v0.9.0" + revision = "d2d2541c53f18d2a059457998ce2876cc8e67cbf" + version = "v0.9.1" [[projects]] name = "gopkg.in/natefinch/lumberjack.v2" packages = ["."] - revision = "20b71e5b60d756d3d2f80def009790325acc2b23" + revision = "a96e63847dc3c67d17befa69c303767e2f84e54f" + version = "v2.1" [[projects]] name = "gopkg.in/yaml.v2" packages = ["."] - revision = "53feefa2559fb8dfa8d81baad31be332c97d6c77" + revision = "5420a8b6744d3b0345ab293f6fcba19c978f1183" + version = "v2.2.1" [[projects]] name = "k8s.io/api" packages = [ + "admission/v1beta1", "admissionregistration/v1alpha1", + "admissionregistration/v1beta1", + "apps/v1", "apps/v1beta1", "apps/v1beta2", "authentication/v1", @@ -451,6 +486,7 @@ "batch/v2alpha1", "certificates/v1beta1", "core/v1", + "events/v1beta1", "extensions/v1beta1", "networking/v1", "policy/v1beta1", @@ -458,11 +494,14 @@ "rbac/v1alpha1", "rbac/v1beta1", "scheduling/v1alpha1", + "scheduling/v1beta1", "settings/v1alpha1", "storage/v1", + "storage/v1alpha1", "storage/v1beta1" ] - revision = "cadaf100c0a3dd6b254f320d6d651df079ec8e0a" + revision = "91b2d7a92a8930454bf5020e0595b8ea0f2a5047" + version = "kubernetes-1.11.0-rc.1" [[projects]] name = "k8s.io/apimachinery" @@ -473,17 +512,13 @@ "pkg/api/resource", "pkg/api/validation", "pkg/api/validation/path", - "pkg/apimachinery", - "pkg/apimachinery/announced", - "pkg/apimachinery/registered", "pkg/apis/meta/internalversion", "pkg/apis/meta/v1", "pkg/apis/meta/v1/unstructured", "pkg/apis/meta/v1/validation", - "pkg/apis/meta/v1alpha1", + "pkg/apis/meta/v1beta1", "pkg/conversion", "pkg/conversion/queryparams", - "pkg/conversion/unstructured", "pkg/fields", "pkg/labels", "pkg/runtime", @@ -501,12 +536,10 @@ "pkg/util/diff", "pkg/util/errors", "pkg/util/framer", - "pkg/util/httpstream", "pkg/util/intstr", "pkg/util/json", "pkg/util/mergepatch", "pkg/util/net", - "pkg/util/proxy", "pkg/util/rand", "pkg/util/runtime", "pkg/util/sets", @@ -515,22 +548,35 @@ "pkg/util/validation", "pkg/util/validation/field", "pkg/util/wait", + "pkg/util/waitgroup", "pkg/util/yaml", "pkg/version", "pkg/watch", "third_party/forked/golang/json", - "third_party/forked/golang/netutil", "third_party/forked/golang/reflect" ] - revision = "3b05bbfa0a45413bfa184edbf9af617e277962fb" - version = "kubernetes-1.9.0-alpha.1" + revision = "fda675fbe85280c4550452dae2a5ebf74e4a59b7" + version = "kubernetes-1.11.0-rc.1" [[projects]] name = "k8s.io/apiserver" packages = [ "pkg/admission", + "pkg/admission/configuration", "pkg/admission/initializer", + "pkg/admission/metrics", + "pkg/admission/plugin/initialization", "pkg/admission/plugin/namespace/lifecycle", + "pkg/admission/plugin/webhook/config", + "pkg/admission/plugin/webhook/config/apis/webhookadmission", + "pkg/admission/plugin/webhook/config/apis/webhookadmission/v1alpha1", + "pkg/admission/plugin/webhook/errors", + "pkg/admission/plugin/webhook/generic", + "pkg/admission/plugin/webhook/mutating", + "pkg/admission/plugin/webhook/namespace", + "pkg/admission/plugin/webhook/request", + "pkg/admission/plugin/webhook/rules", + "pkg/admission/plugin/webhook/validating", "pkg/apis/apiserver", "pkg/apis/apiserver/install", "pkg/apis/apiserver/v1alpha1", @@ -575,6 +621,7 @@ "pkg/server/httplog", "pkg/server/mux", "pkg/server/options", + "pkg/server/resourceconfig", "pkg/server/routes", "pkg/server/routes/data/swagger", "pkg/server/storage", @@ -593,16 +640,19 @@ "pkg/util/flag", "pkg/util/flushwriter", "pkg/util/logs", + "pkg/util/openapi", "pkg/util/trace", "pkg/util/webhook", "pkg/util/wsstream", + "plugin/pkg/audit/buffered", "plugin/pkg/audit/log", + "plugin/pkg/audit/truncate", "plugin/pkg/audit/webhook", "plugin/pkg/authenticator/token/webhook", "plugin/pkg/authorizer/webhook" ] - revision = "c1e53d745d0fe45bf7d5d44697e6eface25fceca" - version = "kubernetes-1.9.0-alpha.1" + revision = "44b612291bb7545430c499a3882c610c727f37b0" + version = "kubernetes-1.11.0-rc.1" [[projects]] name = "k8s.io/client-go" @@ -613,7 +663,9 @@ "informers", "informers/admissionregistration", "informers/admissionregistration/v1alpha1", + "informers/admissionregistration/v1beta1", "informers/apps", + "informers/apps/v1", "informers/apps/v1beta1", "informers/apps/v1beta2", "informers/autoscaling", @@ -627,6 +679,8 @@ "informers/certificates/v1beta1", "informers/core", "informers/core/v1", + "informers/events", + "informers/events/v1beta1", "informers/extensions", "informers/extensions/v1beta1", "informers/internalinterfaces", @@ -640,14 +694,18 @@ "informers/rbac/v1beta1", "informers/scheduling", "informers/scheduling/v1alpha1", + "informers/scheduling/v1beta1", "informers/settings", "informers/settings/v1alpha1", "informers/storage", "informers/storage/v1", + "informers/storage/v1alpha1", "informers/storage/v1beta1", "kubernetes", "kubernetes/scheme", "kubernetes/typed/admissionregistration/v1alpha1", + "kubernetes/typed/admissionregistration/v1beta1", + "kubernetes/typed/apps/v1", "kubernetes/typed/apps/v1beta1", "kubernetes/typed/apps/v1beta2", "kubernetes/typed/authentication/v1", @@ -661,6 +719,7 @@ "kubernetes/typed/batch/v2alpha1", "kubernetes/typed/certificates/v1beta1", "kubernetes/typed/core/v1", + "kubernetes/typed/events/v1beta1", "kubernetes/typed/extensions/v1beta1", "kubernetes/typed/networking/v1", "kubernetes/typed/policy/v1beta1", @@ -668,10 +727,14 @@ "kubernetes/typed/rbac/v1alpha1", "kubernetes/typed/rbac/v1beta1", "kubernetes/typed/scheduling/v1alpha1", + "kubernetes/typed/scheduling/v1beta1", "kubernetes/typed/settings/v1alpha1", "kubernetes/typed/storage/v1", + "kubernetes/typed/storage/v1alpha1", "kubernetes/typed/storage/v1beta1", "listers/admissionregistration/v1alpha1", + "listers/admissionregistration/v1beta1", + "listers/apps/v1", "listers/apps/v1beta1", "listers/apps/v1beta2", "listers/autoscaling/v1", @@ -681,6 +744,7 @@ "listers/batch/v2alpha1", "listers/certificates/v1beta1", "listers/core/v1", + "listers/events/v1beta1", "listers/extensions/v1beta1", "listers/networking/v1", "listers/policy/v1beta1", @@ -688,12 +752,19 @@ "listers/rbac/v1alpha1", "listers/rbac/v1beta1", "listers/scheduling/v1alpha1", + "listers/scheduling/v1beta1", "listers/settings/v1alpha1", "listers/storage/v1", + "listers/storage/v1alpha1", "listers/storage/v1beta1", + "pkg/apis/clientauthentication", + "pkg/apis/clientauthentication/v1alpha1", + "pkg/apis/clientauthentication/v1beta1", "pkg/version", + "plugin/pkg/client/auth/exec", "rest", "rest/watch", + "restmapper", "testing", "tools/auth", "tools/cache", @@ -705,26 +776,30 @@ "tools/pager", "tools/reference", "transport", + "util/buffer", "util/cert", + "util/connrotation", "util/flowcontrol", "util/homedir", - "util/integer" + "util/integer", + "util/retry" ] - revision = "82aa063804cf055e16e8911250f888bc216e8b61" - version = "kubernetes-1.9.0-alpha.1" + revision = "4cacfee698b01630072bc41e3384280562a97d95" + version = "kubernetes-1.11.0-rc.1" [[projects]] + branch = "master" name = "k8s.io/kube-openapi" packages = [ "pkg/builder", "pkg/common", "pkg/handler", - "pkg/util" + "pkg/util", + "pkg/util/proto" ] - revision = "868f2f29720b192240e18284659231b440f9cda5" + revision = "91cfa479c814065e420cee7ed227db0f63a5854e" [[projects]] - branch = "master" name = "k8s.io/metrics" packages = [ "pkg/apis/custom_metrics", @@ -734,11 +809,12 @@ "pkg/apis/external_metrics/install", "pkg/apis/external_metrics/v1beta1" ] - revision = "baa04983db4e01d02a16d9c9fe32dd5b478b3248" + revision = "89f8a18a5efb0c0162a32c75db752bc53ed7f8ee" + version = "kubernetes-1.11.0-rc.1" [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "ffa0ef091b3683f02efac4dc0a1a071ab5a0ff9741346b1532be064fb6c176c4" + inputs-digest = "66b8ba4b829725e88ff6ba39c022bdf9670bdffa53ef9f8094c84e0c1447d2db" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index e93d8b2c..3239762d 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -24,10 +24,59 @@ # go-tests = true # unused-packages = true + +# Utility library deps +[[constraint]] + branch = "master" + name = "github.com/golang/glog" + +[[constraint]] + name = "github.com/prometheus/client_golang" + version = "0.8.0" + +[[constraint]] + branch = "master" + name = "github.com/prometheus/common" + +[[constraint]] + name = "github.com/spf13/cobra" + version = "0.0.3" + +[[constraint]] + name = "gopkg.in/yaml.v2" + version = "2.2.1" + +# Kubernetes incubator deps +[[constraint]] + version = "kubernetes-1.11.0-rc.1" + name = "github.com/kubernetes-incubator/custom-metrics-apiserver" + +# Core Kubernetes deps +[[constraint]] + name = "k8s.io/api" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/apimachinery" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/apiserver" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/client-go" + version = "kubernetes-1.11.0-rc.1" + +[[constraint]] + name = "k8s.io/metrics" + version = "kubernetes-1.11.0-rc.1" + +# Test deps +[[constraint]] + name = "github.com/stretchr/testify" + version = "1.2.2" + [prune] go-tests = true unused-packages = true - -[[constraint]] - name = "github.com/kubernetes-incubator/custom-metrics-apiserver" - revision = "e61f72fec56ab519d74ebd396cd3fcf31b084558" diff --git a/pkg/client/interfaces.go b/pkg/client/interfaces.go index 00ee720d..6c93a0be 100644 --- a/pkg/client/interfaces.go +++ b/pkg/client/interfaces.go @@ -19,6 +19,7 @@ import ( "context" "encoding/json" "fmt" + "strings" "time" "github.com/prometheus/common/model" @@ -121,3 +122,11 @@ func (s *Series) UnmarshalJSON(data []byte) error { return nil } + +func (s *Series) String() string { + lblStrings := make([]string, 0, len(s.Labels)) + for k, v := range s.Labels { + lblStrings = append(lblStrings, fmt.Sprintf("%s=%q", k, v)) + } + return fmt.Sprintf("%s{%s}", s.Name, strings.Join(lblStrings, ",")) +} diff --git a/pkg/config/config.go b/pkg/config/config.go new file mode 100644 index 00000000..6a31e8f1 --- /dev/null +++ b/pkg/config/config.go @@ -0,0 +1,75 @@ +package config + +type MetricsDiscoveryConfig struct { + // Rules specifies how to discover and map Prometheus metrics to + // custom metrics API resources. The rules are applied independently, + // and thus must be mutually exclusive. Rules will the same SeriesQuery + // will make only a single API call. + Rules []DiscoveryRule `yaml:"rules"` +} + +// DiscoveryRule describes on set of rules for transforming Prometheus metrics to/from +// custom metrics API resources. +type DiscoveryRule struct { + // SeriesQuery specifies which metrics this rule should consider via a Prometheus query + // series selector query. + SeriesQuery string `yaml:"seriesQuery"` + // SeriesFilters specifies additional regular expressions to be applied on + // the series names returned from the query. This is useful for constraints + // that can't be represented in the SeriesQuery (e.g. series matching `container_.+` + // not matching `container_.+_total`. A filter will be automatically appended to + // match the form specified in Name. + SeriesFilters []RegexFilter `yaml:"seriesFilters"` + // Resources specifies how associated Kubernetes resources should be discovered for + // the given metrics. + Resources ResourceMapping `yaml:"resources"` + // Name specifies how the metric name should be transformed between custom metric + // API resources, and Prometheus metric names. + Name NameMapping `yaml:"name"` + // MetricsQuery specifies modifications to the metrics query, such as converting + // cumulative metrics to rate metrics. It is a template where `.LabelMatchers` is + // a the comma-separated base label matchers and `.Series` is the series name, and + // `.GroupBy` is the comma-separated expected group-by label names. The delimeters + // are `<<` and `>>`. + MetricsQuery string `yaml:"metricsQuery,omitempty"` +} + +// RegexFilter is a filter that matches positively or negatively against a regex. +// Only one field may be set at a time. +type RegexFilter struct { + Is string `yaml:"is,omitempty"` + IsNot string `yaml:"isNot,omitempty"` +} + +// ResourceMapping specifies how to map Kubernetes resources to Prometheus labels +type ResourceMapping struct { + // Template specifies a golang string template for converting a Kubernetes + // group-resource to a Prometheus label. The template object contains + // the `.Group` and `.Resource` fields. The `.Group` field will have + // dots replaced with underscores, and the `.Resource` field will be + // singularized. The delimiters are `<<` and `>>`. + Template string `yaml:"template,omitempty"` + // Overrides specifies exceptions to the above template, mapping label names + // to group-resources + Overrides map[string]GroupResource `yaml:"overrides,omitempty"` +} + +// GroupResource represents a Kubernetes group-resource. +type GroupResource struct { + Group string `yaml:"group,omitempty"` + Resource string `yaml:"resource"` +} + +// NameMapping specifies how to convert Prometheus metrics +// to/from custom metrics API resources. +type NameMapping struct { + // Matches is a regular expression that is used to match + // Prometheus series names. It may be left blank, in which + // case it is equivalent to `.*`. + Matches string `yaml:"matches"` + // As is the name used in the API. Captures from Matches + // are available for use here. If not specified, it defaults + // to $0 if no capture groups are present in Matches, or $1 + // if only one is present, and will error if multiple are. + As string `yaml:"as"` +} diff --git a/pkg/config/loader.go b/pkg/config/loader.go new file mode 100644 index 00000000..ecc36a23 --- /dev/null +++ b/pkg/config/loader.go @@ -0,0 +1,32 @@ +package config + +import ( + "fmt" + "io/ioutil" + "os" + + yaml "gopkg.in/yaml.v2" +) + +// FromFile loads the configuration from a particular file. +func FromFile(filename string) (*MetricsDiscoveryConfig, error) { + file, err := os.Open(filename) + defer file.Close() + if err != nil { + return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err) + } + contents, err := ioutil.ReadAll(file) + if err != nil { + return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err) + } + return FromYAML(contents) +} + +// FromYAML loads the configuration from a blob of YAML. +func FromYAML(contents []byte) (*MetricsDiscoveryConfig, error) { + var cfg MetricsDiscoveryConfig + if err := yaml.Unmarshal(contents, &cfg); err != nil { + return nil, fmt.Errorf("unable to parse metrics discovery config: %v", err) + } + return &cfg, nil +} diff --git a/pkg/custom-provider/custom_provider.go b/pkg/custom-provider/custom_provider.go index 74d4723c..07f455f8 100644 --- a/pkg/custom-provider/custom_provider.go +++ b/pkg/custom-provider/custom_provider.go @@ -22,59 +22,58 @@ import ( "time" "github.com/golang/glog" - "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" pmodel "github.com/prometheus/common/model" apierr "k8s.io/apimachinery/pkg/api/errors" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/api/resource" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" + utilruntime "k8s.io/apimachinery/pkg/util/runtime" + "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/dynamic" "k8s.io/metrics/pkg/apis/custom_metrics" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" ) +// Runnable represents something that can be run until told to stop. +type Runnable interface { + // Run runs the runnable forever. + Run() + // RunUntil runs the runnable until the given channel is closed. + RunUntil(stopChan <-chan struct{}) +} + type customPrometheusProvider struct { mapper apimeta.RESTMapper - kubeClient dynamic.ClientPool + kubeClient dynamic.Interface promClient prom.Client SeriesRegistry - - rateInterval time.Duration } -func NewCustomPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.ClientPool, promClient prom.Client, labelPrefix string, updateInterval time.Duration, rateInterval time.Duration, stopChan <-chan struct{}) provider.CustomMetricsProvider { +func NewCustomPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []MetricNamer, updateInterval time.Duration) (provider.CustomMetricsProvider, Runnable) { lister := &cachingMetricsLister{ updateInterval: updateInterval, promClient: promClient, + namers: namers, SeriesRegistry: &basicSeriesRegistry{ - namer: metricNamer{ - // TODO: populate the overrides list - overrides: nil, - mapper: mapper, - labelPrefix: labelPrefix, - }, + mapper: mapper, }, } - lister.RunUntil(stopChan) - return &customPrometheusProvider{ mapper: mapper, kubeClient: kubeClient, promClient: promClient, SeriesRegistry: lister, - - rateInterval: rateInterval, - } + }, lister } func (p *customPrometheusProvider) metricFor(value pmodel.SampleValue, groupResource schema.GroupResource, namespace string, name string, metricName string) (*custom_metrics.MetricValue, error) { @@ -91,8 +90,8 @@ func (p *customPrometheusProvider) metricFor(value pmodel.SampleValue, groupReso Namespace: namespace, }, MetricName: metricName, - Timestamp: metaV1.Time{time.Now()}, - Value: *resource.NewMilliQuantity(int64(value*1000.0), resource.DecimalSI).ToDec(), + Timestamp: metav1.Time{time.Now()}, + Value: *resource.NewMilliQuantity(int64(value*1000.0), resource.DecimalSI), }, nil } @@ -131,29 +130,13 @@ func (p *customPrometheusProvider) metricsFor(valueSet pmodel.Vector, info provi } func (p *customPrometheusProvider) buildQuery(info provider.CustomMetricInfo, namespace string, names ...string) (pmodel.Vector, error) { - kind, baseQuery, groupBy, found := p.QueryForMetric(info, namespace, names...) + query, found := p.QueryForMetric(info, namespace, names...) if !found { return nil, provider.NewMetricNotFoundError(info.GroupResource, info.Metric) } - fullQuery := baseQuery - switch kind { - case CounterSeries: - fullQuery = prom.Selector(fmt.Sprintf("rate(%s[%s])", baseQuery, pmodel.Duration(p.rateInterval).String())) - case SecondsCounterSeries: - // TODO: futher modify for seconds? - fullQuery = prom.Selector(prom.Selector(fmt.Sprintf("rate(%s[%s])", baseQuery, pmodel.Duration(p.rateInterval).String()))) - } - - // NB: too small of a rate interval will return no results... - - // sum over all other dimensions of this query (e.g. if we select on route, sum across all pods, - // but if we select on pods, sum across all routes), and split by the dimension of our resource - // TODO: return/populate the by list in SeriesForMetric - fullQuery = prom.Selector(fmt.Sprintf("sum(%s) by (%s)", fullQuery, groupBy)) - // TODO: use an actual context - queryResults, err := p.promClient.Query(context.Background(), pmodel.Now(), fullQuery) + queryResults, err := p.promClient.Query(context.TODO(), pmodel.Now(), query) if err != nil { glog.Errorf("unable to fetch metrics from prometheus: %v", err) // don't leak implementation details to the user @@ -197,23 +180,24 @@ func (p *customPrometheusProvider) getSingle(info provider.CustomMetricInfo, nam } func (p *customPrometheusProvider) getMultiple(info provider.CustomMetricInfo, namespace string, selector labels.Selector) (*custom_metrics.MetricValueList, error) { - // construct a client to list the names of objects matching the label selector - client, err := p.kubeClient.ClientForGroupVersionResource(info.GroupResource.WithVersion("")) + fullResources, err := p.mapper.ResourcesFor(info.GroupResource.WithVersion("")) + if err == nil && len(fullResources) == 0 { + err = fmt.Errorf("no fully versioned resources known for group-resource %v", info.GroupResource) + } if err != nil { - glog.Errorf("unable to construct dynamic client to list matching resource names: %v", err) + glog.Errorf("unable to find preferred version to list matching resource names: %v", err) // don't leak implementation details to the user return nil, apierr.NewInternalError(fmt.Errorf("unable to list matching resources")) } - - // we can construct a this APIResource ourself, since the dynamic client only uses Name and Namespaced - apiRes := &metaV1.APIResource{ - Name: info.GroupResource.Resource, - Namespaced: info.Namespaced, + var client dynamic.ResourceInterface + if namespace != "" { + client = p.kubeClient.Resource(fullResources[0]).Namespace(namespace) + } else { + client = p.kubeClient.Resource(fullResources[0]) } // actually list the objects matching the label selector - matchingObjectsRaw, err := client.Resource(apiRes, namespace). - List(metaV1.ListOptions{LabelSelector: selector.String()}) + matchingObjectsRaw, err := client.List(metav1.ListOptions{LabelSelector: selector.String()}) if err != nil { glog.Errorf("unable to list matching resource names: %v", err) // don't leak implementation details to the user @@ -278,3 +262,86 @@ func (p *customPrometheusProvider) GetNamespacedMetricBySelector(groupResource s } return p.getMultiple(info, namespace, selector) } + +type cachingMetricsLister struct { + SeriesRegistry + + promClient prom.Client + updateInterval time.Duration + namers []MetricNamer +} + +func (l *cachingMetricsLister) Run() { + l.RunUntil(wait.NeverStop) +} + +func (l *cachingMetricsLister) RunUntil(stopChan <-chan struct{}) { + go wait.Until(func() { + if err := l.updateMetrics(); err != nil { + utilruntime.HandleError(err) + } + }, l.updateInterval, stopChan) +} + +type selectorSeries struct { + selector prom.Selector + series []prom.Series +} + +func (l *cachingMetricsLister) updateMetrics() error { + startTime := pmodel.Now().Add(-1 * l.updateInterval) + + // don't do duplicate queries when it's just the matchers that change + seriesCacheByQuery := make(map[prom.Selector][]prom.Series) + + // these can take a while on large clusters, so launch in parallel + // and don't duplicate + selectors := make(map[prom.Selector]struct{}) + selectorSeriesChan := make(chan selectorSeries, len(l.namers)) + errs := make(chan error, len(l.namers)) + for _, namer := range l.namers { + sel := namer.Selector() + if _, ok := selectors[sel]; ok { + errs <- nil + selectorSeriesChan <- selectorSeries{} + continue + } + selectors[sel] = struct{}{} + go func() { + series, err := l.promClient.Series(context.TODO(), pmodel.Interval{startTime, 0}, sel) + if err != nil { + errs <- fmt.Errorf("unable to fetch metrics for query %q: %v", sel, err) + return + } + errs <- nil + selectorSeriesChan <- selectorSeries{ + selector: sel, + series: series, + } + }() + } + + // iterate through, blocking until we've got all results + for range l.namers { + if err := <-errs; err != nil { + return fmt.Errorf("unable to update list of all metrics: %v", err) + } + if ss := <-selectorSeriesChan; ss.series != nil { + seriesCacheByQuery[ss.selector] = ss.series + } + } + close(errs) + + newSeries := make([][]prom.Series, len(l.namers)) + for i, namer := range l.namers { + series, cached := seriesCacheByQuery[namer.Selector()] + if !cached { + return fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", namer.Selector()) + } + newSeries[i] = namer.FilterSeries(series) + } + + glog.V(10).Infof("Set available metric list from Prometheus to: %v", newSeries) + + return l.SetSeries(newSeries, l.namers) +} diff --git a/pkg/custom-provider/custom_provider_test.go b/pkg/custom-provider/custom_provider_test.go index b5a756e0..853f73c5 100644 --- a/pkg/custom-provider/custom_provider_test.go +++ b/pkg/custom-provider/custom_provider_test.go @@ -29,6 +29,7 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" fakedyn "k8s.io/client-go/dynamic/fake" + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" pmodel "github.com/prometheus/common/model" ) @@ -86,20 +87,20 @@ func (c *fakePromClient) QueryRange(_ context.Context, r prom.Range, query prom. return prom.QueryResult{}, nil } -func setupPrometheusProvider(t *testing.T, stopCh <-chan struct{}) (provider.CustomMetricsProvider, *fakePromClient) { +func setupPrometheusProvider(t *testing.T) (provider.CustomMetricsProvider, *fakePromClient) { fakeProm := &fakePromClient{} - fakeKubeClient := &fakedyn.FakeClientPool{} + fakeKubeClient := &fakedyn.FakeDynamicClient{} - prov := NewPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, "", fakeProviderUpdateInterval, 1*time.Minute, stopCh) + cfg := config.DefaultConfig(1*time.Minute, "") + namers, err := NamersFromConfig(cfg, restMapper()) + require.NoError(t, err) + + prov, _ := NewCustomPrometheusProvider(restMapper(), fakeKubeClient, fakeProm, namers, fakeProviderUpdateInterval) containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", "")) namespacedSel := prom.MatchSeries("", prom.LabelNeq("namespace", ""), prom.NameNotMatches("^container_.*")) fakeProm.series = map[prom.Selector][]prom.Series{ containerSel: { - { - Name: "container_actually_gauge_seconds_total", - Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, - }, { Name: "container_some_usage", Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, @@ -130,28 +131,24 @@ func setupPrometheusProvider(t *testing.T, stopCh <-chan struct{}) (provider.Cus func TestListAllMetrics(t *testing.T) { // setup - stopCh := make(chan struct{}) - defer close(stopCh) - prov, fakeProm := setupPrometheusProvider(t, stopCh) + prov, fakeProm := setupPrometheusProvider(t) // assume we have no updates require.Len(t, prov.ListAllMetrics(), 0, "assume: should have no metrics updates at the start") // set the acceptible interval (now until the next update, with a bit of wiggle room) - startTime := pmodel.Now() - endTime := startTime.Add(fakeProviderUpdateInterval + fakeProviderUpdateInterval/10) - fakeProm.acceptibleInterval = pmodel.Interval{Start: startTime, End: endTime} + startTime := pmodel.Now().Add(-1*fakeProviderUpdateInterval - fakeProviderUpdateInterval/10) + fakeProm.acceptibleInterval = pmodel.Interval{Start: startTime, End: 0} - // wait one update interval (with a bit of wiggle room) - time.Sleep(fakeProviderUpdateInterval + fakeProviderUpdateInterval/10) + // update the metrics (without actually calling RunUntil, so we can avoid timing issues) + lister := prov.(*customPrometheusProvider).SeriesRegistry.(*cachingMetricsLister) + require.NoError(t, lister.updateMetrics()) // list/sort the metrics actualMetrics := prov.ListAllMetrics() sort.Sort(metricInfoSorter(actualMetrics)) - expectedMetrics := []provider.MetricInfo{ - {schema.GroupResource{Resource: "pods"}, true, "actually_gauge"}, - {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + expectedMetrics := []provider.CustomMetricInfo{ {schema.GroupResource{Resource: "services"}, true, "ingress_hits"}, {schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"}, {schema.GroupResource{Resource: "pods"}, true, "ingress_hits"}, @@ -160,6 +157,8 @@ func TestListAllMetrics(t *testing.T) { {schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"}, {schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"}, {schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_usage"}, + {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, } sort.Sort(metricInfoSorter(expectedMetrics)) diff --git a/pkg/custom-provider/external_provider.go b/pkg/custom-provider/external_provider.go index 743829a8..544bcd1b 100644 --- a/pkg/custom-provider/external_provider.go +++ b/pkg/custom-provider/external_provider.go @@ -33,40 +33,30 @@ import ( type externalPrometheusProvider struct { mapper apimeta.RESTMapper - kubeClient dynamic.ClientPool + kubeClient dynamic.Interface promClient prom.Client SeriesRegistry - - rateInterval time.Duration } -func NewExternalPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.ClientPool, promClient prom.Client, labelPrefix string, updateInterval time.Duration, rateInterval time.Duration, stopChan <-chan struct{}) provider.ExternalMetricsProvider { +func NewExternalPrometheusProvider(mapper apimeta.RESTMapper, kubeClient dynamic.Interface, promClient prom.Client, namers []MetricNamer, updateInterval time.Duration) (provider.ExternalMetricsProvider, Runnable) { lister := &cachingMetricsLister{ updateInterval: updateInterval, promClient: promClient, + namers: namers, SeriesRegistry: &basicSeriesRegistry{ - namer: metricNamer{ - // TODO: populate the overrides list - overrides: nil, - mapper: mapper, - labelPrefix: labelPrefix, - }, + mapper: mapper, }, } - lister.RunUntil(stopChan) - return &externalPrometheusProvider{ mapper: mapper, kubeClient: kubeClient, promClient: promClient, SeriesRegistry: lister, - - rateInterval: rateInterval, - } + }, lister } func (p *externalPrometheusProvider) GetExternalMetric(namespace string, metricName string, metricSelector labels.Selector) (*external_metrics.ExternalMetricValueList, error) { diff --git a/pkg/custom-provider/metric_namer.go b/pkg/custom-provider/metric_namer.go index a46f3218..7de4f69d 100644 --- a/pkg/custom-provider/metric_namer.go +++ b/pkg/custom-provider/metric_namer.go @@ -1,367 +1,474 @@ -/* -Copyright 2017 The Kubernetes Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - package provider import ( + "bytes" "fmt" + "regexp" "strings" "sync" + "text/template" + "github.com/golang/glog" "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/golang/glog" + "github.com/directxman12/k8s-prometheus-adapter/pkg/config" pmodel "github.com/prometheus/common/model" ) -// NB: container metrics sourced from cAdvisor don't consistently follow naming conventions, -// so we need to whitelist them and handle them on a case-by-case basis. Metrics ending in `_total` -// *should* be counters, but may actually be guages in this case. +var nsGroupResource = schema.GroupResource{Resource: "namespaces"} +var groupNameSanitizer = strings.NewReplacer(".", "_", "-", "_") -// SeriesType represents the kind of series backing a metric. -type SeriesType int - -const ( - CounterSeries SeriesType = iota - SecondsCounterSeries - GaugeSeries -) - -// SeriesRegistry provides conversions between Prometheus series and MetricInfo -type SeriesRegistry interface { - // Selectors produces the appropriate Prometheus selectors to match all series handlable - // by this registry, as an optimization for SetSeries. - Selectors() []prom.Selector - // SetSeries replaces the known series in this registry - SetSeries(series []prom.Series) error - // ListAllMetrics lists all metrics known to this registry - ListAllMetrics() []provider.CustomMetricInfo - // SeriesForMetric looks up the minimum required series information to make a query for the given metric - // against the given resource (namespace may be empty for non-namespaced resources) - QueryForMetric(info provider.CustomMetricInfo, namespace string, resourceNames ...string) (kind SeriesType, query prom.Selector, groupBy string, found bool) - // MatchValuesToNames matches result values to resource names for the given metric and value set - MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) +// MetricNamer knows how to convert Prometheus series names and label names to +// metrics API resources, and vice-versa. MetricNamers should be safe to access +// concurrently. Returned group-resources are "normalized" as per the +// MetricInfo#Normalized method. Group-resources passed as arguments must +// themselves be normalized. +type MetricNamer interface { + // Selector produces the appropriate Prometheus series selector to match all + // series handlable by this namer. + Selector() prom.Selector + // FilterSeries checks to see which of the given series match any additional + // constrains beyond the series query. It's assumed that the series given + // already matche the series query. + FilterSeries(series []prom.Series) []prom.Series + // ResourcesForSeries returns the group-resources associated with the given series, + // as well as whether or not the given series has the "namespace" resource). + ResourcesForSeries(series prom.Series) (res []schema.GroupResource, namespaced bool) + // LabelForResource returns the appropriate label for the given resource. + LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) + // MetricNameForSeries returns the name (as presented in the API) for a given series. + MetricNameForSeries(series prom.Series) (string, error) + // QueryForSeries returns the query for a given series (not API metric name), with + // the given namespace name (if relevant), resource, and resource names. + QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) } -type seriesInfo struct { - // baseSeries represents the minimum information to access a particular series - baseSeries prom.Series - // kind is the type of this series - kind SeriesType - // isContainer indicates if the series is a cAdvisor container_ metric, and thus needs special handling - isContainer bool +// labelGroupResExtractor extracts schema.GroupResources from series labels. +type labelGroupResExtractor struct { + regex *regexp.Regexp + + resourceInd int + groupInd *int + mapper apimeta.RESTMapper } -// overridableSeriesRegistry is a basic SeriesRegistry -type basicSeriesRegistry struct { - mu sync.RWMutex - - // info maps metric info to information about the corresponding series - info map[provider.CustomMetricInfo]seriesInfo - // metrics is the list of all known metrics - metrics []provider.CustomMetricInfo - - // namer is the metricNamer responsible for converting series to metric names and information - namer metricNamer -} - -func (r *basicSeriesRegistry) Selectors() []prom.Selector { - // container-specific metrics from cAdvsior have their own form, and need special handling - // TODO: figure out how to determine which metrics on non-namespaced objects are kubernetes-related - containerSel := prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", "")) - namespacedSel := prom.MatchSeries("", prom.LabelNeq(r.namer.labelPrefix+"namespace", ""), prom.NameNotMatches("^container_.*")) - - return []prom.Selector{containerSel, namespacedSel} -} - -func (r *basicSeriesRegistry) SetSeries(newSeries []prom.Series) error { - newInfo := make(map[provider.CustomMetricInfo]seriesInfo) - for _, series := range newSeries { - if strings.HasPrefix(series.Name, "container_") { - r.namer.processContainerSeries(series, newInfo) - } else if namespaceLabel, hasNamespaceLabel := series.Labels[pmodel.LabelName(r.namer.labelPrefix+"namespace")]; hasNamespaceLabel && namespaceLabel != "" { - // we also handle namespaced metrics here as part of the resource-association logic - if err := r.namer.processNamespacedSeries(series, newInfo); err != nil { - glog.Errorf("Unable to process namespaced series %q: %v", series.Name, err) - continue - } - } else { - if err := r.namer.processRootScopedSeries(series, newInfo); err != nil { - glog.Errorf("Unable to process root-scoped series %q: %v", series.Name, err) - continue - } - } +// newLabelGroupResExtractor creates a new labelGroupResExtractor for labels whose form +// matches the given template. It does so by creating a regular expression from the template, +// so anything in the template which limits resource or group name length will cause issues. +func newLabelGroupResExtractor(labelTemplate *template.Template) (*labelGroupResExtractor, error) { + labelRegexBuff := new(bytes.Buffer) + if err := labelTemplate.Execute(labelRegexBuff, schema.GroupResource{"(?P.+?)", "(?P.+?)"}); err != nil { + return nil, fmt.Errorf("unable to convert label template to matcher: %v", err) } - - newMetrics := make([]provider.CustomMetricInfo, 0, len(newInfo)) - for info := range newInfo { - newMetrics = append(newMetrics, info) + if labelRegexBuff.Len() == 0 { + return nil, fmt.Errorf("unable to convert label template to matcher: empty template") } - - r.mu.Lock() - defer r.mu.Unlock() - - r.info = newInfo - r.metrics = newMetrics - - return nil -} - -func (r *basicSeriesRegistry) ListAllMetrics() []provider.CustomMetricInfo { - r.mu.RLock() - defer r.mu.RUnlock() - - return r.metrics -} - -func (r *basicSeriesRegistry) QueryForMetric(metricInfo provider.CustomMetricInfo, namespace string, resourceNames ...string) (kind SeriesType, query prom.Selector, groupBy string, found bool) { - r.mu.RLock() - defer r.mu.RUnlock() - - if len(resourceNames) == 0 { - glog.Errorf("no resource names requested while producing a query for metric %s", metricInfo.String()) - return 0, "", "", false - } - - metricInfo, singularResource, err := metricInfo.Normalized(r.namer.mapper) + labelRegexRaw := "^" + labelRegexBuff.String() + "$" + labelRegex, err := regexp.Compile(labelRegexRaw) if err != nil { - glog.Errorf("unable to normalize group resource while producing a query: %v", err) - return 0, "", "", false - } - resourceLbl := r.namer.labelPrefix + singularResource - - // TODO: support container metrics - if info, found := r.info[metricInfo]; found { - targetValue := resourceNames[0] - matcher := prom.LabelEq - if len(resourceNames) > 1 { - targetValue = strings.Join(resourceNames, "|") - matcher = prom.LabelMatches - } - - var expressions []string - if info.isContainer { - expressions = []string{matcher("pod_name", targetValue), prom.LabelNeq("container_name", "POD")} - groupBy = "pod_name" - } else { - // TODO: copy base series labels? - expressions = []string{matcher(resourceLbl, targetValue)} - groupBy = resourceLbl - } - - if metricInfo.Namespaced { - prefix := r.namer.labelPrefix - if info.isContainer { - prefix = "" - } - expressions = append(expressions, prom.LabelEq(prefix+"namespace", namespace)) - } - - return info.kind, prom.MatchSeries(info.baseSeries.Name, expressions...), groupBy, true + return nil, fmt.Errorf("unable to convert label template to matcher: %v", err) } - glog.V(10).Infof("metric %v not registered", metricInfo) - return 0, "", "", false + var groupInd *int + var resInd *int + + for i, name := range labelRegex.SubexpNames() { + switch name { + case "group": + ind := i // copy to avoid iteration variable reference + groupInd = &ind + case "resource": + ind := i // copy to avoid iteration variable reference + resInd = &ind + } + } + + if resInd == nil { + return nil, fmt.Errorf("must include at least `{{.Resource}}` in the label template") + } + + return &labelGroupResExtractor{ + regex: labelRegex, + resourceInd: *resInd, + groupInd: groupInd, + }, nil } -func (r *basicSeriesRegistry) MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) { - r.mu.RLock() - defer r.mu.RUnlock() - - metricInfo, singularResource, err := metricInfo.Normalized(r.namer.mapper) - if err != nil { - glog.Errorf("unable to normalize group resource while matching values to names: %v", err) - return nil, false - } - resourceLbl := r.namer.labelPrefix + singularResource - - if info, found := r.info[metricInfo]; found { - res := make(map[string]pmodel.SampleValue, len(values)) - for _, val := range values { - if val == nil { - // skip empty values - continue - } - - labelName := pmodel.LabelName(resourceLbl) - if info.isContainer { - labelName = pmodel.LabelName("pod_name") - } - res[string(val.Metric[labelName])] = val.Value +// GroupResourceForLabel extracts a schema.GroupResource from the given label, if possible. +// The second argument indicates whether or not a potential group-resource was found in this label. +func (e *labelGroupResExtractor) GroupResourceForLabel(lbl pmodel.LabelName) (schema.GroupResource, bool) { + matchGroups := e.regex.FindStringSubmatch(string(lbl)) + if matchGroups != nil { + group := "" + if e.groupInd != nil { + group = matchGroups[*e.groupInd] } - return res, true + return schema.GroupResource{ + Group: group, + Resource: matchGroups[e.resourceInd], + }, true } - return nil, false + return schema.GroupResource{}, false } -// metricNamer knows how to construct MetricInfo out of raw prometheus series descriptions. -type metricNamer struct { - // overrides contains the list of container metrics whose naming we want to override. - // This is used to properly convert certain cAdvisor container metrics. - overrides map[string]seriesSpec - - mapper apimeta.RESTMapper - - labelPrefix string +func (r *metricNamer) Selector() prom.Selector { + return r.seriesQuery } -// seriesSpec specifies how to produce metric info for a particular prometheus series source -type seriesSpec struct { - // metricName is the desired output API metric name - metricName string - // kind indicates whether or not this metric is cumulative, - // and thus has to be calculated as a rate when returning it - kind SeriesType +// reMatcher either positively or negatively matches a regex +type reMatcher struct { + regex *regexp.Regexp + positive bool } -// processContainerSeries performs special work to extract metric definitions -// from cAdvisor-sourced container metrics, which don't particularly follow any useful conventions consistently. -func (n *metricNamer) processContainerSeries(series prom.Series, infos map[provider.CustomMetricInfo]seriesInfo) { +func newReMatcher(cfg config.RegexFilter) (*reMatcher, error) { + if cfg.Is != "" && cfg.IsNot != "" { + return nil, fmt.Errorf("cannot have both an `is` (%q) and `isNot` (%q) expression in a single filter", cfg.Is, cfg.IsNot) + } + if cfg.Is == "" && cfg.IsNot == "" { + return nil, fmt.Errorf("must have either an `is` or `isNot` expression in a filter") + } - originalName := series.Name - - var name string - metricKind := GaugeSeries - if override, hasOverride := n.overrides[series.Name]; hasOverride { - name = override.metricName - metricKind = override.kind + var positive bool + var regexRaw string + if cfg.Is != "" { + positive = true + regexRaw = cfg.Is } else { - // chop of the "container_" prefix - series.Name = series.Name[10:] - name, metricKind = n.metricNameFromSeries(series) + positive = false + regexRaw = cfg.IsNot } - info := provider.CustomMetricInfo{ - GroupResource: schema.GroupResource{Resource: "pods"}, - Namespaced: true, - Metric: name, - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: originalName}, - isContainer: true, - } -} - -// processNamespacedSeries adds the metric info for the given generic namespaced series to -// the map of metric info. -func (n *metricNamer) processNamespacedSeries(series prom.Series, infos map[provider.CustomMetricInfo]seriesInfo) error { - // NB: all errors must occur *before* we save the series info - name, metricKind := n.metricNameFromSeries(series) - resources, err := n.groupResourcesFromSeries(series) + regex, err := regexp.Compile(regexRaw) if err != nil { - return fmt.Errorf("unable to process prometheus series %s: %v", series.Name, err) + return nil, fmt.Errorf("unable to compile series filter %q: %v", regexRaw, err) } - // we add one metric for each resource that this could describe - for _, resource := range resources { - info := provider.CustomMetricInfo{ - GroupResource: resource, - Namespaced: true, - Metric: name, - } - - // metrics describing namespaces aren't considered to be namespaced - if resource == (schema.GroupResource{Resource: "namespaces"}) { - info.Namespaced = false - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: series.Name}, - } - } - - return nil + return &reMatcher{ + regex: regex, + positive: positive, + }, nil } -// processesRootScopedSeries adds the metric info for the given generic namespaced series to -// the map of metric info. -func (n *metricNamer) processRootScopedSeries(series prom.Series, infos map[provider.CustomMetricInfo]seriesInfo) error { - // NB: all errors must occur *before* we save the series info - name, metricKind := n.metricNameFromSeries(series) - resources, err := n.groupResourcesFromSeries(series) - if err != nil { - return fmt.Errorf("unable to process prometheus series %s: %v", series.Name, err) - } - - // we add one metric for each resource that this could describe - for _, resource := range resources { - info := provider.CustomMetricInfo{ - GroupResource: resource, - Namespaced: false, - Metric: name, - } - - infos[info] = seriesInfo{ - kind: metricKind, - baseSeries: prom.Series{Name: series.Name}, - } - } - - return nil +func (m *reMatcher) Matches(val string) bool { + return m.regex.MatchString(val) == m.positive } -// groupResourceFromSeries collects the possible group-resources that this series could describe by -// going through each label, checking to see if it corresponds to a known resource. For instance, -// a series `ingress_http_hits_total{pod="foo",service="bar",ingress="baz",namespace="ns"}` -// would return three GroupResources: "pods", "services", and "ingresses". -// Returned MetricInfo is equilavent to the "normalized" info produced by metricInfo.Normalized. -func (n *metricNamer) groupResourcesFromSeries(series prom.Series) ([]schema.GroupResource, error) { - var res []schema.GroupResource - for label := range series.Labels { - if !strings.HasPrefix(string(label), n.labelPrefix) { - continue - } - label = label[len(n.labelPrefix):] - // TODO: figure out a way to let people specify a fully-qualified name in label-form - gvr, err := n.mapper.ResourceFor(schema.GroupVersionResource{Resource: string(label)}) - if err != nil { - if apimeta.IsNoMatchError(err) { - continue +type metricNamer struct { + seriesQuery prom.Selector + labelTemplate *template.Template + labelResExtractor *labelGroupResExtractor + metricsQueryTemplate *template.Template + nameMatches *regexp.Regexp + nameAs string + seriesMatchers []*reMatcher + + labelResourceMu sync.RWMutex + labelToResource map[pmodel.LabelName]schema.GroupResource + resourceToLabel map[schema.GroupResource]pmodel.LabelName + mapper apimeta.RESTMapper +} + +// queryTemplateArgs are the arguments for the metrics query template. +type queryTemplateArgs struct { + Series string + LabelMatchers string + LabelValuesByName map[string][]string + GroupBy string + GroupBySlice []string +} + +func (n *metricNamer) FilterSeries(initialSeries []prom.Series) []prom.Series { + if len(n.seriesMatchers) == 0 { + return initialSeries + } + + finalSeries := make([]prom.Series, 0, len(initialSeries)) +SeriesLoop: + for _, series := range initialSeries { + for _, matcher := range n.seriesMatchers { + if !matcher.Matches(series.Name) { + continue SeriesLoop } - return nil, err } - res = append(res, gvr.GroupResource()) + finalSeries = append(finalSeries, series) } - return res, nil + return finalSeries } -// metricNameFromSeries extracts a metric name from a series name, and indicates -// whether or not that series was a counter. It also has special logic to deal with time-based -// counters, which general get converted to milli-unit rate metrics. -func (n *metricNamer) metricNameFromSeries(series prom.Series) (name string, kind SeriesType) { - kind = GaugeSeries - name = series.Name - if strings.HasSuffix(name, "_total") { - kind = CounterSeries - name = name[:len(name)-6] +func (n *metricNamer) QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) { + var exprs []string + valuesByName := map[string][]string{} - if strings.HasSuffix(name, "_seconds") { - kind = SecondsCounterSeries - name = name[:len(name)-8] + if namespace != "" { + namespaceLbl, err := n.LabelForResource(nsGroupResource) + if err != nil { + return "", err + } + exprs = append(exprs, prom.LabelEq(string(namespaceLbl), namespace)) + valuesByName[string(namespaceLbl)] = []string{namespace} + } + + resourceLbl, err := n.LabelForResource(resource) + if err != nil { + return "", err + } + matcher := prom.LabelEq + targetValue := names[0] + if len(names) > 1 { + matcher = prom.LabelMatches + targetValue = strings.Join(names, "|") + } + exprs = append(exprs, matcher(string(resourceLbl), targetValue)) + valuesByName[string(resourceLbl)] = names + + args := queryTemplateArgs{ + Series: series, + LabelMatchers: strings.Join(exprs, ","), + LabelValuesByName: valuesByName, + GroupBy: string(resourceLbl), + GroupBySlice: []string{string(resourceLbl)}, + } + queryBuff := new(bytes.Buffer) + if err := n.metricsQueryTemplate.Execute(queryBuff, args); err != nil { + return "", err + } + + if queryBuff.Len() == 0 { + return "", fmt.Errorf("empty query produced by metrics query template") + } + + return prom.Selector(queryBuff.String()), nil +} + +func (n *metricNamer) ResourcesForSeries(series prom.Series) ([]schema.GroupResource, bool) { + // use an updates map to avoid having to drop the read lock to update the cache + // until the end. Since we'll probably have few updates after the first run, + // this should mean that we rarely have to hold the write lock. + var resources []schema.GroupResource + updates := make(map[pmodel.LabelName]schema.GroupResource) + namespaced := false + + // use an anon func to get the right defer behavior + func() { + n.labelResourceMu.RLock() + defer n.labelResourceMu.RUnlock() + + for lbl := range series.Labels { + var groupRes schema.GroupResource + var ok bool + + // check if we have an override + if groupRes, ok = n.labelToResource[lbl]; ok { + resources = append(resources, groupRes) + } else if groupRes, ok = updates[lbl]; ok { + resources = append(resources, groupRes) + } else if n.labelResExtractor != nil { + // if not, check if it matches the form we expect, and if so, + // convert to a group-resource. + if groupRes, ok = n.labelResExtractor.GroupResourceForLabel(lbl); ok { + info, _, err := provider.CustomMetricInfo{GroupResource: groupRes}.Normalized(n.mapper) + if err != nil { + glog.Errorf("unable to normalize group-resource %s from label %q, skipping: %v", groupRes.String(), lbl, err) + continue + } + + groupRes = info.GroupResource + resources = append(resources, groupRes) + updates[lbl] = groupRes + } + } + + if groupRes == nsGroupResource { + namespaced = true + } + } + }() + + // update the cache for next time. This should only be called by discovery, + // so we don't really have to worry about the grap between read and write locks + // (plus, we don't care if someone else updates the cache first, since the results + // are necessarily the same, so at most we've done extra work). + if len(updates) > 0 { + n.labelResourceMu.Lock() + defer n.labelResourceMu.Unlock() + + for lbl, groupRes := range updates { + n.labelToResource[lbl] = groupRes } } - return + return resources, namespaced +} + +func (n *metricNamer) LabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) { + n.labelResourceMu.RLock() + // check if we have a cached copy or override + lbl, ok := n.resourceToLabel[resource] + n.labelResourceMu.RUnlock() // release before we call makeLabelForResource + if ok { + return lbl, nil + } + + // NB: we don't actually care about the gap between releasing read lock + // and acquiring the write lock -- if we do duplicate work sometimes, so be + // it, as long as we're correct. + + // otherwise, use the template and save the result + lbl, err := n.makeLabelForResource(resource) + if err != nil { + return "", fmt.Errorf("unable to convert resource %s into label: %v", resource.String(), err) + } + return lbl, nil +} + +// makeLabelForResource constructs a label name for the given resource, and saves the result. +// It must *not* be called under an existing lock. +func (n *metricNamer) makeLabelForResource(resource schema.GroupResource) (pmodel.LabelName, error) { + if n.labelTemplate == nil { + return "", fmt.Errorf("no generic resource label form specified for this metric") + } + buff := new(bytes.Buffer) + + singularRes, err := n.mapper.ResourceSingularizer(resource.Resource) + if err != nil { + return "", fmt.Errorf("unable to singularize resource %s: %v", resource.String(), err) + } + convResource := schema.GroupResource{ + Group: groupNameSanitizer.Replace(resource.Group), + Resource: singularRes, + } + + if err := n.labelTemplate.Execute(buff, convResource); err != nil { + return "", err + } + if buff.Len() == 0 { + return "", fmt.Errorf("empty label produced by label template") + } + lbl := pmodel.LabelName(buff.String()) + + n.labelResourceMu.Lock() + defer n.labelResourceMu.Unlock() + + n.resourceToLabel[resource] = lbl + n.labelToResource[lbl] = resource + return lbl, nil +} + +func (n *metricNamer) MetricNameForSeries(series prom.Series) (string, error) { + matches := n.nameMatches.FindStringSubmatchIndex(series.Name) + if matches == nil { + return "", fmt.Errorf("series name %q did not match expected pattern %q", series.Name, n.nameMatches.String()) + } + outNameBytes := n.nameMatches.ExpandString(nil, n.nameAs, series.Name, matches) + return string(outNameBytes), nil +} + +// NamersFromConfig produces a MetricNamer for each rule in the given config. +func NamersFromConfig(cfg *config.MetricsDiscoveryConfig, mapper apimeta.RESTMapper) ([]MetricNamer, error) { + namers := make([]MetricNamer, len(cfg.Rules)) + + for i, rule := range cfg.Rules { + var labelTemplate *template.Template + var labelResExtractor *labelGroupResExtractor + var err error + if rule.Resources.Template != "" { + labelTemplate, err = template.New("resource-label").Delims("<<", ">>").Parse(rule.Resources.Template) + if err != nil { + return nil, fmt.Errorf("unable to parse label template %q associated with series query %q: %v", rule.Resources.Template, rule.SeriesQuery, err) + } + + labelResExtractor, err = newLabelGroupResExtractor(labelTemplate) + if err != nil { + return nil, fmt.Errorf("unable to generate label format from template %q associated with series query %q: %v", rule.Resources.Template, rule.SeriesQuery, err) + } + } + + metricsQueryTemplate, err := template.New("metrics-query").Delims("<<", ">>").Parse(rule.MetricsQuery) + if err != nil { + return nil, fmt.Errorf("unable to parse metrics query template %q associated with series query %q: %v", rule.MetricsQuery, rule.SeriesQuery, err) + } + + seriesMatchers := make([]*reMatcher, len(rule.SeriesFilters)) + for i, filterRaw := range rule.SeriesFilters { + matcher, err := newReMatcher(filterRaw) + if err != nil { + return nil, fmt.Errorf("unable to generate series name filter associated with series query %q: %v", rule.SeriesQuery, err) + } + seriesMatchers[i] = matcher + } + if rule.Name.Matches != "" { + matcher, err := newReMatcher(config.RegexFilter{Is: rule.Name.Matches}) + if err != nil { + return nil, fmt.Errorf("unable to generate series name filter from name rules associated with series query %q: %v", rule.SeriesQuery, err) + } + seriesMatchers = append(seriesMatchers, matcher) + } + + var nameMatches *regexp.Regexp + if rule.Name.Matches != "" { + nameMatches, err = regexp.Compile(rule.Name.Matches) + if err != nil { + return nil, fmt.Errorf("unable to compile series name match expression %q associated with series query %q: %v", rule.Name.Matches, rule.SeriesQuery, err) + } + } else { + // this will always succeed + nameMatches = regexp.MustCompile(".*") + } + nameAs := rule.Name.As + if nameAs == "" { + // check if we have an obvious default + subexpNames := nameMatches.SubexpNames() + if len(subexpNames) == 1 { + // no capture groups, use the whole thing + nameAs = "$0" + } else if len(subexpNames) == 2 { + // one capture group, use that + nameAs = "$1" + } else { + return nil, fmt.Errorf("must specify an 'as' value for name matcher %q associated with series query %q", rule.Name.Matches, rule.SeriesQuery) + } + } + + namer := &metricNamer{ + seriesQuery: prom.Selector(rule.SeriesQuery), + labelTemplate: labelTemplate, + labelResExtractor: labelResExtractor, + metricsQueryTemplate: metricsQueryTemplate, + mapper: mapper, + nameMatches: nameMatches, + nameAs: nameAs, + seriesMatchers: seriesMatchers, + + labelToResource: make(map[pmodel.LabelName]schema.GroupResource), + resourceToLabel: make(map[schema.GroupResource]pmodel.LabelName), + } + + // invert the structure for consistency with the template + for lbl, groupRes := range rule.Resources.Overrides { + infoRaw := provider.CustomMetricInfo{ + GroupResource: schema.GroupResource{ + Group: groupRes.Group, + Resource: groupRes.Resource, + }, + } + info, _, err := infoRaw.Normalized(mapper) + if err != nil { + return nil, fmt.Errorf("unable to normalize group-resource %v: %v", groupRes, err) + } + + namer.labelToResource[pmodel.LabelName(lbl)] = info.GroupResource + namer.resourceToLabel[info.GroupResource] = pmodel.LabelName(lbl) + } + + namers[i] = namer + } + + return namers, nil } diff --git a/pkg/custom-provider/series_registry.go b/pkg/custom-provider/series_registry.go new file mode 100644 index 00000000..a4701cf4 --- /dev/null +++ b/pkg/custom-provider/series_registry.go @@ -0,0 +1,198 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "fmt" + "sync" + + "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" + apimeta "k8s.io/apimachinery/pkg/api/meta" + + prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + "github.com/golang/glog" + pmodel "github.com/prometheus/common/model" +) + +// NB: container metrics sourced from cAdvisor don't consistently follow naming conventions, +// so we need to whitelist them and handle them on a case-by-case basis. Metrics ending in `_total` +// *should* be counters, but may actually be guages in this case. + +// SeriesType represents the kind of series backing a metric. +type SeriesType int + +const ( + CounterSeries SeriesType = iota + SecondsCounterSeries + GaugeSeries +) + +// SeriesRegistry provides conversions between Prometheus series and MetricInfo +type SeriesRegistry interface { + // SetSeries replaces the known series in this registry. + // Each slice in series should correspond to a MetricNamer in namers. + SetSeries(series [][]prom.Series, namers []MetricNamer) error + // ListAllMetrics lists all metrics known to this registry + ListAllMetrics() []provider.CustomMetricInfo + // SeriesForMetric looks up the minimum required series information to make a query for the given metric + // against the given resource (namespace may be empty for non-namespaced resources) + QueryForMetric(info provider.CustomMetricInfo, namespace string, resourceNames ...string) (query prom.Selector, found bool) + // MatchValuesToNames matches result values to resource names for the given metric and value set + MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) +} + +type seriesInfo struct { + // seriesName is the name of the corresponding Prometheus series + seriesName string + + // namer is the MetricNamer used to name this series + namer MetricNamer +} + +// overridableSeriesRegistry is a basic SeriesRegistry +type basicSeriesRegistry struct { + mu sync.RWMutex + + // info maps metric info to information about the corresponding series + info map[provider.CustomMetricInfo]seriesInfo + // metrics is the list of all known metrics + metrics []provider.CustomMetricInfo + + mapper apimeta.RESTMapper +} + +func (r *basicSeriesRegistry) SetSeries(newSeriesSlices [][]prom.Series, namers []MetricNamer) error { + if len(newSeriesSlices) != len(namers) { + return fmt.Errorf("need one set of series per namer") + } + + newInfo := make(map[provider.CustomMetricInfo]seriesInfo) + for i, newSeries := range newSeriesSlices { + namer := namers[i] + for _, series := range newSeries { + // TODO: warn if it doesn't match any resources + resources, namespaced := namer.ResourcesForSeries(series) + name, err := namer.MetricNameForSeries(series) + if err != nil { + glog.Errorf("unable to name series %q, skipping: %v", series.String(), err) + continue + } + for _, resource := range resources { + info := provider.CustomMetricInfo{ + GroupResource: resource, + Namespaced: namespaced, + Metric: name, + } + + // namespace metrics aren't counted as namespaced + if resource == nsGroupResource { + info.Namespaced = false + } + + // we don't need to re-normalize, because the metric namer should have already normalized for us + newInfo[info] = seriesInfo{ + seriesName: series.Name, + namer: namer, + } + } + } + } + + // regenerate metrics + newMetrics := make([]provider.CustomMetricInfo, 0, len(newInfo)) + for info := range newInfo { + newMetrics = append(newMetrics, info) + } + + r.mu.Lock() + defer r.mu.Unlock() + + r.info = newInfo + r.metrics = newMetrics + + return nil +} + +func (r *basicSeriesRegistry) ListAllMetrics() []provider.CustomMetricInfo { + r.mu.RLock() + defer r.mu.RUnlock() + + return r.metrics +} + +func (r *basicSeriesRegistry) QueryForMetric(metricInfo provider.CustomMetricInfo, namespace string, resourceNames ...string) (prom.Selector, bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + if len(resourceNames) == 0 { + glog.Errorf("no resource names requested while producing a query for metric %s", metricInfo.String()) + return "", false + } + + metricInfo, _, err := metricInfo.Normalized(r.mapper) + if err != nil { + glog.Errorf("unable to normalize group resource while producing a query: %v", err) + return "", false + } + + info, infoFound := r.info[metricInfo] + if !infoFound { + glog.V(10).Infof("metric %v not registered", metricInfo) + return "", false + } + + query, err := info.namer.QueryForSeries(info.seriesName, metricInfo.GroupResource, namespace, resourceNames...) + if err != nil { + glog.Errorf("unable to construct query for metric %s: %v", metricInfo.String(), err) + return "", false + } + + return query, true +} + +func (r *basicSeriesRegistry) MatchValuesToNames(metricInfo provider.CustomMetricInfo, values pmodel.Vector) (matchedValues map[string]pmodel.SampleValue, found bool) { + r.mu.RLock() + defer r.mu.RUnlock() + + metricInfo, _, err := metricInfo.Normalized(r.mapper) + if err != nil { + glog.Errorf("unable to normalize group resource while matching values to names: %v", err) + return nil, false + } + + info, infoFound := r.info[metricInfo] + if !infoFound { + return nil, false + } + + resourceLbl, err := info.namer.LabelForResource(metricInfo.GroupResource) + if err != nil { + glog.Errorf("unable to construct resource label for metric %s: %v", metricInfo.String(), err) + return nil, false + } + + res := make(map[string]pmodel.SampleValue, len(values)) + for _, val := range values { + if val == nil { + // skip empty values + continue + } + res[string(val.Metric[resourceLbl])] = val.Value + } + + return res, true +} diff --git a/pkg/custom-provider/series_registry_test.go b/pkg/custom-provider/series_registry_test.go new file mode 100644 index 00000000..ed5eb11f --- /dev/null +++ b/pkg/custom-provider/series_registry_test.go @@ -0,0 +1,320 @@ +/* +Copyright 2017 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package provider + +import ( + "sort" + "testing" + "time" + + "github.com/kubernetes-incubator/custom-metrics-apiserver/pkg/provider" + pmodel "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + coreapi "k8s.io/api/core/v1" + extapi "k8s.io/api/extensions/v1beta1" + apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime/schema" + + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" + prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" +) + +// restMapper creates a RESTMapper with just the types we need for +// these tests. +func restMapper() apimeta.RESTMapper { + mapper := apimeta.NewDefaultRESTMapper([]schema.GroupVersion{coreapi.SchemeGroupVersion}) + + mapper.Add(coreapi.SchemeGroupVersion.WithKind("Pod"), apimeta.RESTScopeNamespace) + mapper.Add(coreapi.SchemeGroupVersion.WithKind("Service"), apimeta.RESTScopeNamespace) + mapper.Add(extapi.SchemeGroupVersion.WithKind("Ingress"), apimeta.RESTScopeNamespace) + mapper.Add(extapi.SchemeGroupVersion.WithKind("Deployment"), apimeta.RESTScopeNamespace) + + mapper.Add(coreapi.SchemeGroupVersion.WithKind("Node"), apimeta.RESTScopeRoot) + mapper.Add(coreapi.SchemeGroupVersion.WithKind("PersistentVolume"), apimeta.RESTScopeRoot) + mapper.Add(coreapi.SchemeGroupVersion.WithKind("Namespace"), apimeta.RESTScopeRoot) + + return mapper +} + +func setupMetricNamer(t testing.TB) []MetricNamer { + cfg := config.DefaultConfig(1*time.Minute, "kube_") + namers, err := NamersFromConfig(cfg, restMapper()) + require.NoError(t, err) + return namers +} + +var seriesRegistryTestSeries = [][]prom.Series{ + // container series + { + { + Name: "container_some_time_seconds_total", + Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, + }, + }, + { + { + Name: "container_some_count_total", + Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, + }, + }, + { + { + Name: "container_some_usage", + Labels: pmodel.LabelSet{"pod_name": "somepod", "namespace": "somens", "container_name": "somecont"}, + }, + }, + { + // guage metrics + { + Name: "node_gigawatts", + Labels: pmodel.LabelSet{"kube_node": "somenode"}, + }, + { + Name: "service_proxy_packets", + Labels: pmodel.LabelSet{"kube_service": "somesvc", "kube_namespace": "somens"}, + }, + }, + { + // cumulative --> rate metrics + { + Name: "ingress_hits_total", + Labels: pmodel.LabelSet{"kube_ingress": "someingress", "kube_service": "somesvc", "kube_pod": "backend1", "kube_namespace": "somens"}, + }, + { + Name: "ingress_hits_total", + Labels: pmodel.LabelSet{"kube_ingress": "someingress", "kube_service": "somesvc", "kube_pod": "backend2", "kube_namespace": "somens"}, + }, + { + Name: "volume_claims_total", + Labels: pmodel.LabelSet{"kube_persistentvolume": "somepv"}, + }, + }, + { + // cumulative seconds --> rate metrics + { + Name: "work_queue_wait_seconds_total", + Labels: pmodel.LabelSet{"kube_deployment": "somedep", "kube_namespace": "somens"}, + }, + { + Name: "node_fan_seconds_total", + Labels: pmodel.LabelSet{"kube_node": "somenode"}, + }, + }, +} + +func TestSeriesRegistry(t *testing.T) { + assert := assert.New(t) + require := require.New(t) + + namers := setupMetricNamer(t) + registry := &basicSeriesRegistry{ + mapper: restMapper(), + } + + // set up the registry + require.NoError(registry.SetSeries(seriesRegistryTestSeries, namers)) + + // make sure each metric got registered and can form queries + testCases := []struct { + title string + info provider.CustomMetricInfo + namespace string + resourceNames []string + + expectedQuery string + }{ + // container metrics + { + title: "container metrics gauge / multiple resource names", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + namespace: "somens", + resourceNames: []string{"somepod1", "somepod2"}, + + expectedQuery: "sum(container_some_usage{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}) by (pod_name)", + }, + { + title: "container metrics counter", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_count"}, + namespace: "somens", + resourceNames: []string{"somepod1", "somepod2"}, + + expectedQuery: "sum(rate(container_some_count_total{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}[1m])) by (pod_name)", + }, + { + title: "container metrics seconds counter", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pods"}, true, "some_time"}, + namespace: "somens", + resourceNames: []string{"somepod1", "somepod2"}, + + expectedQuery: "sum(rate(container_some_time_seconds_total{namespace=\"somens\",pod_name=~\"somepod1|somepod2\",container_name!=\"POD\"}[1m])) by (pod_name)", + }, + // namespaced metrics + { + title: "namespaced metrics counter / multidimensional (service)", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "service"}, true, "ingress_hits"}, + namespace: "somens", + resourceNames: []string{"somesvc"}, + + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_service=\"somesvc\"}[1m])) by (kube_service)", + }, + { + title: "namespaced metrics counter / multidimensional (ingress)", + info: provider.CustomMetricInfo{schema.GroupResource{Group: "extensions", Resource: "ingress"}, true, "ingress_hits"}, + namespace: "somens", + resourceNames: []string{"someingress"}, + + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_ingress=\"someingress\"}[1m])) by (kube_ingress)", + }, + { + title: "namespaced metrics counter / multidimensional (pod)", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "pod"}, true, "ingress_hits"}, + namespace: "somens", + resourceNames: []string{"somepod"}, + + expectedQuery: "sum(rate(ingress_hits_total{kube_namespace=\"somens\",kube_pod=\"somepod\"}[1m])) by (kube_pod)", + }, + { + title: "namespaced metrics gauge", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "service"}, true, "service_proxy_packets"}, + namespace: "somens", + resourceNames: []string{"somesvc"}, + + expectedQuery: "sum(service_proxy_packets{kube_namespace=\"somens\",kube_service=\"somesvc\"}) by (kube_service)", + }, + { + title: "namespaced metrics seconds counter", + info: provider.CustomMetricInfo{schema.GroupResource{Group: "extensions", Resource: "deployment"}, true, "work_queue_wait"}, + namespace: "somens", + resourceNames: []string{"somedep"}, + + expectedQuery: "sum(rate(work_queue_wait_seconds_total{kube_namespace=\"somens\",kube_deployment=\"somedep\"}[1m])) by (kube_deployment)", + }, + // non-namespaced series + { + title: "root scoped metrics gauge", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "node"}, false, "node_gigawatts"}, + resourceNames: []string{"somenode"}, + + expectedQuery: "sum(node_gigawatts{kube_node=\"somenode\"}) by (kube_node)", + }, + { + title: "root scoped metrics counter", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "persistentvolume"}, false, "volume_claims"}, + resourceNames: []string{"somepv"}, + + expectedQuery: "sum(rate(volume_claims_total{kube_persistentvolume=\"somepv\"}[1m])) by (kube_persistentvolume)", + }, + { + title: "root scoped metrics seconds counter", + info: provider.CustomMetricInfo{schema.GroupResource{Resource: "node"}, false, "node_fan"}, + resourceNames: []string{"somenode"}, + + expectedQuery: "sum(rate(node_fan_seconds_total{kube_node=\"somenode\"}[1m])) by (kube_node)", + }, + } + + for _, testCase := range testCases { + outputQuery, found := registry.QueryForMetric(testCase.info, testCase.namespace, testCase.resourceNames...) + if !assert.True(found, "%s: metric %v should available", testCase.title, testCase.info) { + continue + } + + assert.Equal(prom.Selector(testCase.expectedQuery), outputQuery, "%s: metric %v should have produced the correct query for %v in namespace %s", testCase.title, testCase.info, testCase.resourceNames, testCase.namespace) + } + + allMetrics := registry.ListAllMetrics() + expectedMetrics := []provider.CustomMetricInfo{ + {schema.GroupResource{Resource: "pods"}, true, "some_count"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_count"}, + {schema.GroupResource{Resource: "pods"}, true, "some_time"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_time"}, + {schema.GroupResource{Resource: "pods"}, true, "some_usage"}, + {schema.GroupResource{Resource: "namespaces"}, false, "some_usage"}, + {schema.GroupResource{Resource: "services"}, true, "ingress_hits"}, + {schema.GroupResource{Group: "extensions", Resource: "ingresses"}, true, "ingress_hits"}, + {schema.GroupResource{Resource: "pods"}, true, "ingress_hits"}, + {schema.GroupResource{Resource: "namespaces"}, false, "ingress_hits"}, + {schema.GroupResource{Resource: "services"}, true, "service_proxy_packets"}, + {schema.GroupResource{Resource: "namespaces"}, false, "service_proxy_packets"}, + {schema.GroupResource{Group: "extensions", Resource: "deployments"}, true, "work_queue_wait"}, + {schema.GroupResource{Resource: "namespaces"}, false, "work_queue_wait"}, + {schema.GroupResource{Resource: "nodes"}, false, "node_gigawatts"}, + {schema.GroupResource{Resource: "persistentvolumes"}, false, "volume_claims"}, + {schema.GroupResource{Resource: "nodes"}, false, "node_fan"}, + } + + // sort both for easy comparison + sort.Sort(metricInfoSorter(allMetrics)) + sort.Sort(metricInfoSorter(expectedMetrics)) + + assert.Equal(expectedMetrics, allMetrics, "should have listed all expected metrics") +} + +func BenchmarkSetSeries(b *testing.B) { + namers := setupMetricNamer(b) + registry := &basicSeriesRegistry{ + mapper: restMapper(), + } + + numDuplicates := 10000 + newSeriesSlices := make([][]prom.Series, len(seriesRegistryTestSeries)) + for i, seriesSlice := range seriesRegistryTestSeries { + newSlice := make([]prom.Series, len(seriesSlice)*numDuplicates) + for j, series := range seriesSlice { + for k := 0; k < numDuplicates; k++ { + newSlice[j*numDuplicates+k] = series + } + } + newSeriesSlices[i] = newSlice + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + registry.SetSeries(newSeriesSlices, namers) + } +} + +// metricInfoSorter is a sort.Interface for sorting provider.CustomMetricInfos +type metricInfoSorter []provider.CustomMetricInfo + +func (s metricInfoSorter) Len() int { + return len(s) +} + +func (s metricInfoSorter) Less(i, j int) bool { + infoI := s[i] + infoJ := s[j] + + if infoI.Metric == infoJ.Metric { + if infoI.GroupResource == infoJ.GroupResource { + return infoI.Namespaced + } + + if infoI.GroupResource.Group == infoJ.GroupResource.Group { + return infoI.GroupResource.Resource < infoJ.GroupResource.Resource + } + + return infoI.GroupResource.Group < infoJ.GroupResource.Group + } + + return infoI.Metric < infoJ.Metric +} + +func (s metricInfoSorter) Swap(i, j int) { + s[i], s[j] = s[j], s[i] +}