diff --git a/cmd/adapter/app/start.go b/cmd/adapter/app/start.go index 46bb3590..1a1e0fe3 100644 --- a/cmd/adapter/app/start.go +++ b/cmd/adapter/app/start.go @@ -44,9 +44,7 @@ func NewCommandStartPrometheusAdapterServer(out, errOut io.Writer, stopCh <-chan o := PrometheusAdapterServerOptions{ CustomMetricsAdapterServerOptions: baseOpts, MetricsRelistInterval: 10 * time.Minute, - RateInterval: 5 * time.Minute, PrometheusURL: "https://localhost", - DiscoveryInterval: 10 * time.Minute, } cmd := &cobra.Command{ @@ -77,25 +75,19 @@ func NewCommandStartPrometheusAdapterServer(out, errOut io.Writer, stopCh <-chan "any described objets") flags.DurationVar(&o.MetricsRelistInterval, "metrics-relist-interval", o.MetricsRelistInterval, ""+ "interval at which to re-list the set of all available metrics from Prometheus") - flags.DurationVar(&o.RateInterval, "rate-interval", o.RateInterval, ""+ - "period of time used to calculate rate metrics from cumulative metrics") flags.DurationVar(&o.DiscoveryInterval, "discovery-interval", o.DiscoveryInterval, ""+ "interval at which to refresh API discovery information") flags.StringVar(&o.PrometheusURL, "prometheus-url", o.PrometheusURL, - "URL for connecting to Prometheus. Query parameters are used to configure the connection") + "URL for connecting to Prometheus.") flags.BoolVar(&o.PrometheusAuthInCluster, "prometheus-auth-incluster", o.PrometheusAuthInCluster, "use auth details from the in-cluster kubeconfig when connecting to prometheus.") flags.StringVar(&o.PrometheusAuthConf, "prometheus-auth-config", o.PrometheusAuthConf, "kubeconfig file used to configure auth when connecting to Prometheus.") - flags.StringVar(&o.LabelPrefix, "label-prefix", o.LabelPrefix, - "Prefix to expect on labels referring to pod resources. For example, if the prefix is "+ - "'kube_', any series with the 'kube_pod' label would be considered a pod metric") flags.StringVar(&o.AdapterConfigFile, "config", o.AdapterConfigFile, "Configuration file containing details of how to transform between Prometheus metrics "+ "and custom metrics API resources") - flags.MarkDeprecated("label-prefix", "use --config instead") - flags.MarkDeprecated("discovery-interval", "use --config instead") + cmd.MarkFlagRequired("config") return cmd } @@ -136,15 +128,13 @@ func makeHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, er } func (o PrometheusAdapterServerOptions) RunCustomMetricsAdapterServer(stopCh <-chan struct{}) error { - var metricsConfig *adaptercfg.MetricsDiscoveryConfig - if o.AdapterConfigFile != "" { - var err error - metricsConfig, err = adaptercfg.FromFile(o.AdapterConfigFile) - if err != nil { - return fmt.Errorf("unable to load metrics discovery configuration: %v", err) - } - } else { - metricsConfig = adaptercfg.DefaultConfig(o.RateInterval, o.LabelPrefix) + if o.AdapterConfigFile == "" { + return fmt.Errorf("no discovery configuration file specified") + } + + metricsConfig, err := adaptercfg.FromFile(o.AdapterConfigFile) + if err != nil { + return fmt.Errorf("unable to load metrics discovery configuration: %v", err) } config, err := o.Config() @@ -217,8 +207,6 @@ type PrometheusAdapterServerOptions struct { RemoteKubeConfigFile string // MetricsRelistInterval is the interval at which to relist the set of available metrics MetricsRelistInterval time.Duration - // RateInterval is the period of time used to calculate rate metrics - RateInterval time.Duration // DiscoveryInterval is the interval at which discovery information is refreshed DiscoveryInterval time.Duration // PrometheusURL is the URL describing how to connect to Prometheus. Query parameters configure connection options. @@ -227,9 +215,6 @@ type PrometheusAdapterServerOptions struct { PrometheusAuthInCluster bool // PrometheusAuthConf is the kubeconfig file that contains auth details used to connect to Prometheus PrometheusAuthConf string - // LabelPrefix is the prefix to expect on labels for Kubernetes resources - // (e.g. if the prefix is "kube_", we'd expect a "kube_pod" label for pod metrics). - LabelPrefix string // AdapterConfigFile points to the file containing the metrics discovery configuration. AdapterConfigFile string } diff --git a/cmd/config-gen/main.go b/cmd/config-gen/main.go new file mode 100644 index 00000000..b3d8f7e1 --- /dev/null +++ b/cmd/config-gen/main.go @@ -0,0 +1,44 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/spf13/cobra" + yaml "gopkg.in/yaml.v2" + + "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" +) + +func main() { + var labelPrefix string + var rateInterval time.Duration + + cmd := &cobra.Command{ + Short: "Generate a config matching the legacy discovery rules", + Long: `Generate a config that produces the same functionality +as the legacy discovery rules. This includes discovering metrics and associating +resources according to the Kubernetes instrumention conventions and the cAdvisor +conventions, and auto-converting cumulative metrics into rate metrics.`, + RunE: func(c *cobra.Command, args []string) error { + cfg := utils.DefaultConfig(rateInterval, labelPrefix) + enc := yaml.NewEncoder(os.Stdout) + if err := enc.Encode(cfg); err != nil { + return err + } + return enc.Close() + }, + } + + cmd.Flags().StringVar(&labelPrefix, "label-prefix", "", + "Prefix to expect on labels referring to pod resources. For example, if the prefix is "+ + "'kube_', any series with the 'kube_pod' label would be considered a pod metric") + cmd.Flags().DurationVar(&rateInterval, "rate-interval", 5*time.Minute, + "Period of time used to calculate rate metrics from cumulative metrics") + + if err := cmd.Execute(); err != nil { + fmt.Fprintf(os.Stderr, "Unable to generate config: %v\n", err) + os.Exit(1) + } +} diff --git a/pkg/config/default.go b/cmd/config-gen/utils/default.go similarity index 75% rename from pkg/config/default.go rename to cmd/config-gen/utils/default.go index f780e509..d8873d96 100644 --- a/pkg/config/default.go +++ b/cmd/config-gen/utils/default.go @@ -1,16 +1,17 @@ -package config +package utils import ( "fmt" "time" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + . "github.com/directxman12/k8s-prometheus-adapter/pkg/config" pmodel "github.com/prometheus/common/model" ) // DefaultConfig returns a configuration equivalent to the former // pre-advanced-config settings. This means that "normal" series labels -// will be of the form `${.Resource}$`, cadvisor series will be +// will be of the form `<<.Resource>>`, cadvisor series will be // of the form `container_`, and have the label `pod_name`. Any series ending // in total will be treated as a rate metric. func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDiscoveryConfig { @@ -26,7 +27,7 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco }, }, Name: NameMapping{Matches: "^container_(.*)_seconds_total$"}, - MetricsQuery: fmt.Sprintf(`sum(rate(${.Series}${${.LabelMatchers}$,container_name!="POD"}[%s])) by (${.GroupBy}$)`, pmodel.Duration(rateInterval).String()), + MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()), }, // container rate metrics @@ -40,7 +41,7 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco }, }, Name: NameMapping{Matches: "^container_(.*)_total$"}, - MetricsQuery: fmt.Sprintf(`sum(rate(${.Series}${${.LabelMatchers}$,container_name!="POD"}[%s])) by (${.GroupBy}$)`, pmodel.Duration(rateInterval).String()), + MetricsQuery: fmt.Sprintf(`sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[%s])) by (<<.GroupBy>>)`, pmodel.Duration(rateInterval).String()), }, // container non-cumulative metrics @@ -54,7 +55,7 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco }, }, Name: NameMapping{Matches: "^container_(.*)$"}, - MetricsQuery: `sum(${.Series}${${.LabelMatchers}$,container_name!="POD"}) by (${.GroupBy}$)`, + MetricsQuery: `sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>)`, }, // normal non-cumulative metrics @@ -62,9 +63,9 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))), SeriesFilters: []RegexFilter{{IsNot: ".*_total$"}}, Resources: ResourceMapping{ - Template: fmt.Sprintf("%s${.Resource}$", labelPrefix), + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), }, - MetricsQuery: "sum(${.Series}${${.LabelMatchers}$}) by (${.GroupBy}$)", + MetricsQuery: "sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>)", }, // normal rate metrics @@ -73,9 +74,9 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco SeriesFilters: []RegexFilter{{IsNot: ".*_seconds_total"}}, Name: NameMapping{Matches: "^(.*)_total$"}, Resources: ResourceMapping{ - Template: fmt.Sprintf("%s${.Resource}$", labelPrefix), + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), }, - MetricsQuery: fmt.Sprintf("sum(rate(${.Series}${${.LabelMatchers}$}[%s])) by (${.GroupBy}$)", pmodel.Duration(rateInterval).String()), + MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()), }, // seconds rate metrics @@ -83,9 +84,9 @@ func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDisco SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))), Name: NameMapping{Matches: "^(.*)_seconds_total$"}, Resources: ResourceMapping{ - Template: fmt.Sprintf("%s${.Resource}$", labelPrefix), + Template: fmt.Sprintf("%s<<.Resource>>", labelPrefix), }, - MetricsQuery: fmt.Sprintf("sum(rate(${.Series}${${.LabelMatchers}$}[%s])) by (${.GroupBy}$)", pmodel.Duration(rateInterval).String()), + MetricsQuery: fmt.Sprintf("sum(rate(<<.Series>>{<<.LabelMatchers>>}[%s])) by (<<.GroupBy>>)", pmodel.Duration(rateInterval).String()), }, }, } diff --git a/deploy/manifests/custom-metrics-apiserver-deployment.yaml b/deploy/manifests/custom-metrics-apiserver-deployment.yaml index 848d4ec3..5ca0e55f 100644 --- a/deploy/manifests/custom-metrics-apiserver-deployment.yaml +++ b/deploy/manifests/custom-metrics-apiserver-deployment.yaml @@ -28,15 +28,21 @@ spec: - --logtostderr=true - --prometheus-url=http://prometheus.prom.svc:9090/ - --metrics-relist-interval=30s - - --rate-interval=5m - --v=10 + - --config=/default-config.yaml ports: - containerPort: 6443 volumeMounts: - mountPath: /var/run/serving-cert name: volume-serving-cert readOnly: true + - mountPath: /etc/adapter/ + name: config + readOnly: true volumes: - name: volume-serving-cert secret: secretName: cm-adapter-serving-certs + - name: config + configMap: + name: adapter-config diff --git a/deploy/manifests/custom-metrics-config-map.yaml b/deploy/manifests/custom-metrics-config-map.yaml new file mode 100644 index 00000000..04ee0c67 --- /dev/null +++ b/deploy/manifests/custom-metrics-config-map.yaml @@ -0,0 +1,74 @@ +apiVersion: v1 +kind: ConfigMap +metadata: + name: adapter-config + namespace: custom-metrics +data: + config.yaml: | + rules: + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: [] + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_seconds_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}[5m])) + by (<<.GroupBy>>) + - seriesQuery: '{__name__=~"^container_.*",container_name!="POD",namespace!="",pod_name!=""}' + seriesFilters: + - isNot: ^container_.*_total$ + resources: + overrides: + namespace: + resource: namespace + pod_name: + resource: pod + name: + matches: ^container_(.*)$ + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>,container_name!="POD"}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_total$ + resources: + template: <<.Resource>> + name: + matches: "" + as: "" + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: + - isNot: .*_seconds_total + resources: + template: <<.Resource>> + name: + matches: ^(.*)_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) + - seriesQuery: '{namespace!="",__name__!~"^container_.*"}' + seriesFilters: [] + resources: + template: <<.Resource>> + name: + matches: ^(.*)_seconds_total$ + as: "" + metricsQuery: sum(rate(<<.Series>>{<<.LabelMatchers>>}[5m])) by (<<.GroupBy>>) diff --git a/pkg/config/config.go b/pkg/config/config.go index 82443f0b..6a31e8f1 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -19,7 +19,7 @@ type DiscoveryRule struct { // that can't be represented in the SeriesQuery (e.g. series matching `container_.+` // not matching `container_.+_total`. A filter will be automatically appended to // match the form specified in Name. - SeriesFilters []RegexFilter `yaml:"seriesFilter"` + SeriesFilters []RegexFilter `yaml:"seriesFilters"` // Resources specifies how associated Kubernetes resources should be discovered for // the given metrics. Resources ResourceMapping `yaml:"resources"` diff --git a/pkg/custom-provider/metric_namer.go b/pkg/custom-provider/metric_namer.go index 650c4be4..c88e9bf0 100644 --- a/pkg/custom-provider/metric_namer.go +++ b/pkg/custom-provider/metric_namer.go @@ -338,7 +338,7 @@ func (n *metricNamer) makeLabelForResource(resource schema.GroupResource) (pmode singularRes, err := n.mapper.ResourceSingularizer(resource.Resource) if err != nil { - return "", fmt.Errorf("unable to singularize resource %s: %v", resource.String, err) + return "", fmt.Errorf("unable to singularize resource %s: %v", resource.String(), err) } convResource := schema.GroupResource{ Group: groupNameSanitizer.Replace(resource.Group), diff --git a/pkg/custom-provider/provider_test.go b/pkg/custom-provider/provider_test.go index 06a69c03..aa0f54d8 100644 --- a/pkg/custom-provider/provider_test.go +++ b/pkg/custom-provider/provider_test.go @@ -29,8 +29,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" fakedyn "k8s.io/client-go/dynamic/fake" + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/directxman12/k8s-prometheus-adapter/pkg/config" pmodel "github.com/prometheus/common/model" ) diff --git a/pkg/custom-provider/series_registry_test.go b/pkg/custom-provider/series_registry_test.go index 6bef6c45..b23bf5f8 100644 --- a/pkg/custom-provider/series_registry_test.go +++ b/pkg/custom-provider/series_registry_test.go @@ -30,8 +30,8 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + config "github.com/directxman12/k8s-prometheus-adapter/cmd/config-gen/utils" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/directxman12/k8s-prometheus-adapter/pkg/config" ) // restMapper creates a RESTMapper with just the types we need for