Advanced Configuration

This commit introduces advanced configuration.  The rate-interval and
label-prefix flags are removed, and replaced by a configuration file
that allows you to specify series queries and the rules for transforming
those into metrics queries and API resources.

pkg/config/config.go

@@ -0,0 +1,75 @@
+package config
+
+type MetricsDiscoveryConfig struct {
+	// Rules specifies how to discover and map Prometheus metrics to
+	// custom metrics API resources.  The rules are applied independently,
+	// and thus must be mutually exclusive.  Rules will the same SeriesQuery
+	// will make only a single API call.
+	Rules []DiscoveryRule `yaml:"rules"`
+}
+
+// DiscoveryRule describes on set of rules for transforming Prometheus metrics to/from
+// custom metrics API resources.
+type DiscoveryRule struct {
+	// SeriesQuery specifies which metrics this rule should consider via a Prometheus query
+	// series selector query.
+	SeriesQuery string `yaml:"seriesQuery"`
+	// SeriesFilters specifies additional regular expressions to be applied on
+	// the series names returned from the query.  This is useful for constraints
+	// that can't be represented in the SeriesQuery (e.g. series matching `container_.+`
+	// not matching `container_.+_total`.  A filter will be automatically appended to
+	// match the form specified in Name.
+	SeriesFilters []RegexFilter `yaml:"seriesFilter"`
+	// Resources specifies how associated Kubernetes resources should be discovered for
+	// the given metrics.
+	Resources ResourceMapping `yaml:"resources"`
+	// Name specifies how the metric name should be transformed between custom metric
+	// API resources, and Prometheus metric names.
+	Name NameMapping `yaml:"name"`
+	// MetricsQuery specifies modifications to the metrics query, such as converting
+	// cumulative metrics to rate metrics.  It is a template where `.LabelMatchers` is
+	// a the comma-separated base label matchers and `.Series` is the series name, and
+	// `.GroupBy` is the comma-separated expected group-by label names. The delimeters
+	// are `${` and `}$`.
+	MetricsQuery string `yaml:"metricsQueries,omitempty"`
+}
+
+// RegexFilter is a filter that matches positively or negatively against a regex.
+// Only one field may be set at a time.
+type RegexFilter struct {
+	Is    string `yaml:"is,omitempty"`
+	IsNot string `yaml:"isNot,omitempty"`
+}
+
+// ResourceMapping specifies how to map Kubernetes resources to Prometheus labels
+type ResourceMapping struct {
+	// Template specifies a golang string template for converting a Kubernetes
+	// group-resource to a Prometheus label.  The template object contains
+	// the `.Group` and `.Resource` fields.  The `.Group` field will have
+	// dots replaced with underscores, and the `.Resource` field will be
+	// singularized.  The delimiters are `${` and `}$`.
+	Template string `yaml:"template,omitempty"`
+	// Overrides specifies exceptions to the above template, mapping label names
+	// to group-resources
+	Overrides map[string]GroupResource `yaml:"overrides,omitempty"`
+}
+
+// GroupResource represents a Kubernetes group-resource.
+type GroupResource struct {
+	Group    string `yaml:"group,omitempty"`
+	Resource string `yaml:"resource"`
+}
+
+// NameMapping specifies how to convert Prometheus metrics
+// to/from custom metrics API resources.
+type NameMapping struct {
+	// Matches is a regular expression that is used to match
+	// Prometheus series names.  It may be left blank, in which
+	// case it is equivalent to `.*`.
+	Matches string `yaml:"prometheus"`
+	// As is the name used in the API.  Captures from Matches
+	// are available for use here.  If not specified, it defaults
+	// to $0 if no capture groups are present in Matches, or $1
+	// if only one is present, and will error if multiple are.
+	As string `yaml:"as"`
+}

pkg/config/default.go

@@ -0,0 +1,92 @@
+package config
+
+import (
+	"fmt"
+	"time"
+
+	prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client"
+	pmodel "github.com/prometheus/common/model"
+)
+
+// DefaultConfig returns a configuration equivalent to the former
+// pre-advanced-config settings.  This means that "normal" series labels
+// will be of the form `<prefix>${.Resource}$`, cadvisor series will be
+// of the form `container_`, and have the label `pod_name`.  Any series ending
+// in total will be treated as a rate metric.
+func DefaultConfig(rateInterval time.Duration, labelPrefix string) *MetricsDiscoveryConfig {
+	return &MetricsDiscoveryConfig{
+		Rules: []DiscoveryRule{
+			// container seconds rate metrics
+			{
+				SeriesQuery: string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))),
+				Resources: ResourceMapping{
+					Overrides: map[string]GroupResource{
+						"namespace": {Resource: "namespace"},
+						"pod_name":  {Resource: "pod"},
+					},
+				},
+				Name:         NameMapping{Matches: "^container_(.*)_seconds_total$"},
+				MetricsQuery: fmt.Sprintf(`sum(rate(${.Series}${${.LabelMatchers}$,container_name!="POD"}[%s])) by (${.GroupBy}$)`, pmodel.Duration(rateInterval).String()),
+			},
+
+			// container rate metrics
+			{
+				SeriesQuery:   string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))),
+				SeriesFilters: []RegexFilter{{IsNot: "^container_.*_seconds_total$"}},
+				Resources: ResourceMapping{
+					Overrides: map[string]GroupResource{
+						"namespace": {Resource: "namespace"},
+						"pod_name":  {Resource: "pod"},
+					},
+				},
+				Name:         NameMapping{Matches: "^container_(.*)_total$"},
+				MetricsQuery: fmt.Sprintf(`sum(rate(${.Series}${${.LabelMatchers}$,container_name!="POD"}[%s])) by (${.GroupBy}$)`, pmodel.Duration(rateInterval).String()),
+			},
+
+			// container non-cumulative metrics
+			{
+				SeriesQuery:   string(prom.MatchSeries("", prom.NameMatches("^container_.*"), prom.LabelNeq("container_name", "POD"), prom.LabelNeq("namespace", ""), prom.LabelNeq("pod_name", ""))),
+				SeriesFilters: []RegexFilter{{IsNot: "^container_.*_total$"}},
+				Resources: ResourceMapping{
+					Overrides: map[string]GroupResource{
+						"namespace": {Resource: "namespace"},
+						"pod_name":  {Resource: "pod"},
+					},
+				},
+				Name:         NameMapping{Matches: "^container_(.*)$"},
+				MetricsQuery: `sum(${.Series}${${.LabelMatchers}$,container_name!="POD"}) by (${.GroupBy}$)`,
+			},
+
+			// normal non-cumulative metrics
+			{
+				SeriesQuery:   string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
+				SeriesFilters: []RegexFilter{{IsNot: ".*_total$"}},
+				Resources: ResourceMapping{
+					Template: fmt.Sprintf("%s${.Resource}$", labelPrefix),
+				},
+				MetricsQuery: "sum(${.Series}${${.LabelMatchers}$}) by (${.GroupBy}$)",
+			},
+
+			// normal rate metrics
+			{
+				SeriesQuery:   string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
+				SeriesFilters: []RegexFilter{{IsNot: ".*_seconds_total"}},
+				Name:          NameMapping{Matches: "^(.*)_total$"},
+				Resources: ResourceMapping{
+					Template: fmt.Sprintf("%s${.Resource}$", labelPrefix),
+				},
+				MetricsQuery: fmt.Sprintf("sum(rate(${.Series}${${.LabelMatchers}$}[%s])) by (${.GroupBy}$)", pmodel.Duration(rateInterval).String()),
+			},
+
+			// seconds rate metrics
+			{
+				SeriesQuery: string(prom.MatchSeries("", prom.LabelNeq(fmt.Sprintf("%snamespace", labelPrefix), ""), prom.NameNotMatches("^container_.*"))),
+				Name:        NameMapping{Matches: "^(.*)_seconds_total$"},
+				Resources: ResourceMapping{
+					Template: fmt.Sprintf("%s${.Resource}$", labelPrefix),
+				},
+				MetricsQuery: fmt.Sprintf("sum(rate(${.Series}${${.LabelMatchers}$}[%s])) by (${.GroupBy}$)", pmodel.Duration(rateInterval).String()),
+			},
+		},
+	}
+}

pkg/config/loader.go

@@ -0,0 +1,32 @@
+package config
+
+import (
+	"fmt"
+	"io/ioutil"
+	"os"
+
+	yaml "gopkg.in/yaml.v2"
+)
+
+// FromFile loads the configuration from a particular file.
+func FromFile(filename string) (*MetricsDiscoveryConfig, error) {
+	file, err := os.Open(filename)
+	defer file.Close()
+	if err != nil {
+		return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err)
+	}
+	contents, err := ioutil.ReadAll(file)
+	if err != nil {
+		return nil, fmt.Errorf("unable to load metrics discovery config file: %v", err)
+	}
+	return FromYAML(contents)
+}
+
+// FromYAML loads the configuration from a blob of YAML.
+func FromYAML(contents []byte) (*MetricsDiscoveryConfig, error) {
+	var cfg MetricsDiscoveryConfig
+	if err := yaml.Unmarshal(contents, &cfg); err != nil {
+		return nil, fmt.Errorf("unable to parse metrics discovery config: %v", err)
+	}
+	return &cfg, nil
+}