diff --git a/docs/externalmetrics.md b/docs/externalmetrics.md new file mode 100644 index 00000000..461b8788 --- /dev/null +++ b/docs/externalmetrics.md @@ -0,0 +1,84 @@ +External Metrics +=========== + +It's possible to configure [Autoscaling on metrics not related to Kubernetes objects](https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects) in Kubernetes. This is done with a special `External Metrics` system. Using external metrics in Kubernetes with the adapter requires you to configure special `external` rules in the configuration. + +The configuration for `external` metrics rules is almost identical to the normal `rules`: + +```yaml +external: +- seriesQuery: '{__name__="queue_consumer_lag",name!=""}' + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name) + resources: + overrides: { namespace: {resource: "namespace"} } +``` + +Namespacing +----------- + +All Kubernetes Horizontal Pod Autoscaler (HPA) resources are namespaced. And when you create an HPA that +references an external metric the adapter will automatically add a `namespace` label to the `seriesQuery` you have configured. + +This is done because the External Merics API Specification *requires* a namespace component in the URL: + +```shell +kubectl get --raw "/apis/external.metrics.k8s.io/v1beta1/namespaces/default/queue_consumer_lag" +``` + +Cross-Namespace or No Namespace Queries +--------------------------------------- + +A semi-common scenario is to have a `workload` in one namespace that needs to scale based on a metric from a different namespace. This is normally not +possible with `external` rules because the `namespace` label is set to match that of the source `workload`. + +However, you can explicitly disable the automatic add of the HPA namepace to the query, and instead opt to not set a namespace at all, or to target a different namespace. + +This is done by setting `namespaced: false` in the `resources` section of the `external` rule: + +```yaml +# rules: ... + +external: +- seriesQuery: '{__name__="queue_depth",name!=""}' + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name) + resources: + namespaced: false +``` + +Given the `external` rules defined above any `External` metric query for `queue_depth` will simply ignore the source `namespace` of the HPA. This allows you to explicilty not put a namespace into an external query, or to set the namespace to one that might be different from that of the HPA. + +```yaml +apiVersion: autoscaling/v1 +kind: HorizontalPodAutoscaler +metadata: + name: external-queue-scaler + # the HPA and scaleTargetRef must exist in a namespace + namespace: default + annotations: + # The "External" metric below targets a metricName that has namespaced=false + # and this allows the metric to explicitly query a different + # namespace than that of the HPA and scaleTargetRef + autoscaling.alpha.kubernetes.io/metrics: | + [ + { + "type": "External", + "external": { + "metricName": "queue_depth", + "metricSelector": { + "matchLabels": { + "namespace": "queue", + "name": "my-sample-queue" + } + }, + "targetAverageValue": "50" + } + } + ] +spec: + maxReplicas: 5 + minReplicas: 1 + scaleTargetRef: + apiVersion: apps/v1 + kind: Deployment + name: my-app +``` diff --git a/docs/sample-config.yaml b/docs/sample-config.yaml index 61676918..61c9dda6 100644 --- a/docs/sample-config.yaml +++ b/docs/sample-config.yaml @@ -65,5 +65,19 @@ rules: brand: {group: "cheese.io", resource: "brand"} metricQuery: 'count(cheddar{sharp="true"})' +# external rules are not tied to a Kubernetes resource and can reference any metric +# https://kubernetes.io/docs/tasks/run-application/horizontal-pod-autoscale-walkthrough/#autoscaling-on-metrics-not-related-to-kubernetes-objects +external: +- seriesQuery: '{__name__="queue_consumer_lag",name!=""}' + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name) +- seriesQuery: '{__name__="queue_depth",topic!=""}' + metricsQuery: sum(<<.Series>>{<<.LabelMatchers>>}) by (name) + # Kubernetes metric queries include a namespace in the query by default + # but you can explicitly disable namespaces if needed with "namespaced: false" + # this is useful if you have an HPA with an external metric in namespace A + # but want to query for metrics from namespace B + resources: + namespaced: false + # TODO: should we be able to map to a constant instance of a resource # (e.g. `resources: {constant: [{resource: "namespace", name: "kube-system"}}]`)? diff --git a/pkg/config/config.go b/pkg/config/config.go index fee4e006..62c83cbe 100644 --- a/pkg/config/config.go +++ b/pkg/config/config.go @@ -58,6 +58,8 @@ type ResourceMapping struct { // Overrides specifies exceptions to the above template, mapping label names // to group-resources Overrides map[string]GroupResource `json:"overrides,omitempty" yaml:"overrides,omitempty"` + // Namespaced ignores the source namespace of the requester and requires one in the query + Namespaced *bool `json:"namespaced,omitempty" yaml:"namespaced,omitempty"` } // GroupResource represents a Kubernetes group-resource. diff --git a/pkg/naming/metric_namer.go b/pkg/naming/metric_namer.go index 54ac7183..b92c7610 100644 --- a/pkg/naming/metric_namer.go +++ b/pkg/naming/metric_namer.go @@ -131,8 +131,6 @@ func (n *metricNamer) QueryForSeries(series string, resource schema.GroupResourc } func (n *metricNamer) QueryForExternalSeries(series string, namespace string, metricSelector labels.Selector) (prom.Selector, error) { - //test := prom.Selector() - //return test, nil return n.metricsQuery.BuildExternal(series, namespace, "", []string{}, metricSelector) } @@ -155,7 +153,13 @@ func NamersFromConfig(cfg []config.DiscoveryRule, mapper apimeta.RESTMapper) ([] return nil, err } - metricsQuery, err := NewMetricsQuery(rule.MetricsQuery, resConv) + // queries are namespaced by default unless the rule specifically disables it + namespaced := true + if rule.Resources.Namespaced != nil { + namespaced = *rule.Resources.Namespaced + } + + metricsQuery, err := NewExternalMetricsQuery(rule.MetricsQuery, resConv, namespaced) if err != nil { return nil, fmt.Errorf("unable to construct metrics query associated with series query %q: %v", rule.SeriesQuery, err) } diff --git a/pkg/naming/metrics_query.go b/pkg/naming/metrics_query.go index 2f289262..0dbc13a5 100644 --- a/pkg/naming/metrics_query.go +++ b/pkg/naming/metrics_query.go @@ -59,6 +59,27 @@ func NewMetricsQuery(queryTemplate string, resourceConverter ResourceConverter) return &metricsQuery{ resConverter: resourceConverter, template: templ, + namespaced: true, + }, nil +} + +// NewExternalMetricsQuery constructs a new MetricsQuery by compiling the given Go template. +// The delimiters on the template are `<<` and `>>`, and it may use the following fields: +// - Series: the series in question +// - LabelMatchers: a pre-stringified form of the label matchers for the resources in the query +// - LabelMatchersByName: the raw map-form of the above matchers +// - GroupBy: the group-by clause to use for the resources in the query (stringified) +// - GroupBySlice: the raw slice form of the above group-by clause +func NewExternalMetricsQuery(queryTemplate string, resourceConverter ResourceConverter, namespaced bool) (MetricsQuery, error) { + templ, err := template.New("metrics-query").Delims("<<", ">>").Parse(queryTemplate) + if err != nil { + return nil, fmt.Errorf("unable to parse metrics query template %q: %v", queryTemplate, err) + } + + return &metricsQuery{ + resConverter: resourceConverter, + template: templ, + namespaced: namespaced, }, nil } @@ -68,6 +89,7 @@ func NewMetricsQuery(queryTemplate string, resourceConverter ResourceConverter) type metricsQuery struct { resConverter ResourceConverter template *template.Template + namespaced bool } // queryTemplateArgs contains the arguments for the template used in metricsQuery. @@ -150,7 +172,7 @@ func (q *metricsQuery) BuildExternal(seriesName string, namespace string, groupB // Build up the query parts from the selector. queryParts = append(queryParts, q.createQueryPartsFromSelector(metricSelector)...) - if namespace != "" { + if q.namespaced && namespace != "" { namespaceLbl, err := q.resConverter.LabelForResource(NsGroupResource) if err != nil { return "", err diff --git a/pkg/naming/metrics_query_test.go b/pkg/naming/metrics_query_test.go index 9661d15d..ebd23ce7 100644 --- a/pkg/naming/metrics_query_test.go +++ b/pkg/naming/metrics_query_test.go @@ -272,7 +272,15 @@ func TestBuildSelector(t *testing.T) { func TestBuildExternalSelector(t *testing.T) { mustNewQuery := func(queryTemplate string) MetricsQuery { - mq, err := NewMetricsQuery(queryTemplate, &resourceConverterMock{true}) + mq, err := NewExternalMetricsQuery(queryTemplate, &resourceConverterMock{true}, true) + if err != nil { + t.Fatal(err) + } + return mq + } + + mustNewNonNamespacedQuery := func(queryTemplate string) MetricsQuery { + mq, err := NewExternalMetricsQuery(queryTemplate, &resourceConverterMock{true}, false) if err != nil { t.Fatal(err) } @@ -348,6 +356,19 @@ func TestBuildExternalSelector(t *testing.T) { hasSelector("default [foo bar]"), ), }, + { + name: "multiple GroupBySlice values with namespace disabled", + + mq: mustNewNonNamespacedQuery(`<> <<.GroupBySlice>>`), + namespace: "default", + groupBySlice: []string{"foo", "bar"}, + metricSelector: labels.NewSelector(), + + check: checks( + hasError(nil), + hasSelector(" [foo bar]"), + ), + }, { name: "single LabelMatchers value",