From ff409a099481bf23e937dd4f7669c0d48c0b51c2 Mon Sep 17 00:00:00 2001 From: John Delivuk Date: Tue, 26 Mar 2019 18:57:02 -0400 Subject: [PATCH] Refactoring external to leverage naming --- cmd/adapter/adapter.go | 12 +- pkg/external-provider/basic_metric_lister.go | 35 +- .../external_series_registry.go | 42 ++- pkg/external-provider/provider.go | 6 +- pkg/external-provider/query_builder.go | 219 ----------- pkg/external-provider/query_builder_test.go | 356 ------------------ pkg/external-provider/series_converter.go | 286 -------------- pkg/external-provider/series_filterer.go | 79 ---- pkg/external-provider/series_filterer_test.go | 127 ------- pkg/{external-provider => naming}/errors.go | 2 +- pkg/naming/metric_namer.go | 51 +-- pkg/naming/metrics_query.go | 201 ++++++++++ 12 files changed, 269 insertions(+), 1147 deletions(-) delete mode 100644 pkg/external-provider/query_builder.go delete mode 100644 pkg/external-provider/query_builder_test.go delete mode 100644 pkg/external-provider/series_converter.go delete mode 100644 pkg/external-provider/series_filterer.go delete mode 100644 pkg/external-provider/series_filterer_test.go rename pkg/{external-provider => naming}/errors.go (98%) diff --git a/cmd/adapter/adapter.go b/cmd/adapter/adapter.go index 2463810d..eb2aee4f 100644 --- a/cmd/adapter/adapter.go +++ b/cmd/adapter/adapter.go @@ -161,7 +161,7 @@ func (cmd *PrometheusAdapter) makeProvider(promClient prom.Client, stopCh <-chan } // extract the namers - namers, err := naming.NamersFromConfig(cmd.metricsConfig, mapper) + namers, err := naming.NamersFromConfig(cmd.metricsConfig.Rules, mapper) if err != nil { return nil, fmt.Errorf("unable to construct naming scheme from metrics rules: %v", err) } @@ -184,14 +184,14 @@ func (cmd *PrometheusAdapter) makeExternalProvider(promClient prom.Client, stopC return nil, fmt.Errorf("unable to construct RESTMapper: %v", err) } - // collect series converters for adapter - converters, errs := extprov.ConvertersFromConfig(cmd.metricsConfig, mapper) - if len(errs) > 0 { - return nil, fmt.Errorf("unable to construct naming scheme from metrics rules: %v", errs) + // extract the namers + namers, err := naming.NamersFromConfig(cmd.metricsConfig.ExternalRules, mapper) + if err != nil { + return nil, fmt.Errorf("unable to construct naming scheme from metrics rules: %v", err) } // construct the provider and start it - emProvider, runner := extprov.NewExternalPrometheusProvider(promClient, converters, cmd.MetricsRelistInterval) + emProvider, runner := extprov.NewExternalPrometheusProvider(promClient, namers, cmd.MetricsRelistInterval) runner.RunUntil(stopCh) return emProvider, nil diff --git a/pkg/external-provider/basic_metric_lister.go b/pkg/external-provider/basic_metric_lister.go index 789baa01..d0e56c4a 100644 --- a/pkg/external-provider/basic_metric_lister.go +++ b/pkg/external-provider/basic_metric_lister.go @@ -25,6 +25,7 @@ import ( pmodel "github.com/prometheus/common/model" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + "github.com/directxman12/k8s-prometheus-adapter/pkg/naming" ) // Runnable represents something that can be run until told to stop. @@ -56,15 +57,15 @@ type MetricListerWithNotification interface { type basicMetricLister struct { promClient prom.Client - converters []SeriesConverter + namers []naming.MetricNamer lookback time.Duration } // NewBasicMetricLister creates a MetricLister that is capable of interactly directly with Prometheus to list metrics. -func NewBasicMetricLister(promClient prom.Client, converters []SeriesConverter, lookback time.Duration) MetricLister { +func NewBasicMetricLister(promClient prom.Client, namers []naming.MetricNamer, lookback time.Duration) MetricLister { lister := basicMetricLister{ promClient: promClient, - converters: converters, + namers: namers, lookback: lookback, } @@ -78,8 +79,8 @@ type selectorSeries struct { func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) { result := MetricUpdateResult{ - series: make([][]prom.Series, 0), - converters: make([]SeriesConverter, 0), + series: make([][]prom.Series, 0), + namers: make([]naming.MetricNamer, 0), } startTime := pmodel.Now().Add(-1 * l.lookback) @@ -87,9 +88,9 @@ func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) { // these can take a while on large clusters, so launch in parallel // and don't duplicate selectors := make(map[prom.Selector]struct{}) - selectorSeriesChan := make(chan selectorSeries, len(l.converters)) - errs := make(chan error, len(l.converters)) - for _, converter := range l.converters { + selectorSeriesChan := make(chan selectorSeries, len(l.namers)) + errs := make(chan error, len(l.namers)) + for _, converter := range l.namers { sel := converter.Selector() if _, ok := selectors[sel]; ok { errs <- nil @@ -118,7 +119,7 @@ func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) { // iterate through, blocking until we've got all results // We know that, from above, we should have pushed one item into the channel // for each converter. So here, we'll assume that we should receive one item per converter. - for range l.converters { + for range l.namers { if err := <-errs; err != nil { return result, fmt.Errorf("unable to update list of all metrics: %v", err) } @@ -133,21 +134,21 @@ func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) { // Now that we've collected all of the results into `seriesCacheByQuery` // we can start processing them. - newSeries := make([][]prom.Series, len(l.converters)) - for i, converter := range l.converters { - series, cached := seriesCacheByQuery[converter.Selector()] + newSeries := make([][]prom.Series, len(l.namers)) + for i, namer := range l.namers { + series, cached := seriesCacheByQuery[namer.Selector()] if !cached { - return result, fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", converter.Selector()) + return result, fmt.Errorf("unable to update list of all metrics: no metrics retrieved for query %q", namer.Selector()) } // Because converters provide a "post-filtering" option, it's not enough to // simply take all the series that were produced. We need to further filter them. - newSeries[i] = converter.SeriesFilterer().FilterSeries(series) + newSeries[i] = namer.FilterSeries(series) } glog.V(10).Infof("Set available metric list from Prometheus to: %v", newSeries) result.series = newSeries - result.converters = l.converters + result.namers = l.namers return result, nil } @@ -156,8 +157,8 @@ func (l *basicMetricLister) ListAllMetrics() (MetricUpdateResult, error) { // It includes both the series data the Prometheus exposed, as well as the configurational // object that led to their discovery. type MetricUpdateResult struct { - series [][]prom.Series - converters []SeriesConverter + series [][]prom.Series + namers []naming.MetricNamer } // MetricUpdateCallback is a function signature for receiving periodic updates about diff --git a/pkg/external-provider/external_series_registry.go b/pkg/external-provider/external_series_registry.go index 48bc7993..77549080 100644 --- a/pkg/external-provider/external_series_registry.go +++ b/pkg/external-provider/external_series_registry.go @@ -21,6 +21,7 @@ import ( "k8s.io/apimachinery/pkg/labels" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" + "github.com/directxman12/k8s-prometheus-adapter/pkg/naming" ) // ExternalSeriesRegistry acts as the top-level converter for transforming Kubernetes requests @@ -33,20 +34,28 @@ type ExternalSeriesRegistry interface { // overridableSeriesRegistry is a basic SeriesRegistry type externalSeriesRegistry struct { - // We lock when reading/writing metrics, and rawMetrics to prevent inconsistencies. + // We lock when reading/writing metrics, and metricsInfo to prevent inconsistencies. mu sync.RWMutex // metrics is the list of all known metrics, ready to return from the API metrics []provider.ExternalMetricInfo - // rawMetrics is a lookup from a metric to SeriesConverter for the sake of generating queries - rawMetrics map[string]SeriesConverter + // metricsInfo is a lookup from a metric to SeriesConverter for the sake of generating queries + metricsInfo map[string]seriesInfo +} + +type seriesInfo struct { + // seriesName is the name of the corresponding Prometheus series + seriesName string + + // namer is the MetricNamer used to name this series + namer naming.MetricNamer } // NewExternalSeriesRegistry creates an ExternalSeriesRegistry driven by the data from the provided MetricLister. func NewExternalSeriesRegistry(lister MetricListerWithNotification) ExternalSeriesRegistry { var registry = externalSeriesRegistry{ - metrics: make([]provider.ExternalMetricInfo, 0), - rawMetrics: map[string]SeriesConverter{}, + metrics: make([]provider.ExternalMetricInfo, 0), + metricsInfo: map[string]seriesInfo{}, } lister.AddNotificationReceiver(registry.filterAndStoreMetrics) @@ -56,26 +65,29 @@ func NewExternalSeriesRegistry(lister MetricListerWithNotification) ExternalSeri func (r *externalSeriesRegistry) filterAndStoreMetrics(result MetricUpdateResult) { newSeriesSlices := result.series - converters := result.converters + namers := result.namers - if len(newSeriesSlices) != len(converters) { + if len(newSeriesSlices) != len(namers) { glog.Fatal("need one set of series per converter") } apiMetricsCache := make([]provider.ExternalMetricInfo, 0) - rawMetricsCache := make(map[string]SeriesConverter) + rawMetricsCache := make(map[string]seriesInfo) for i, newSeries := range newSeriesSlices { - converter := converters[i] + namer := namers[i] for _, series := range newSeries { - identity, err := converter.IdentifySeries(series) + identity, err := namer.MetricNameForSeries(series) if err != nil { glog.Errorf("unable to name series %q, skipping: %v", series.String(), err) continue } - name := identity.name - rawMetricsCache[name] = converter + name := identity + rawMetricsCache[name] = seriesInfo{ + seriesName: series.Name, + namer: namer, + } } } @@ -89,7 +101,7 @@ func (r *externalSeriesRegistry) filterAndStoreMetrics(result MetricUpdateResult defer r.mu.Unlock() r.metrics = apiMetricsCache - r.rawMetrics = rawMetricsCache + r.metricsInfo = rawMetricsCache } @@ -104,13 +116,13 @@ func (r *externalSeriesRegistry) QueryForMetric(namespace string, metricName str r.mu.RLock() defer r.mu.RUnlock() - converter, found := r.rawMetrics[metricName] + info, found := r.metricsInfo[metricName] if !found { glog.V(10).Infof("external metric %q not found", metricName) return "", false, nil } + query, err := info.namer.QueryForExternalSeries(info.seriesName, namespace, metricSelector) - query, err := converter.QueryForExternalSeries(namespace, metricName, metricSelector) return query, found, err } diff --git a/pkg/external-provider/provider.go b/pkg/external-provider/provider.go index 7cc954a7..3b6622dc 100644 --- a/pkg/external-provider/provider.go +++ b/pkg/external-provider/provider.go @@ -51,7 +51,7 @@ func (p *externalPrometheusProvider) GetExternalMetric(namespace string, metricS if !found { return nil, provider.NewMetricNotFoundError(p.selectGroupResource(namespace), info.Metric) } - + // Here is where we're making the query, need to be before here xD queryResults, err := p.promClient.Query(context.TODO(), pmodel.Now(), selector) if err != nil { @@ -78,9 +78,9 @@ func (p *externalPrometheusProvider) selectGroupResource(namespace string) schem } // NewExternalPrometheusProvider creates an ExternalMetricsProvider capable of responding to Kubernetes requests for external metric data -func NewExternalPrometheusProvider(promClient prom.Client, converters []SeriesConverter, updateInterval time.Duration) (provider.ExternalMetricsProvider, Runnable) { +func NewExternalPrometheusProvider(promClient prom.Client, namers []naming.MetricNamer, updateInterval time.Duration) (provider.ExternalMetricsProvider, Runnable) { metricConverter := NewMetricConverter() - basicLister := NewBasicMetricLister(promClient, converters, updateInterval) + basicLister := NewBasicMetricLister(promClient, namers, updateInterval) periodicLister, _ := NewPeriodicMetricLister(basicLister, updateInterval) seriesRegistry := NewExternalSeriesRegistry(periodicLister) return &externalPrometheusProvider{ diff --git a/pkg/external-provider/query_builder.go b/pkg/external-provider/query_builder.go deleted file mode 100644 index c4e9aa18..00000000 --- a/pkg/external-provider/query_builder.go +++ /dev/null @@ -1,219 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "bytes" - "errors" - "fmt" - "strings" - "text/template" - - "k8s.io/apimachinery/pkg/selection" - - prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" -) - -// QueryBuilder provides functions for generating Prometheus queries. -type QueryBuilder interface { - BuildSelector(seriesName string, groupBy string, groupBySlice []string, queryParts []queryPart) (prom.Selector, error) -} - -type queryBuilder struct { - metricsQueryTemplate *template.Template -} - -// NewQueryBuilder creates a QueryBuilder. -func NewQueryBuilder(metricsQuery string) (QueryBuilder, error) { - metricsQueryTemplate, err := template.New("metrics-query").Delims("<<", ">>").Parse(metricsQuery) - if err != nil { - return nil, fmt.Errorf("unable to parse metrics query template %q: %v", metricsQuery, err) - } - - return &queryBuilder{ - metricsQueryTemplate: metricsQueryTemplate, - }, nil -} - -func (n *queryBuilder) BuildSelector(seriesName string, groupBy string, groupBySlice []string, queryParts []queryPart) (prom.Selector, error) { - // Convert our query parts into the types we need for our template. - exprs, valuesByName, err := n.processQueryParts(queryParts) - - if err != nil { - return "", err - } - - args := queryTemplateArgs{ - Series: seriesName, - LabelMatchers: strings.Join(exprs, ","), - LabelValuesByName: valuesByName, - GroupBy: groupBy, - GroupBySlice: groupBySlice, - } - - selector, err := n.createSelectorFromTemplateArgs(args) - if err != nil { - return "", err - } - - return selector, nil -} - -func (n *queryBuilder) createSelectorFromTemplateArgs(args queryTemplateArgs) (prom.Selector, error) { - //Turn our template arguments into a Selector. - queryBuff := new(bytes.Buffer) - if err := n.metricsQueryTemplate.Execute(queryBuff, args); err != nil { - return "", err - } - - if queryBuff.Len() == 0 { - return "", fmt.Errorf("empty query produced by metrics query template") - } - - return prom.Selector(queryBuff.String()), nil -} - -func (n *queryBuilder) processQueryParts(queryParts []queryPart) ([]string, map[string][]string, error) { - // We've take the approach here that if we can't perfectly map their query into a Prometheus - // query that we should abandon the effort completely. - // The concern is that if we don't get a perfect match on their query parameters, the query result - // might contain unexpected data that would cause them to take an erroneous action based on the result. - - // Contains the expressions that we want to include as part of the query to Prometheus. - // e.g. "namespace=my-namespace" - // e.g. "some_label=some-value" - var exprs []string - - // Contains the list of label values we're targeting, by namespace. - // e.g. "some_label" => ["value-one", "value-two"] - valuesByName := map[string][]string{} - - // Convert our query parts into template arguments. - for _, qPart := range queryParts { - // Be resilient against bad inputs. - // We obviously can't generate label filters for these cases. - fmt.Println("This is queryPart", qPart.labelName, qPart.operator, qPart.values) - if qPart.labelName == "" { - return nil, nil, ErrLabelNotSpecified - } - - if !n.operatorIsSupported(qPart.operator) { - return nil, nil, ErrUnsupportedOperator - } - - matcher, err := n.selectMatcher(qPart.operator, qPart.values) - - if err != nil { - return nil, nil, err - } - - targetValue, err := n.selectTargetValue(qPart.operator, qPart.values) - if err != nil { - return nil, nil, err - } - - expression := matcher(qPart.labelName, targetValue) - exprs = append(exprs, expression) - valuesByName[qPart.labelName] = qPart.values - } - - return exprs, valuesByName, nil -} - -func (n *queryBuilder) selectMatcher(operator selection.Operator, values []string) (func(string, string) string, error) { - - numValues := len(values) - if numValues == 0 { - switch operator { - case selection.Exists: - return prom.LabelNeq, nil - case selection.DoesNotExist: - return prom.LabelEq, nil - case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: - return nil, ErrMalformedQuery - } - } else if numValues == 1 { - switch operator { - case selection.Equals, selection.DoubleEquals: - return prom.LabelEq, nil - case selection.NotEquals: - return prom.LabelNeq, nil - case selection.In, selection.Exists: - return prom.LabelMatches, nil - case selection.DoesNotExist, selection.NotIn: - return prom.LabelNotMatches, nil - } - } else { - // Since labels can only have one value, providing multiple - // values results in a regex match, even if that's not what the user - // asked for. - switch operator { - case selection.Equals, selection.DoubleEquals, selection.In, selection.Exists: - return prom.LabelMatches, nil - case selection.NotEquals, selection.DoesNotExist, selection.NotIn: - return prom.LabelNotMatches, nil - } - } - - return nil, errors.New("operator not supported by query builder") -} - -func (n *queryBuilder) selectTargetValue(operator selection.Operator, values []string) (string, error) { - numValues := len(values) - if numValues == 0 { - switch operator { - case selection.Exists, selection.DoesNotExist: - // Return an empty string when values are equal to 0 - // When the operator is LabelNotMatches this will select series without the label - // or with the label but a value of "". - // When the operator is LabelMatches this will select series with the label - // whose value is NOT "". - return "", nil - case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: - return "", ErrMalformedQuery - } - } else if numValues == 1 { - switch operator { - case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: - // Pass the value through as-is. - // It's somewhat strange to do this for both the regex and equality - // operators, but if we do it this way it gives the user a little more control. - // They might choose to send an "IN" request and give a list of static values - // or they could send a single value that's a regex, giving them a passthrough - // for their label selector. - return values[0], nil - case selection.Exists, selection.DoesNotExist: - return "", errors.New("operator does not support values") - } - } else { - switch operator { - case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: - // Pass the value through as-is. - // It's somewhat strange to do this for both the regex and equality - // operators, but if we do it this way it gives the user a little more control. - // They might choose to send an "IN" request and give a list of static values - // or they could send a single value that's a regex, giving them a passthrough - // for their label selector. - return strings.Join(values, "|"), nil - case selection.Exists, selection.DoesNotExist: - return "", ErrQueryUnsupportedValues - } - } - - return "", errors.New("operator not supported by query builder") -} - -func (n *queryBuilder) operatorIsSupported(operator selection.Operator) bool { - return operator != selection.GreaterThan && operator != selection.LessThan -} diff --git a/pkg/external-provider/query_builder_test.go b/pkg/external-provider/query_builder_test.go deleted file mode 100644 index bb3a9de0..00000000 --- a/pkg/external-provider/query_builder_test.go +++ /dev/null @@ -1,356 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "testing" - - "k8s.io/apimachinery/pkg/selection" - - "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/stretchr/testify/require" -) - -func TestBadQueryPartsDontBuildQueries(t *testing.T) { - builder, _ := NewQueryBuilder("rate(<<.Series>>{<<.LabelMatchers>>}[2m])") - _, err := builder.BuildSelector("my_series", "", []string{}, []queryPart{ - { - labelName: "", - values: nil, - }, - { - labelName: "", - values: []string{}, - }, - }) - - require.Error(t, err) -} - -func runQueryBuilderTest(t *testing.T, queryParts []queryPart, expectation string) { - builder, _ := NewQueryBuilder("rate(<<.Series>>{<<.LabelMatchers>>}[2m])") - selector, err := builder.BuildSelector("my_series", "", []string{}, queryParts) - - expectError := expectation == "" - - if expectError { - require.Error(t, err) - } else { - selectorExpectation := client.Selector(expectation) - require.NoError(t, err) - require.Equal(t, selector, selectorExpectation) - } -} - -/* -func TestSimpleQuery(t *testing.T) { - runQueryBuilderTest(t, []queryPart{}, "") -} -*/ - -//Equals -func TestEqualsQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.Equals, - }, - }, "") -} - -func TestEqualsQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.Equals, - }, - }, "rate(my_series{target_label=\"one\"}[2m])") -} - -func TestEqualsQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.Equals, - }, - }, "rate(my_series{target_label=~\"one|two\"}[2m])") -} - -//Double Equals -func TestDoubleEqualsQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.DoubleEquals, - }, - }, "") -} - -func TestDoubleEqualsQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.DoubleEquals, - }, - }, "rate(my_series{target_label=\"one\"}[2m])") -} - -func TestDoubleEqualsQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.DoubleEquals, - }, - }, "rate(my_series{target_label=~\"one|two\"}[2m])") -} - -//Not Equals -func TestNotEqualsQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.NotEquals, - }, - }, "") -} - -func TestNotEqualsQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.NotEquals, - }, - }, "rate(my_series{target_label!=\"one\"}[2m])") -} - -func TestNotEqualsQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.NotEquals, - }, - }, "rate(my_series{target_label!~\"one|two\"}[2m])") -} - -//In -func TestInQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.In, - }, - }, "") -} - -func TestInQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.In, - }, - }, "rate(my_series{target_label=~\"one\"}[2m])") -} - -func TestInQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.In, - }, - }, "rate(my_series{target_label=~\"one|two\"}[2m])") -} - -//NotIn -func TestNotInQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.NotIn, - }, - }, "") -} - -func TestNotInQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.NotIn, - }, - }, "rate(my_series{target_label!~\"one\"}[2m])") -} - -func TestNotInQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.NotIn, - }, - }, "rate(my_series{target_label!~\"one|two\"}[2m])") -} - -//Exists -func TestExistsQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.Exists, - }, - }, "rate(my_series{target_label!=\"\"}[2m])") -} - -func TestExistsQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.Exists, - }, - }, "") -} - -func TestExistsQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.Exists, - }, - }, "") -} - -//DoesNotExist -func TestDoesNotExistsQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.DoesNotExist, - }, - }, "rate(my_series{target_label=\"\"}[2m])") -} - -func TestDoesNotExistsQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.DoesNotExist, - }, - }, "") -} - -func TestDoesNotExistsQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.DoesNotExist, - }, - }, "") -} - -//GreaterThan -func TestGreaterThanQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.GreaterThan, - }, - }, "") -} - -func TestGreaterThanQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.GreaterThan, - }, - }, "") -} - -func TestGreaterThanQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.GreaterThan, - }, - }, "") -} - -//LessThan -func TestLessThanQueryWithNoLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{}, - operator: selection.LessThan, - }, - }, "") -} - -func TestLessThanQueryWithOneLabelValue(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one"}, - operator: selection.LessThan, - }, - }, "") -} - -func TestLessThanQueryWithMultipleLabelValues(t *testing.T) { - runQueryBuilderTest(t, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.LessThan, - }, - }, "") -} - -func TestQueryWithGroupBy(t *testing.T) { - builder, _ := NewQueryBuilder("sum(rate(<<.Series>>{<<.LabelMatchers>>}[2m])) by (<<.GroupBy>>)") - - selector, _ := builder.BuildSelector("my_series", "my_grouping", []string{}, []queryPart{ - { - labelName: "target_label", - values: []string{"one", "two"}, - operator: selection.In, - }, - }) - - expectation := client.Selector("sum(rate(my_series{target_label=~\"one|two\"}[2m])) by (my_grouping)") - require.Equal(t, selector, expectation) -} - -// TODO: Ensure that the LabelValuesByName and GroupBySlice placeholders function correctly. diff --git a/pkg/external-provider/series_converter.go b/pkg/external-provider/series_converter.go deleted file mode 100644 index ff4a606d..00000000 --- a/pkg/external-provider/series_converter.go +++ /dev/null @@ -1,286 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "fmt" - - apimeta "k8s.io/apimachinery/pkg/api/meta" - "k8s.io/apimachinery/pkg/labels" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/apimachinery/pkg/selection" - - prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/directxman12/k8s-prometheus-adapter/pkg/config" - "github.com/directxman12/k8s-prometheus-adapter/pkg/naming" -) - -// SeriesConverter knows how to convert Prometheus series names and label names to -// metrics API resources, and vice-versa. SeriesConverters should be safe to access -// concurrently. Returned group-resources are "normalized" as per the -// MetricInfo#Normalized method. Group-resources passed as arguments must -// themselves be normalized. -type SeriesConverter interface { - // Selector produces the appropriate Prometheus series selector to match all - // series handlable by this converter. - Selector() prom.Selector - // FilterSeries checks to see which of the given series match any additional - // constrains beyond the series query. It's assumed that the series given - // already matche the series query. - // FilterSeries(series []prom.Series) []prom.Series - SeriesFilterer() SeriesFilterer - ResourceConverter() naming.ResourceConverter - - QueryForExternalSeries(namespace string, series string, metricSelector labels.Selector) (prom.Selector, error) - IdentifySeries(series prom.Series) (seriesIdentity, error) -} - -type seriesIdentity struct { - resources []schema.GroupResource - namespaced bool - name string -} - -func (c *seriesConverter) Selector() prom.Selector { - return c.seriesQuery -} - -type seriesConverter struct { - seriesQuery prom.Selector - - resourceConverter naming.ResourceConverter - queryBuilder QueryBuilder - seriesFilterer SeriesFilterer - metricNamer naming.MetricNamer -} - -// queryTemplateArgs are the arguments for the metrics query template. -type queryTemplateArgs struct { - Series string - LabelMatchers string - LabelValuesByName map[string][]string - GroupBy string - GroupBySlice []string -} - -func (c *seriesConverter) IdentifySeries(series prom.Series) (seriesIdentity, error) { - // TODO: warn if it doesn't match any resources - - resources, _ := c.resourceConverter.ResourcesForSeries(series) - name, err := c.metricNamer.MetricNameForSeries(series) - - result := seriesIdentity{ - resources: resources, - namespaced: false, - name: name, - } - - return result, err -} - -func (c *seriesConverter) SeriesFilterer() SeriesFilterer { - return c.seriesFilterer -} - -func (c *seriesConverter) ResourceConverter() naming.ResourceConverter { - return c.resourceConverter -} - -func (c *seriesConverter) createQueryPartsFromSelector(metricSelector labels.Selector) []queryPart { - requirements, _ := metricSelector.Requirements() - - selectors := []queryPart{} - for i := 0; i < len(requirements); i++ { - selector := c.convertRequirement(requirements[i]) - - selectors = append(selectors, selector) - } - - return selectors -} - -func (c *seriesConverter) convertRequirement(requirement labels.Requirement) queryPart { - labelName := requirement.Key() - values := requirement.Values().List() - - return queryPart{ - labelName: labelName, - values: values, - operator: requirement.Operator(), - } -} - -type queryPart struct { - labelName string - values []string - operator selection.Operator -} - -func (c *seriesConverter) buildNamespaceQueryPartForSeries(namespace string) (queryPart, error) { - result := queryPart{} - - // If we've been given a namespace, then we need to set up - // the label requirements to target that namespace. - if namespace != "default" { - namespaceLbl, err := c.resourceConverter.LabelForResource(naming.NsGroupResource) - if err != nil { - return result, err - } - - values := []string{namespace} - - result = queryPart{ - values: values, - labelName: string(namespaceLbl), - operator: selection.Equals, - } - } - - return result, nil -} - -func (c *seriesConverter) buildResourceQueryPartForSeries(resource schema.GroupResource, names ...string) (queryPart, error) { - result := queryPart{} - - // If we've been given a resource, then we need to set up - // the label requirements to target that resource. - resourceLbl, err := c.resourceConverter.LabelForResource(resource) - if err != nil { - return result, err - } - - result = queryPart{ - labelName: string(resourceLbl), - values: names, - operator: selection.Equals, - } - - return result, nil -} - -func (c *seriesConverter) QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) { - queryParts := []queryPart{} - - // Build up the namespace part of the query. - namespaceQueryPart, err := c.buildNamespaceQueryPartForSeries(namespace) - if err != nil { - return "", err - } - - if namespaceQueryPart.labelName != "" { - queryParts = append(queryParts, namespaceQueryPart) - } - - // Build up the resource part of the query. - resourceQueryPart, err := c.buildResourceQueryPartForSeries(resource, names...) - if err != nil { - return "", err - } - - if resourceQueryPart.labelName != "" { - queryParts = append(queryParts, resourceQueryPart) - } - - return c.queryBuilder.BuildSelector(series, resourceQueryPart.labelName, []string{resourceQueryPart.labelName}, queryParts) -} - -// ConvertersFromConfig produces a MetricNamer for each rule in the given config. -func ConvertersFromConfig(cfg *config.MetricsDiscoveryConfig, mapper apimeta.RESTMapper) ([]SeriesConverter, []error) { - errs := []error{} - converters := []SeriesConverter{} - - for _, rule := range cfg.ExternalRules { - if externalConverter, err := converterFromRule(rule, mapper); err == nil { - converters = append(converters, externalConverter) - } else { - errs = append(errs, err) - } - } - return converters, errs -} - -func converterFromRule(rule config.DiscoveryRule, mapper apimeta.RESTMapper) (SeriesConverter, error) { - var ( - err error - ) - - resourceConverter, err := naming.NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper) - if err != nil { - return nil, fmt.Errorf("unable to create ResourceConverter associated with series query %q: %v", rule.SeriesQuery, err) - } - - queryBuilder, err := NewQueryBuilder(rule.MetricsQuery) - if err != nil { - return nil, fmt.Errorf("unable to create a QueryBuilder associated with series query %q: %v", rule.SeriesQuery, err) - } - - seriesFilterer, err := NewSeriesFilterer(rule.SeriesFilters) - if err != nil { - return nil, fmt.Errorf("unable to create a SeriesFilter associated with series query %q: %v", rule.SeriesQuery, err) - } - - if rule.Name.Matches != "" { - err := seriesFilterer.AddRequirement(config.RegexFilter{Is: rule.Name.Matches}) - if err != nil { - return nil, fmt.Errorf("unable to apply the series name filter from name rules associated with series query %q: %v", rule.SeriesQuery, err) - } - } - - metricNamer, err := naming.NewMetricNamer(rule.Name) - if err != nil { - return nil, fmt.Errorf("unable to create a MetricNamer associated with series query %q: %v", rule.SeriesQuery, err) - } - - return &seriesConverter{ - seriesQuery: prom.Selector(rule.SeriesQuery), - resourceConverter: resourceConverter, - queryBuilder: queryBuilder, - seriesFilterer: seriesFilterer, - metricNamer: metricNamer, - }, nil -} - -func (c *seriesConverter) buildNamespaceQueryPartForExternalSeries(namespace string) (queryPart, error) { - namespaceLbl, err := c.resourceConverter.LabelForResource(naming.NsGroupResource) - - return queryPart{ - labelName: string(namespaceLbl), - values: []string{namespace}, - operator: selection.Equals, - }, err -} - -func (c *seriesConverter) QueryForExternalSeries(namespace string, series string, metricSelector labels.Selector) (prom.Selector, error) { - queryParts := []queryPart{} - - if namespace != "default" { - // Build up the namespace part of the query. - namespaceQueryPart, err := c.buildNamespaceQueryPartForExternalSeries(namespace) - if err != nil { - return "", err - } - - queryParts = append(queryParts, namespaceQueryPart) - } - - // Build up the query parts from the selector. - queryParts = append(queryParts, c.createQueryPartsFromSelector(metricSelector)...) - - selector, err := c.queryBuilder.BuildSelector(series, "", []string{}, queryParts) - if err != nil { - return "", err - } - - return selector, nil -} diff --git a/pkg/external-provider/series_filterer.go b/pkg/external-provider/series_filterer.go deleted file mode 100644 index 90d2d24a..00000000 --- a/pkg/external-provider/series_filterer.go +++ /dev/null @@ -1,79 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "fmt" - - prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/directxman12/k8s-prometheus-adapter/pkg/config" - "github.com/directxman12/k8s-prometheus-adapter/pkg/naming" -) - -// SeriesFilterer provides functions for filtering collections of Prometheus series -// to only those that meet certain requirements. -type SeriesFilterer interface { - FilterSeries(series []prom.Series) []prom.Series - AddRequirement(filter config.RegexFilter) error -} - -type seriesFilterer struct { - seriesMatchers []*naming.ReMatcher -} - -// NewSeriesFilterer creates a SeriesFilterer that will remove any series that do not -// meet the requirements of the provided RegexFilter(s). -func NewSeriesFilterer(filters []config.RegexFilter) (SeriesFilterer, error) { - seriesMatchers := make([]*naming.ReMatcher, len(filters)) - for i, filterRaw := range filters { - matcher, err := naming.NewReMatcher(filterRaw) - if err != nil { - return nil, fmt.Errorf("unable to generate series name filter: %v", err) - } - seriesMatchers[i] = matcher - } - - return &seriesFilterer{ - seriesMatchers: seriesMatchers, - }, nil -} - -func (n *seriesFilterer) AddRequirement(filterRaw config.RegexFilter) error { - matcher, err := naming.NewReMatcher(filterRaw) - if err != nil { - return fmt.Errorf("unable to generate series name filter: %v", err) - } - - n.seriesMatchers = append(n.seriesMatchers, matcher) - return nil -} - -func (n *seriesFilterer) FilterSeries(initialSeries []prom.Series) []prom.Series { - if len(n.seriesMatchers) == 0 { - return initialSeries - } - - finalSeries := make([]prom.Series, 0, len(initialSeries)) -SeriesLoop: - for _, series := range initialSeries { - for _, matcher := range n.seriesMatchers { - if !matcher.Matches(series.Name) { - continue SeriesLoop - } - } - finalSeries = append(finalSeries, series) - } - - return finalSeries -} diff --git a/pkg/external-provider/series_filterer_test.go b/pkg/external-provider/series_filterer_test.go deleted file mode 100644 index 7171ad76..00000000 --- a/pkg/external-provider/series_filterer_test.go +++ /dev/null @@ -1,127 +0,0 @@ -/* -Copyright 2017 The Kubernetes Authors. -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - http://www.apache.org/licenses/LICENSE-2.0 -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package provider - -import ( - "testing" - - "github.com/stretchr/testify/require" - - prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" - "github.com/directxman12/k8s-prometheus-adapter/pkg/config" -) - -func TestPositiveFilter(t *testing.T) { - filters := []config.RegexFilter{ - { - Is: "one_of_(yes|positive|ok)", - }, - } - - series := []string{"one_of_yes", "one_of_no"} - expectedSeries := []string{"one_of_yes"} - - RunSeriesFiltererTest(t, filters, series, expectedSeries) -} - -func TestNegativeFilter(t *testing.T) { - filters := []config.RegexFilter{ - { - IsNot: "one_of_(yes|positive|ok)", - }, - } - - series := []string{"one_of_yes", "one_of_no"} - expectedSeries := []string{"one_of_no"} - - RunSeriesFiltererTest(t, filters, series, expectedSeries) -} - -func TestPositiveAndNegativeFilterError(t *testing.T) { - filters := []config.RegexFilter{ - { - Is: "series_\\d+", - IsNot: "series_[2-3]+", - }, - } - - _, err := NewSeriesFilterer(filters) - require.Error(t, err) -} - -func TestAddRequirementAppliesFilter(t *testing.T) { - seriesNames := []string{"series_1", "series_2", "series_3"} - series := BuildSeriesFromNames(seriesNames) - - filters := []config.RegexFilter{ - { - Is: "series_\\d+", - }, - } - - filterer, err := NewSeriesFilterer(filters) - require.NoError(t, err) - - // Test it once with the default filters. - result := filterer.FilterSeries(series) - expectedSeries := []string{"series_1", "series_2", "series_3"} - VerifyMatches(t, result, expectedSeries) - - // Add a new filter and test again. - filterer.AddRequirement(config.RegexFilter{ - Is: "series_[2-3]", - }) - result = filterer.FilterSeries(series) - expectedSeries = []string{"series_2", "series_3"} - VerifyMatches(t, result, expectedSeries) -} - -func RunSeriesFiltererTest(t *testing.T, filters []config.RegexFilter, seriesNames []string, expectedResults []string) { - series := BuildSeriesFromNames(seriesNames) - - filterer, err := NewSeriesFilterer(filters) - require.NoError(t, err) - - matches := filterer.FilterSeries(series) - - VerifyMatches(t, matches, expectedResults) -} - -func VerifyMatches(t *testing.T, series []prom.Series, expectedResults []string) { - require.Equal(t, len(series), len(expectedResults)) - - existingSeries := make(map[string]bool) - - for _, series := range series { - existingSeries[series.Name] = true - } - - for _, expectedResult := range expectedResults { - _, exists := existingSeries[expectedResult] - - require.True(t, exists) - } -} - -func BuildSeriesFromNames(seriesNames []string) []prom.Series { - series := make([]prom.Series, len(seriesNames)) - - for i, name := range seriesNames { - series[i] = prom.Series{ - Name: name, - } - } - - return series -} diff --git a/pkg/external-provider/errors.go b/pkg/naming/errors.go similarity index 98% rename from pkg/external-provider/errors.go rename to pkg/naming/errors.go index 10b38dd2..e31b813f 100644 --- a/pkg/external-provider/errors.go +++ b/pkg/naming/errors.go @@ -11,7 +11,7 @@ See the License for the specific language governing permissions and limitations under the License. */ -package provider +package naming import "errors" diff --git a/pkg/naming/metric_namer.go b/pkg/naming/metric_namer.go index 9db64d30..1d160bdc 100644 --- a/pkg/naming/metric_namer.go +++ b/pkg/naming/metric_namer.go @@ -21,6 +21,7 @@ import ( "regexp" apimeta "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" @@ -45,6 +46,9 @@ type MetricNamer interface { // QueryForSeries returns the query for a given series (not API metric name), with // the given namespace name (if relevant), resource, and resource names. QueryForSeries(series string, resource schema.GroupResource, namespace string, names ...string) (prom.Selector, error) + // QueryForExternalSeries returns the query for a given series (not API metric name), with + // the given namespace name (if relevant), resource, and resource names. + QueryForExternalSeries(series string, namespace string, targetLabels labels.Selector) (prom.Selector, error) ResourceConverter } @@ -126,6 +130,12 @@ func (n *metricNamer) QueryForSeries(series string, resource schema.GroupResourc return n.metricsQuery.Build(series, resource, namespace, nil, names...) } +func (n *metricNamer) QueryForExternalSeries(series string, namespace string, metricSelector labels.Selector) (prom.Selector, error) { + //test := prom.Selector() + //return test, nil + return n.metricsQuery.BuildExternal(series, namespace, "", []string{}, metricSelector) +} + func (n *metricNamer) MetricNameForSeries(series prom.Series) (string, error) { matches := n.nameMatches.FindStringSubmatchIndex(series.Name) if matches == nil { @@ -136,10 +146,10 @@ func (n *metricNamer) MetricNameForSeries(series prom.Series) (string, error) { } // NamersFromConfig produces a MetricNamer for each rule in the given config. -func NamersFromConfig(cfg *config.MetricsDiscoveryConfig, mapper apimeta.RESTMapper) ([]MetricNamer, error) { - namers := make([]MetricNamer, len(cfg.Rules)) +func NamersFromConfig(cfg []config.DiscoveryRule, mapper apimeta.RESTMapper) ([]MetricNamer, error) { + namers := make([]MetricNamer, len(cfg)) - for i, rule := range cfg.Rules { + for i, rule := range cfg { resConv, err := NewResourceConverter(rule.Resources.Template, rule.Resources.Overrides, mapper) if err != nil { return nil, err @@ -205,38 +215,3 @@ func NamersFromConfig(cfg *config.MetricsDiscoveryConfig, mapper apimeta.RESTMap return namers, nil } - -// NewMetricNamer creates a MetricNamer capable of translating Prometheus series names -// into custom metric names. -func NewMetricNamer(mapping config.NameMapping) (MetricNamer, error) { - var nameMatches *regexp.Regexp - var err error - if mapping.Matches != "" { - nameMatches, err = regexp.Compile(mapping.Matches) - if err != nil { - return nil, fmt.Errorf("unable to compile series name match expression %q: %v", mapping.Matches, err) - } - } else { - // this will always succeed - nameMatches = regexp.MustCompile(".*") - } - nameAs := mapping.As - if nameAs == "" { - // check if we have an obvious default - subexpNames := nameMatches.SubexpNames() - if len(subexpNames) == 1 { - // no capture groups, use the whole thing - nameAs = "$0" - } else if len(subexpNames) == 2 { - // one capture group, use that - nameAs = "$1" - } else { - return nil, fmt.Errorf("must specify an 'as' value for name matcher %q", mapping.Matches) - } - } - - return &metricNamer{ - nameMatches: nameMatches, - nameAs: nameAs, - }, nil -} diff --git a/pkg/naming/metrics_query.go b/pkg/naming/metrics_query.go index 31a1dc62..2fa8856f 100644 --- a/pkg/naming/metrics_query.go +++ b/pkg/naming/metrics_query.go @@ -18,11 +18,14 @@ package naming import ( "bytes" + "errors" "fmt" "strings" "text/template" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/selection" prom "github.com/directxman12/k8s-prometheus-adapter/pkg/client" ) @@ -37,6 +40,7 @@ type MetricsQuery interface { // where we need to scope down more specifically than just the group-resource // (e.g. container metrics). Build(series string, groupRes schema.GroupResource, namespace string, extraGroupBy []string, resourceNames ...string) (prom.Selector, error) + BuildExternal(seriesName string, namespace string, groupBy string, groupBySlice []string, metricSelector labels.Selector) (prom.Selector, error) } // NewMetricsQuery constructs a new MetricsQuery by compiling the given Go template. @@ -75,6 +79,12 @@ type queryTemplateArgs struct { GroupBySlice []string } +type queryPart struct { + labelName string + values []string + operator selection.Operator +} + func (q *metricsQuery) Build(series string, resource schema.GroupResource, namespace string, extraGroupBy []string, names ...string) (prom.Selector, error) { var exprs []string valuesByName := map[string][]string{} @@ -123,3 +133,194 @@ func (q *metricsQuery) Build(series string, resource schema.GroupResource, names return prom.Selector(queryBuff.String()), nil } + +func (q *metricsQuery) BuildExternal(seriesName string, namespace string, groupBy string, groupBySlice []string, metricSelector labels.Selector) (prom.Selector, error) { + queryParts := []queryPart{} + + // Build up the query parts from the selector. + queryParts = append(queryParts, q.createQueryPartsFromSelector(metricSelector)...) + + // Convert our query parts into the types we need for our template. + exprs, valuesByName, err := q.processQueryParts(queryParts) + + if err != nil { + return "", err + } + + args := queryTemplateArgs{ + Series: seriesName, + LabelMatchers: strings.Join(exprs, ","), + LabelValuesByName: valuesByName, + GroupBy: groupBy, + GroupBySlice: groupBySlice, + } + + queryBuff := new(bytes.Buffer) + if err := q.template.Execute(queryBuff, args); err != nil { + return "", err + } + + if queryBuff.Len() == 0 { + return "", fmt.Errorf("empty query produced by metrics query template") + } + + return prom.Selector(queryBuff.String()), nil +} + +func (q *metricsQuery) createQueryPartsFromSelector(metricSelector labels.Selector) []queryPart { + requirements, _ := metricSelector.Requirements() + + selectors := []queryPart{} + for i := 0; i < len(requirements); i++ { + selector := q.convertRequirement(requirements[i]) + + selectors = append(selectors, selector) + } + + return selectors +} + +func (q *metricsQuery) convertRequirement(requirement labels.Requirement) queryPart { + labelName := requirement.Key() + values := requirement.Values().List() + + return queryPart{ + labelName: labelName, + values: values, + operator: requirement.Operator(), + } +} + +func (q *metricsQuery) processQueryParts(queryParts []queryPart) ([]string, map[string][]string, error) { + // We've take the approach here that if we can't perfectly map their query into a Prometheus + // query that we should abandon the effort completely. + // The concern is that if we don't get a perfect match on their query parameters, the query result + // might contain unexpected data that would cause them to take an erroneous action based on the result. + + // Contains the expressions that we want to include as part of the query to Prometheus. + // e.g. "namespace=my-namespace" + // e.g. "some_label=some-value" + var exprs []string + + // Contains the list of label values we're targeting, by namespace. + // e.g. "some_label" => ["value-one", "value-two"] + valuesByName := map[string][]string{} + + // Convert our query parts into template arguments. + for _, qPart := range queryParts { + // Be resilient against bad inputs. + // We obviously can't generate label filters for these cases. + fmt.Println("This is queryPart", qPart.labelName, qPart.operator, qPart.values) + if qPart.labelName == "" { + return nil, nil, ErrLabelNotSpecified + } + + if !q.operatorIsSupported(qPart.operator) { + return nil, nil, ErrUnsupportedOperator + } + + matcher, err := q.selectMatcher(qPart.operator, qPart.values) + + if err != nil { + return nil, nil, err + } + + targetValue, err := q.selectTargetValue(qPart.operator, qPart.values) + if err != nil { + return nil, nil, err + } + + expression := matcher(qPart.labelName, targetValue) + exprs = append(exprs, expression) + valuesByName[qPart.labelName] = qPart.values + } + + return exprs, valuesByName, nil +} + +func (q *metricsQuery) selectMatcher(operator selection.Operator, values []string) (func(string, string) string, error) { + + numValues := len(values) + if numValues == 0 { + switch operator { + case selection.Exists: + return prom.LabelNeq, nil + case selection.DoesNotExist: + return prom.LabelEq, nil + case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: + return nil, ErrMalformedQuery + } + } else if numValues == 1 { + switch operator { + case selection.Equals, selection.DoubleEquals: + return prom.LabelEq, nil + case selection.NotEquals: + return prom.LabelNeq, nil + case selection.In, selection.Exists: + return prom.LabelMatches, nil + case selection.DoesNotExist, selection.NotIn: + return prom.LabelNotMatches, nil + } + } else { + // Since labels can only have one value, providing multiple + // values results in a regex match, even if that's not what the user + // asked for. + switch operator { + case selection.Equals, selection.DoubleEquals, selection.In, selection.Exists: + return prom.LabelMatches, nil + case selection.NotEquals, selection.DoesNotExist, selection.NotIn: + return prom.LabelNotMatches, nil + } + } + + return nil, errors.New("operator not supported by query builder") +} + +func (q *metricsQuery) selectTargetValue(operator selection.Operator, values []string) (string, error) { + numValues := len(values) + if numValues == 0 { + switch operator { + case selection.Exists, selection.DoesNotExist: + // Return an empty string when values are equal to 0 + // When the operator is LabelNotMatches this will select series without the label + // or with the label but a value of "". + // When the operator is LabelMatches this will select series with the label + // whose value is NOT "". + return "", nil + case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: + return "", ErrMalformedQuery + } + } else if numValues == 1 { + switch operator { + case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: + // Pass the value through as-is. + // It's somewhat strange to do this for both the regex and equality + // operators, but if we do it this way it gives the user a little more control. + // They might choose to send an "IN" request and give a list of static values + // or they could send a single value that's a regex, giving them a passthrough + // for their label selector. + return values[0], nil + case selection.Exists, selection.DoesNotExist: + return "", ErrQueryUnsupportedValues + } + } else { + switch operator { + case selection.Equals, selection.DoubleEquals, selection.NotEquals, selection.In, selection.NotIn: + // Pass the value through as-is. + // It's somewhat strange to do this for both the regex and equality + // operators, but if we do it this way it gives the user a little more control. + // They might choose to send an "IN" request and give a list of static values + // or they could send a single value that's a regex, giving them a passthrough + // for their label selector. + return strings.Join(values, "|"), nil + case selection.Exists, selection.DoesNotExist: + return "", ErrQueryUnsupportedValues + } + } + + return "", errors.New("operator not supported by query builder") +} + +func (q *metricsQuery) operatorIsSupported(operator selection.Operator) bool { + return operator != selection.GreaterThan && operator != selection.LessThan +}