Add label value replacers to the config and query

pkg/config/config.go

@@ -38,6 +38,16 @@ type DiscoveryRule struct {
 	// `.GroupBy` is the comma-separated expected group-by label names. The delimeters
 	// are `<<` and `>>`.
 	MetricsQuery string `json:"metricsQuery,omitempty" yaml:"metricsQuery,omitempty"`
+	// LabelValueReplacements is a list of find/replace strings that will be run against
+	// all label values before they are sent to Prometheus. This is sometimes required
+	// when prometheus labels have values that cannot be represented in Kubernetes.
+	// for example: "#" is valid as prometheus label value but not for kubernetes
+	LabelValueReplacements []LabelValueReplacement `json:"labelValueReplacements,omitempty" yaml:"labelValueReplacements,omitempty"`
+}
+
+type LabelValueReplacement struct {
+	Find    string `json:"find,omitempty" yaml:"find,omitempty"`
+	Replace string `json:"replace,omitempty" yaml:"replace,omitempty"`
 }
 
 // RegexFilter is a filter that matches positively or negatively against a regex.

pkg/naming/metric_namer.go

@@ -97,11 +97,12 @@ func (m *ReMatcher) Matches(val string) bool {
 }
 
 type metricNamer struct {
-	seriesQuery    prom.Selector
+	seriesQuery       prom.Selector
-	metricsQuery   MetricsQuery
+	metricsQuery      MetricsQuery
-	nameMatches    *regexp.Regexp
+	nameMatches       *regexp.Regexp
-	nameAs         string
+	nameAs            string
-	seriesMatchers []*ReMatcher
+	seriesMatchers    []*ReMatcher
+	labelReplacements []config.LabelValueReplacement
 
 	ResourceConverter
 }
@@ -155,7 +156,7 @@ func NamersFromConfig(cfg []config.DiscoveryRule, mapper apimeta.RESTMapper) ([]
 			return nil, err
 		}
 
-		metricsQuery, err := NewMetricsQuery(rule.MetricsQuery, resConv)
+		metricsQuery, err := NewMetricsQuery(rule.MetricsQuery, resConv, rule.LabelValueReplacements)
 		if err != nil {
 			return nil, fmt.Errorf("unable to construct metrics query associated with series query %q: %v", rule.SeriesQuery, err)
 		}

pkg/naming/metrics_query.go

@@ -28,6 +28,7 @@ import (
 	"k8s.io/apimachinery/pkg/selection"
 
 	prom "github.com/kubernetes-sigs/prometheus-adapter/pkg/client"
+	"github.com/kubernetes-sigs/prometheus-adapter/pkg/config"
 )
 
 // MetricsQuery represents a compiled metrics query for some set of
@@ -50,15 +51,16 @@ type MetricsQuery interface {
 // - LabelMatchersByName: the raw map-form of the above matchers
 // - GroupBy: the group-by clause to use for the resources in the query (stringified)
 // - GroupBySlice: the raw slice form of the above group-by clause
-func NewMetricsQuery(queryTemplate string, resourceConverter ResourceConverter) (MetricsQuery, error) {
+func NewMetricsQuery(queryTemplate string, resourceConverter ResourceConverter, labelValueReplacements []config.LabelValueReplacement) (MetricsQuery, error) {
 	templ, err := template.New("metrics-query").Delims("<<", ">>").Parse(queryTemplate)
 	if err != nil {
 		return nil, fmt.Errorf("unable to parse metrics query template %q: %v", queryTemplate, err)
 	}
 
 	return &metricsQuery{
-		resConverter: resourceConverter,
+		resConverter:           resourceConverter,
-		template:     templ,
+		template:               templ,
+		labelValueReplacements: labelValueReplacements,
 	}, nil
 }
 
@@ -66,8 +68,9 @@ func NewMetricsQuery(queryTemplate string, resourceConverter ResourceConverter)
 // with the delimiters as `<<` and `>>`, and the arguments found in
 // queryTemplateArgs.
 type metricsQuery struct {
-	resConverter ResourceConverter
+	resConverter           ResourceConverter
-	template     *template.Template
+	template               *template.Template
+	labelValueReplacements []config.LabelValueReplacement
 }
 
 // queryTemplateArgs contains the arguments for the template used in metricsQuery.
@@ -170,6 +173,12 @@ func (q *metricsQuery) BuildExternal(seriesName string, namespace string, groupB
 		return "", err
 	}
 
+	for _, repl := range q.labelValueReplacements {
+		for key, value := range valuesByName {
+			valuesByName[key] = strings.ReplaceAll(value, repl.Find, repl.Replace)
+		}
+	}
+
 	args := queryTemplateArgs{
 		Series:            seriesName,
 		LabelMatchers:     strings.Join(exprs, ","),

pkg/naming/metrics_query_test.go

@@ -76,7 +76,7 @@ func checks(cs ...checkFunc) checkFunc {
 
 func TestBuildSelector(t *testing.T) {
 	mustNewQuery := func(queryTemplate string, namespaced bool) MetricsQuery {
-		mq, err := NewMetricsQuery(queryTemplate, &resourceConverterMock{namespaced})
+		mq, err := NewMetricsQuery(queryTemplate, &resourceConverterMock{namespaced}, nil)
 		if err != nil {
 			t.Fatal(err)
 		}
@@ -272,7 +272,7 @@ func TestBuildSelector(t *testing.T) {
 
 func TestBuildExternalSelector(t *testing.T) {
 	mustNewQuery := func(queryTemplate string) MetricsQuery {
-		mq, err := NewMetricsQuery(queryTemplate, &resourceConverterMock{true})
+		mq, err := NewMetricsQuery(queryTemplate, &resourceConverterMock{true}, nil)
 		if err != nil {
 			t.Fatal(err)
 		}

pkg/resourceprovider/provider.go

@@ -54,11 +54,11 @@ func newResourceQuery(cfg config.ResourceRule, mapper apimeta.RESTMapper) (resou
 		return resourceQuery{}, fmt.Errorf("unable to construct label-resource converter: %v", err)
 	}
 
-	contQuery, err := naming.NewMetricsQuery(cfg.ContainerQuery, converter)
+	contQuery, err := naming.NewMetricsQuery(cfg.ContainerQuery, converter, nil)
 	if err != nil {
 		return resourceQuery{}, fmt.Errorf("unable to construct container metrics query: %v", err)
 	}
-	nodeQuery, err := naming.NewMetricsQuery(cfg.NodeQuery, converter)
+	nodeQuery, err := naming.NewMetricsQuery(cfg.NodeQuery, converter, nil)
 	if err != nil {
 		return resourceQuery{}, fmt.Errorf("unable to construct node metrics query: %v", err)
 	}