vendor: revendor

vendor/k8s.io/component-base/logs/datapol/datapol.go

@@ -0,0 +1,99 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package datapol contains functions to determine if objects contain sensitive
+// data to e.g. make decisions on whether to log them or not.
+package datapol
+
+import (
+	"reflect"
+	"strings"
+
+	"k8s.io/klog/v2"
+)
+
+// Verify returns a list of the datatypes contained in the argument that can be
+// considered sensitive w.r.t. to logging
+func Verify(value interface{}) []string {
+	defer func() {
+		if r := recover(); r != nil {
+			//TODO maybe export a metric
+			klog.Warningf("Error while inspecting arguments for sensitive data: %v", r)
+		}
+	}()
+	t := reflect.ValueOf(value)
+	if t.Kind() == reflect.Ptr {
+		t = t.Elem()
+	}
+	return datatypes(t)
+}
+
+func datatypes(v reflect.Value) []string {
+	if types := byType(v.Type()); len(types) > 0 {
+		// Slices, and maps can be nil or empty, only the nil case is zero
+		switch v.Kind() {
+		case reflect.Slice, reflect.Map:
+			if !v.IsZero() && v.Len() > 0 {
+				return types
+			}
+		default:
+			if !v.IsZero() {
+				return types
+			}
+		}
+	}
+	switch v.Kind() {
+	case reflect.Interface:
+		return datatypes(v.Elem())
+	case reflect.Slice, reflect.Array:
+		for i := 0; i < v.Len(); i++ {
+			if types := datatypes(v.Index(i)); len(types) > 0 {
+				return types
+			}
+		}
+	case reflect.Map:
+		mapIter := v.MapRange()
+		for mapIter.Next() {
+			k := mapIter.Key()
+			v := mapIter.Value()
+			if types := datatypes(k); len(types) > 0 {
+				return types
+			}
+			if types := datatypes(v); len(types) > 0 {
+				return types
+			}
+		}
+	case reflect.Struct:
+		t := v.Type()
+		numField := t.NumField()
+
+		for i := 0; i < numField; i++ {
+			f := t.Field(i)
+			if f.Type.Kind() == reflect.Ptr {
+				continue
+			}
+			if reason, ok := f.Tag.Lookup("datapolicy"); ok {
+				if !v.Field(i).IsZero() {
+					return strings.Split(reason, ",")
+				}
+			}
+			if types := datatypes(v.Field(i)); len(types) > 0 {
+				return types
+			}
+		}
+	}
+	return nil
+}

vendor/k8s.io/component-base/logs/datapol/externaltypes.go

@@ -0,0 +1,49 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package datapol
+
+import (
+	"fmt"
+	"reflect"
+)
+
+const (
+	httpHeader      = "net/http.Header"
+	httpCookie      = "net/http.Cookie"
+	x509Certificate = "crypto/x509.Certificate"
+)
+
+// GlobalDatapolicyMapping returns the list of sensitive datatypes are embedded
+// in types not native to Kubernetes.
+func GlobalDatapolicyMapping(v interface{}) []string {
+	return byType(reflect.TypeOf(v))
+}
+
+func byType(t reflect.Type) []string {
+	// Use string representation of the type to prevent taking a depency on the actual type.
+	switch fmt.Sprintf("%s.%s", t.PkgPath(), t.Name()) {
+	case httpHeader:
+		return []string{"password", "token"}
+	case httpCookie:
+		return []string{"token"}
+	case x509Certificate:
+		return []string{"security-key"}
+	default:
+		return nil
+	}
+
+}

vendor/k8s.io/component-base/logs/options.go

@@ -24,6 +24,7 @@ import (
 	"github.com/go-logr/logr"
 	"github.com/spf13/pflag"
 
+	"k8s.io/component-base/logs/sanitization"
 	"k8s.io/klog/v2"
 )
 
@@ -40,7 +41,8 @@ var supportedLogsFlags = map[string]struct{}{
 
 // Options has klog format parameters
 type Options struct {
-	LogFormat string
+	LogFormat       string
+	LogSanitization bool
 }
 
 // NewOptions return new klog options
@@ -88,6 +90,8 @@ func (o *Options) AddFlags(fs *pflag.FlagSet) {
 
 	// No new log formats should be added after generation is of flag options
 	logRegistry.Freeze()
+	fs.BoolVar(&o.LogSanitization, "experimental-logging-sanitization", o.LogSanitization, `[Experimental] When enabled prevents logging of fields tagged as sensitive (passwords, keys, tokens).
+Runtime log sanitization may introduce significant computation overhead and therefore should not be enabled in production.`)
 }
 
 // Apply set klog logger from LogFormat type
@@ -95,6 +99,9 @@ func (o *Options) Apply() {
 	// if log format not exists, use nil loggr
 	loggr, _ := o.Get()
 	klog.SetLogger(loggr)
+	if o.LogSanitization {
+		klog.SetLogFilter(&sanitization.SanitizingFilter{})
+	}
 }
 
 // Get logger with LogFormat field

vendor/k8s.io/component-base/logs/sanitization/sanitization.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sanitization
+
+import (
+	"fmt"
+
+	"k8s.io/component-base/logs/datapol"
+)
+
+const (
+	datapolMsgFmt = "Log message has been redacted. Log argument #%d contains: %v"
+	datapolMsg    = "Log message has been redacted."
+)
+
+// SanitizingFilter implements the LogFilter interface from klog with a set of functions that inspects the arguments with the datapol library
+type SanitizingFilter struct{}
+
+// Filter is the filter function for the non-formatting logging functions of klog.
+func (sf *SanitizingFilter) Filter(args []interface{}) []interface{} {
+	for i, v := range args {
+		types := datapol.Verify(v)
+		if len(types) > 0 {
+			return []interface{}{fmt.Sprintf(datapolMsgFmt, i, types)}
+		}
+	}
+	return args
+}
+
+// FilterF is the filter function for the formatting logging functions of klog
+func (sf *SanitizingFilter) FilterF(fmt string, args []interface{}) (string, []interface{}) {
+	for i, v := range args {
+		types := datapol.Verify(v)
+		if len(types) > 0 {
+			return datapolMsgFmt, []interface{}{i, types}
+		}
+	}
+	return fmt, args
+
+}
+
+// FilterS is the filter for the structured logging functions of klog.
+func (sf *SanitizingFilter) FilterS(msg string, keysAndValues []interface{}) (string, []interface{}) {
+	for i, v := range keysAndValues {
+		types := datapol.Verify(v)
+		if len(types) > 0 {
+			if i%2 == 0 {
+				return datapolMsg, []interface{}{"key_index", i, "types", types}
+			}
+			// since we scanned linearly we can safely log the key.
+			return datapolMsg, []interface{}{"key", keysAndValues[i-1], "types", types}
+		}
+	}
+	return msg, keysAndValues
+}

vendor/k8s.io/component-base/metrics/metric.go

@@ -188,9 +188,6 @@ func (c *selfCollector) Collect(ch chan<- prometheus.Metric) {
 // no-op vecs for convenience
 var noopCounterVec = &prometheus.CounterVec{}
 var noopHistogramVec = &prometheus.HistogramVec{}
-
-// lint:ignore U1000 Keep it for future use
-var noopSummaryVec = &prometheus.SummaryVec{}
 var noopGaugeVec = &prometheus.GaugeVec{}
 var noopObserverVec = &noopObserverVector{}
 

vendor/k8s.io/component-base/metrics/processstarttime.go

@@ -43,6 +43,12 @@ func RegisterProcessStartTime(registrationFunc func(Registerable) error) error {
 		klog.Errorf("Could not get process start time, %v", err)
 		start = float64(time.Now().Unix())
 	}
+	// processStartTime is a lazy metric which only get initialized after registered.
+	// so we have to explicitly create it before setting the label value. Otherwise
+	// it is a noop.
+	if !processStartTime.IsCreated() {
+		processStartTime.initializeMetric()
+	}
 	processStartTime.WithLabelValues().Set(start)
 	return registrationFunc(processStartTime)
 }
@@ -54,7 +60,7 @@ func getProcessStart() (float64, error) {
 		return 0, err
 	}
 
-	if stat, err := p.NewStat(); err == nil {
+	if stat, err := p.Stat(); err == nil {
 		return stat.StartTime()
 	}
 	return 0, err

vendor/k8s.io/component-base/metrics/prometheus/workqueue/metrics.go

@@ -0,0 +1,130 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package workqueue
+
+import (
+	"k8s.io/client-go/util/workqueue"
+	k8smetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+// Package prometheus sets the workqueue DefaultMetricsFactory to produce
+// prometheus metrics. To use this package, you just have to import it.
+
+// Metrics subsystem and keys used by the workqueue.
+const (
+	WorkQueueSubsystem         = "workqueue"
+	DepthKey                   = "depth"
+	AddsKey                    = "adds_total"
+	QueueLatencyKey            = "queue_duration_seconds"
+	WorkDurationKey            = "work_duration_seconds"
+	UnfinishedWorkKey          = "unfinished_work_seconds"
+	LongestRunningProcessorKey = "longest_running_processor_seconds"
+	RetriesKey                 = "retries_total"
+)
+
+var (
+	depth = k8smetrics.NewGaugeVec(&k8smetrics.GaugeOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      DepthKey,
+		Help:      "Current depth of workqueue",
+	}, []string{"name"})
+
+	adds = k8smetrics.NewCounterVec(&k8smetrics.CounterOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      AddsKey,
+		Help:      "Total number of adds handled by workqueue",
+	}, []string{"name"})
+
+	latency = k8smetrics.NewHistogramVec(&k8smetrics.HistogramOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      QueueLatencyKey,
+		Help:      "How long in seconds an item stays in workqueue before being requested.",
+		Buckets:   k8smetrics.ExponentialBuckets(10e-9, 10, 10),
+	}, []string{"name"})
+
+	workDuration = k8smetrics.NewHistogramVec(&k8smetrics.HistogramOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      WorkDurationKey,
+		Help:      "How long in seconds processing an item from workqueue takes.",
+		Buckets:   k8smetrics.ExponentialBuckets(10e-9, 10, 10),
+	}, []string{"name"})
+
+	unfinished = k8smetrics.NewGaugeVec(&k8smetrics.GaugeOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      UnfinishedWorkKey,
+		Help: "How many seconds of work has done that " +
+			"is in progress and hasn't been observed by work_duration. Large " +
+			"values indicate stuck threads. One can deduce the number of stuck " +
+			"threads by observing the rate at which this increases.",
+	}, []string{"name"})
+
+	longestRunningProcessor = k8smetrics.NewGaugeVec(&k8smetrics.GaugeOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      LongestRunningProcessorKey,
+		Help: "How many seconds has the longest running " +
+			"processor for workqueue been running.",
+	}, []string{"name"})
+
+	retries = k8smetrics.NewCounterVec(&k8smetrics.CounterOpts{
+		Subsystem: WorkQueueSubsystem,
+		Name:      RetriesKey,
+		Help:      "Total number of retries handled by workqueue",
+	}, []string{"name"})
+
+	metrics = []k8smetrics.Registerable{
+		depth, adds, latency, workDuration, unfinished, longestRunningProcessor, retries,
+	}
+)
+
+type prometheusMetricsProvider struct {
+}
+
+func init() {
+	for _, m := range metrics {
+		legacyregistry.MustRegister(m)
+	}
+	workqueue.SetProvider(prometheusMetricsProvider{})
+}
+
+func (prometheusMetricsProvider) NewDepthMetric(name string) workqueue.GaugeMetric {
+	return depth.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewAddsMetric(name string) workqueue.CounterMetric {
+	return adds.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewLatencyMetric(name string) workqueue.HistogramMetric {
+	return latency.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewWorkDurationMetric(name string) workqueue.HistogramMetric {
+	return workDuration.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewUnfinishedWorkSecondsMetric(name string) workqueue.SettableGaugeMetric {
+	return unfinished.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewLongestRunningProcessorSecondsMetric(name string) workqueue.SettableGaugeMetric {
+	return longestRunningProcessor.WithLabelValues(name)
+}
+
+func (prometheusMetricsProvider) NewRetriesMetric(name string) workqueue.CounterMetric {
+	return retries.WithLabelValues(name)
+}

vendor/k8s.io/component-base/metrics/testutil/metrics.go

@@ -86,7 +86,7 @@ func ParseMetrics(data string, output *Metrics) error {
 			continue
 		}
 		for _, metric := range v {
-			name := string(metric.Metric[model.MetricNameLabel])
+			name := string(metric.Metric[MetricNameLabel])
 			(*output)[name] = append((*output)[name], metric)
 		}
 	}
@@ -101,28 +101,6 @@ func TextToMetricFamilies(in io.Reader) (map[string]*dto.MetricFamily, error) {
 	return textParser.TextToMetricFamilies(in)
 }
 
-// ExtractMetricSamples parses the prometheus metric samples from the input string.
-func ExtractMetricSamples(metricsBlob string) ([]*model.Sample, error) {
-	dec := expfmt.NewDecoder(strings.NewReader(metricsBlob), expfmt.FmtText)
-	decoder := expfmt.SampleDecoder{
-		Dec:  dec,
-		Opts: &expfmt.DecodeOptions{},
-	}
-
-	var samples []*model.Sample
-	for {
-		var v model.Vector
-		if err := decoder.Decode(&v); err != nil {
-			if err == io.EOF {
-				// Expected loop termination condition.
-				return samples, nil
-			}
-			return nil, err
-		}
-		samples = append(samples, v...)
-	}
-}
-
 // PrintSample returns formatted representation of metric Sample
 func PrintSample(sample *model.Sample) string {
 	buf := make([]string, 0)

vendor/k8s.io/component-base/metrics/testutil/promlint.go

@@ -33,11 +33,6 @@ var exceptionMetrics = []string{
 	// k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/egressselector
 	"apiserver_egress_dialer_dial_failure_count", // counter metrics should have "_total" suffix
 
-	// k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset
-	"apiserver_flowcontrol_current_inqueue_requests",   // label names should be written in 'snake_case' not 'camelCase',
-	"apiserver_flowcontrol_current_executing_requests", // label names should be written in 'snake_case' not 'camelCase'
-	"apiserver_flowcontrol_rejected_requests_total",    // label names should be written in 'snake_case' not 'camelCase'
-
 	// k8s.io/kubernetes/vendor/k8s.io/apiserver/pkg/server/healthz
 	"apiserver_request_total", // label names should be written in 'snake_case' not 'camelCase'