vendor: Update vendor logic

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -19,14 +19,11 @@ reviewers:
 - timothysc
 - hongchaodeng
 - krousey
-- fgrzadkowski
 - xiang90
 - mml
 - ingvagabund
 - resouer
 - mbohlool
-- lixiaobing10051267
 - mqliang
-- feihujiang
 - rrati
 - enj

vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -24,8 +24,6 @@ import (
 	"sync"
 	"time"
 
-	"k8s.io/klog"
-
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -41,16 +39,26 @@ import (
 	"k8s.io/apiserver/pkg/storage"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/cache"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	"k8s.io/klog"
 	utiltrace "k8s.io/utils/trace"
-
-	"github.com/prometheus/client_golang/prometheus"
 )
 
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
 var (
-	initCounter = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "apiserver_init_events_total",
-			Help: "Counter of init events processed in watchcache broken by resource type",
+	initCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "apiserver_init_events_total",
+			Help:           "Counter of init events processed in watchcache broken by resource type",
+			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"resource"},
 	)
@@ -64,7 +72,7 @@ const (
 )
 
 func init() {
-	prometheus.MustRegister(initCounter)
+	legacyregistry.MustRegister(initCounter)
 }
 
 // Config contains the configuration for a given Cache.
@@ -88,9 +96,9 @@ type Config struct {
 	// GetAttrsFunc is used to get object labels, fields
 	GetAttrsFunc func(runtime.Object) (label labels.Set, field fields.Set, err error)
 
-	// TriggerPublisherFunc is used for optimizing amount of watchers that
+	// IndexerFuncs is used for optimizing amount of watchers that
 	// needs to process an incoming event.
-	TriggerPublisherFunc storage.TriggerPublisherFunc
+	IndexerFuncs storage.IndexerFuncs
 
 	// NewFunc is a function that creates new empty object storing a object of type Type.
 	NewFunc func() runtime.Object
@@ -154,17 +162,17 @@ func (i *indexedWatchers) terminateAll(objectType reflect.Type, done func(*cache
 		klog.Warningf("Terminating all watchers from cacher %v", objectType)
 	}
 	i.allWatchers.terminateAll(done)
-	for index, watchers := range i.valueWatchers {
+	for _, watchers := range i.valueWatchers {
 		watchers.terminateAll(done)
-		delete(i.valueWatchers, index)
 	}
+	i.valueWatchers = map[string]watchersMap{}
 }
 
 // As we don't need a high precision here, we keep all watchers timeout within a
 // second in a bucket, and pop up them once at the timeout. To be more specific,
 // if you set fire time at X, you can get the bookmark within (X-1,X+1) period.
-// This is NOT thread-safe.
 type watcherBookmarkTimeBuckets struct {
+	lock            sync.Mutex
 	watchersBuckets map[int64][]*cacheWatcher
 	startBucketID   int64
 	clock           clock.Clock
@@ -186,6 +194,8 @@ func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
 		return false
 	}
 	bucketID := nextTime.Unix()
+	t.lock.Lock()
+	defer t.lock.Unlock()
 	if bucketID < t.startBucketID {
 		bucketID = t.startBucketID
 	}
@@ -198,6 +208,8 @@ func (t *watcherBookmarkTimeBuckets) popExpiredWatchers() [][]*cacheWatcher {
 	currentBucketID := t.clock.Now().Unix()
 	// There should be one or two elements in almost all cases
 	expiredWatchers := make([][]*cacheWatcher, 0, 2)
+	t.lock.Lock()
+	defer t.lock.Unlock()
 	for ; t.startBucketID <= currentBucketID; t.startBucketID++ {
 		if watchers, ok := t.watchersBuckets[t.startBucketID]; ok {
 			delete(t.watchersBuckets, t.startBucketID)
@@ -209,6 +221,11 @@ func (t *watcherBookmarkTimeBuckets) popExpiredWatchers() [][]*cacheWatcher {
 
 type filterWithAttrsFunc func(key string, l labels.Set, f fields.Set) bool
 
+type indexedTriggerFunc struct {
+	indexName   string
+	indexerFunc storage.IndexerFunc
+}
+
 // Cacher is responsible for serving WATCH and LIST requests for a given
 // resource from its internal cache and updating its cache in the background
 // based on the underlying storage contents.
@@ -248,9 +265,9 @@ type Cacher struct {
 	// newFunc is a function that creates new empty object storing a object of type Type.
 	newFunc func() runtime.Object
 
-	// triggerFunc is used for optimizing amount of watchers that needs to process
+	// indexedTrigger is used for optimizing amount of watchers that needs to process
 	// an incoming event.
-	triggerFunc storage.TriggerPublisherFunc
+	indexedTrigger *indexedTriggerFunc
 	// watchers is mapping from the value of trigger function that a
 	// watcher is interested into the watchers
 	watcherIdx int
@@ -276,37 +293,54 @@ type Cacher struct {
 	// watchersBuffer is a list of watchers potentially interested in currently
 	// dispatched event.
 	watchersBuffer []*cacheWatcher
+	// blockedWatchers is a list of watchers whose buffer is currently full.
+	blockedWatchers []*cacheWatcher
 	// watchersToStop is a list of watchers that were supposed to be stopped
 	// during current dispatching, but stopping was deferred to the end of
 	// dispatching that event to avoid race with closing channels in watchers.
 	watchersToStop []*cacheWatcher
 	// Maintain a timeout queue to send the bookmark event before the watcher times out.
 	bookmarkWatchers *watcherBookmarkTimeBuckets
-	// watchBookmark feature-gate
-	watchBookmarkEnabled bool
 }
 
 // NewCacherFromConfig creates a new Cacher responsible for servicing WATCH and LIST requests from
 // its internal cache and updating its cache in the background based on the
 // given configuration.
-func NewCacherFromConfig(config Config) *Cacher {
+func NewCacherFromConfig(config Config) (*Cacher, error) {
 	stopCh := make(chan struct{})
 	obj := config.NewFunc()
 	// Give this error when it is constructed rather than when you get the
 	// first watch item, because it's much easier to track down that way.
 	if err := runtime.CheckCodec(config.Codec, obj); err != nil {
-		panic("storage codec doesn't seem to match given type: " + err.Error())
+		return nil, fmt.Errorf("storage codec doesn't seem to match given type: %v", err)
+	}
+
+	var indexedTrigger *indexedTriggerFunc
+	if config.IndexerFuncs != nil {
+		// For now, we don't support multiple trigger functions defined
+		// for a given resource.
+		if len(config.IndexerFuncs) > 1 {
+			return nil, fmt.Errorf("cacher %s doesn't support more than one IndexerFunc: ", reflect.TypeOf(obj).String())
+		}
+		for key, value := range config.IndexerFuncs {
+			if value != nil {
+				indexedTrigger = &indexedTriggerFunc{
+					indexName:   key,
+					indexerFunc: value,
+				}
+			}
+		}
 	}
 
 	clock := clock.RealClock{}
 	cacher := &Cacher{
-		ready:       newReady(),
-		storage:     config.Storage,
-		objectType:  reflect.TypeOf(obj),
-		versioner:   config.Versioner,
-		newFunc:     config.NewFunc,
-		triggerFunc: config.TriggerPublisherFunc,
-		watcherIdx:  0,
+		ready:          newReady(),
+		storage:        config.Storage,
+		objectType:     reflect.TypeOf(obj),
+		versioner:      config.Versioner,
+		newFunc:        config.NewFunc,
+		indexedTrigger: indexedTrigger,
+		watcherIdx:     0,
 		watchers: indexedWatchers{
 			allWatchers:   make(map[int]*cacheWatcher),
 			valueWatchers: make(map[string]watchersMap),
@@ -319,11 +353,10 @@ func NewCacherFromConfig(config Config) *Cacher {
 		// - reflector.ListAndWatch
 		// and there are no guarantees on the order that they will stop.
 		// So we will be simply closing the channel, and synchronizing on the WaitGroup.
-		stopCh:               stopCh,
-		clock:                clock,
-		timer:                time.NewTimer(time.Duration(0)),
-		bookmarkWatchers:     newTimeBucketWatchers(clock),
-		watchBookmarkEnabled: utilfeature.DefaultFeatureGate.Enabled(features.WatchBookmark),
+		stopCh:           stopCh,
+		clock:            clock,
+		timer:            time.NewTimer(time.Duration(0)),
+		bookmarkWatchers: newTimeBucketWatchers(clock),
 	}
 
 	// Ensure that timer is stopped.
@@ -361,7 +394,7 @@ func NewCacherFromConfig(config Config) *Cacher {
 		)
 	}()
 
-	return cacher
+	return cacher, nil
 }
 
 func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
@@ -417,23 +450,27 @@ func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string,
 	c.ready.wait()
 
 	triggerValue, triggerSupported := "", false
-	// TODO: Currently we assume that in a given Cacher object, any <predicate> that is
-	// passed here is aware of exactly the same trigger (at most one).
-	// Thus, either 0 or 1 values will be returned.
-	if matchValues := pred.MatcherIndex(); len(matchValues) > 0 {
-		triggerValue, triggerSupported = matchValues[0].Value, true
+	if c.indexedTrigger != nil {
+		for _, field := range pred.IndexFields {
+			if field == c.indexedTrigger.indexName {
+				if value, ok := pred.Field.RequiresExactMatch(field); ok {
+					triggerValue, triggerSupported = value, true
+				}
+			}
+		}
 	}
 
-	// If there is triggerFunc defined, but triggerSupported is false,
+	// If there is indexedTrigger defined, but triggerSupported is false,
 	// we can't narrow the amount of events significantly at this point.
 	//
-	// That said, currently triggerFunc is defined only for Pods and Nodes,
-	// and there is only constant number of watchers for which triggerSupported
-	// is false (excluding those issues explicitly by users).
+	// That said, currently indexedTrigger is defined only for couple resources:
+	// Pods, Nodes, Secrets and ConfigMaps and there is only a constant
+	// number of watchers for which triggerSupported is false (excluding those
+	// issued explicitly by users).
 	// Thus, to reduce the risk of those watchers blocking all watchers of a
 	// given resource in the system, we increase the sizes of buffers for them.
 	chanSize := 10
-	if c.triggerFunc != nil && !triggerSupported {
+	if c.indexedTrigger != nil && !triggerSupported {
 		// TODO: We should tune this value and ideally make it dependent on the
 		// number of objects of a given type and/or their churn.
 		chanSize = 1000
@@ -476,8 +513,8 @@ func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string,
 		watcher.forget = forgetWatcher(c, c.watcherIdx, triggerValue, triggerSupported)
 		c.watchers.addWatcher(watcher, c.watcherIdx, triggerValue, triggerSupported)
 
-		// Add it to the queue only when server and client support watch bookmarks.
-		if c.watchBookmarkEnabled && watcher.allowWatchBookmarks {
+		// Add it to the queue only when the client support watch bookmarks.
+		if watcher.allowWatchBookmarks {
 			c.bookmarkWatchers.addWatcher(watcher)
 		}
 		c.watcherIdx++
@@ -572,7 +609,7 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 		return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
 	}
 
-	trace := utiltrace.New(fmt.Sprintf("cacher %v: List", c.objectType.String()))
+	trace := utiltrace.New("cacher list", utiltrace.Field{"type", c.objectType.String()})
 	defer trace.LogIfLong(500 * time.Millisecond)
 
 	c.ready.wait()
@@ -584,7 +621,10 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 		return err
 	}
 	listVal, err := conversion.EnforcePtr(listPtr)
-	if err != nil || listVal.Kind() != reflect.Slice {
+	if err != nil {
+		return err
+	}
+	if listVal.Kind() != reflect.Slice {
 		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
 	}
 	filter := filterWithAttrsFunction(key, pred)
@@ -641,7 +681,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 		return c.storage.List(ctx, key, resourceVersion, pred, listObj)
 	}
 
-	trace := utiltrace.New(fmt.Sprintf("cacher %v: List", c.objectType.String()))
+	trace := utiltrace.New("cacher list", utiltrace.Field{"type", c.objectType.String()})
 	defer trace.LogIfLong(500 * time.Millisecond)
 
 	c.ready.wait()
@@ -653,7 +693,10 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 		return err
 	}
 	listVal, err := conversion.EnforcePtr(listPtr)
-	if err != nil || listVal.Kind() != reflect.Slice {
+	if err != nil {
+		return err
+	}
+	if listVal.Kind() != reflect.Slice {
 		return fmt.Errorf("need a pointer to slice, got %v", listVal.Kind())
 	}
 	filter := filterWithAttrsFunction(key, pred)
@@ -662,7 +705,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 	if err != nil {
 		return err
 	}
-	trace.Step(fmt.Sprintf("Listed %d items from cache", len(objs)))
+	trace.Step("Listed items from cache", utiltrace.Field{"count", len(objs)})
 	if len(objs) > listVal.Cap() && pred.Label.Empty() && pred.Field.Empty() {
 		// Resize the slice appropriately, since we already know that none
 		// of the elements will be filtered out.
@@ -678,7 +721,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 			listVal.Set(reflect.Append(listVal, reflect.ValueOf(elem.Object).Elem()))
 		}
 	}
-	trace.Step(fmt.Sprintf("Filtered %d items", listVal.Len()))
+	trace.Step("Filtered items", utiltrace.Field{"count", listVal.Len()})
 	if c.versioner != nil {
 		if err := c.versioner.UpdateList(listObj, readResourceVersion, "", nil); err != nil {
 			return err
@@ -708,30 +751,29 @@ func (c *Cacher) Count(pathPrefix string) (int64, error) {
 	return c.storage.Count(pathPrefix)
 }
 
-func (c *Cacher) triggerValues(event *watchCacheEvent) ([]string, bool) {
-	// TODO: Currently we assume that in a given Cacher object, its <c.triggerFunc>
-	// is aware of exactly the same trigger (at most one). Thus calling:
-	//   c.triggerFunc(<some object>)
-	// can return only 0 or 1 values.
-	// That means, that triggerValues itself may return up to 2 different values.
-	if c.triggerFunc == nil {
+// baseObjectThreadUnsafe omits locking for cachingObject.
+func baseObjectThreadUnsafe(object runtime.Object) runtime.Object {
+	if co, ok := object.(*cachingObject); ok {
+		return co.object
+	}
+	return object
+}
+
+func (c *Cacher) triggerValuesThreadUnsafe(event *watchCacheEvent) ([]string, bool) {
+	if c.indexedTrigger == nil {
 		return nil, false
 	}
+
 	result := make([]string, 0, 2)
-	matchValues := c.triggerFunc(event.Object)
-	if len(matchValues) > 0 {
-		result = append(result, matchValues[0].Value)
-	}
+	result = append(result, c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.Object)))
 	if event.PrevObject == nil {
-		return result, len(result) > 0
+		return result, true
 	}
-	prevMatchValues := c.triggerFunc(event.PrevObject)
-	if len(prevMatchValues) > 0 {
-		if len(result) == 0 || result[0] != prevMatchValues[0].Value {
-			result = append(result, prevMatchValues[0].Value)
-		}
+	prevTriggerValue := c.indexedTrigger.indexerFunc(baseObjectThreadUnsafe(event.PrevObject))
+	if result[0] != prevTriggerValue {
+		result = append(result, prevTriggerValue)
 	}
-	return result, len(result) > 0
+	return result, true
 }
 
 func (c *Cacher) processEvent(event *watchCacheEvent) {
@@ -745,10 +787,6 @@ func (c *Cacher) processEvent(event *watchCacheEvent) {
 func (c *Cacher) dispatchEvents() {
 	// Jitter to help level out any aggregate load.
 	bookmarkTimer := c.clock.NewTimer(wait.Jitter(time.Second, 0.25))
-	// Stop the timer when watchBookmarkFeatureGate is not enabled.
-	if !c.watchBookmarkEnabled && !bookmarkTimer.Stop() {
-		<-bookmarkTimer.C()
-	}
 	defer bookmarkTimer.Stop()
 
 	lastProcessedResourceVersion := uint64(0)
@@ -765,6 +803,8 @@ func (c *Cacher) dispatchEvents() {
 			// Never send a bookmark event if we did not see an event here, this is fine
 			// because we don't provide any guarantees on sending bookmarks.
 			if lastProcessedResourceVersion == 0 {
+				// pop expired watchers in case there has been no update
+				c.bookmarkWatchers.popExpiredWatchers()
 				continue
 			}
 			bookmarkEvent := &watchCacheEvent{
@@ -783,19 +823,99 @@ func (c *Cacher) dispatchEvents() {
 	}
 }
 
+func setCachingObjects(event *watchCacheEvent, versioner storage.Versioner) {
+	switch event.Type {
+	case watch.Added, watch.Modified:
+		if object, err := newCachingObject(event.Object); err == nil {
+			event.Object = object
+		} else {
+			klog.Errorf("couldn't create cachingObject from: %#v", event.Object)
+		}
+		// Don't wrap PrevObject for update event (for create events it is nil).
+		// We only encode those to deliver DELETE watch events, so if
+		// event.Object is not nil it can be used only for watchers for which
+		// selector was satisfied for its previous version and is no longer
+		// satisfied for the current version.
+		// This is rare enough that it doesn't justify making deep-copy of the
+		// object (done by newCachingObject) every time.
+	case watch.Deleted:
+		// Don't wrap Object for delete events - these are not to deliver any
+		// events. Only wrap PrevObject.
+		if object, err := newCachingObject(event.PrevObject); err == nil {
+			// Update resource version of the underlying object.
+			// event.PrevObject is used to deliver DELETE watch events and
+			// for them, we set resourceVersion to <current> instead of
+			// the resourceVersion of the last modification of the object.
+			updateResourceVersionIfNeeded(object.object, versioner, event.ResourceVersion)
+			event.PrevObject = object
+		} else {
+			klog.Errorf("couldn't create cachingObject from: %#v", event.Object)
+		}
+	}
+}
+
 func (c *Cacher) dispatchEvent(event *watchCacheEvent) {
 	c.startDispatching(event)
 	defer c.finishDispatching()
 	// Watchers stopped after startDispatching will be delayed to finishDispatching,
 
 	// Since add() can block, we explicitly add when cacher is unlocked.
+	// Dispatching event in nonblocking way first, which make faster watchers
+	// not be blocked by slower ones.
 	if event.Type == watch.Bookmark {
 		for _, watcher := range c.watchersBuffer {
 			watcher.nonblockingAdd(event)
 		}
 	} else {
+		// Set up caching of object serializations only for dispatching this event.
+		//
+		// Storing serializations in memory would result in increased memory usage,
+		// but it would help for caching encodings for watches started from old
+		// versions. However, we still don't have a convincing data that the gain
+		// from it justifies increased memory usage, so for now we drop the cached
+		// serializations after dispatching this event.
+		//
+		// Given the deep-copies that are done to create cachingObjects,
+		// we try to cache serializations only if there are at least 3 watchers.
+		if len(c.watchersBuffer) >= 3 {
+			// Make a shallow copy to allow overwriting Object and PrevObject.
+			wcEvent := *event
+			setCachingObjects(&wcEvent, c.versioner)
+			event = &wcEvent
+		}
+
+		c.blockedWatchers = c.blockedWatchers[:0]
 		for _, watcher := range c.watchersBuffer {
-			watcher.add(event, c.timer, c.dispatchTimeoutBudget)
+			if !watcher.nonblockingAdd(event) {
+				c.blockedWatchers = append(c.blockedWatchers, watcher)
+			}
+		}
+
+		if len(c.blockedWatchers) > 0 {
+			// dispatchEvent is called very often, so arrange
+			// to reuse timers instead of constantly allocating.
+			startTime := time.Now()
+			timeout := c.dispatchTimeoutBudget.takeAvailable()
+			c.timer.Reset(timeout)
+
+			// Make sure every watcher will try to send event without blocking first,
+			// even if the timer has already expired.
+			timer := c.timer
+			for _, watcher := range c.blockedWatchers {
+				if !watcher.add(event, timer) {
+					// fired, clean the timer by set it to nil.
+					timer = nil
+				}
+			}
+
+			// Stop the timer if it is not fired
+			if timer != nil && !timer.Stop() {
+				// Consume triggered (but not yet received) timer event
+				// so that future reuse does not get a spurious timeout.
+				<-timer.C
+			}
+
+			c.dispatchTimeoutBudget.returnUnused(timeout - time.Since(startTime))
 		}
 	}
 }
@@ -805,7 +925,8 @@ func (c *Cacher) startDispatchingBookmarkEvents() {
 	// as we don't delete watcher from bookmarkWatchers when it is stopped.
 	for _, watchers := range c.bookmarkWatchers.popExpiredWatchers() {
 		for _, watcher := range watchers {
-			// watcher.stop() is protected by c.Lock()
+			// c.Lock() is held here.
+			// watcher.stopThreadUnsafe() is protected by c.Lock()
 			if watcher.stopped {
 				continue
 			}
@@ -820,7 +941,10 @@ func (c *Cacher) startDispatchingBookmarkEvents() {
 // startDispatching chooses watchers potentially interested in a given event
 // a marks dispatching as true.
 func (c *Cacher) startDispatching(event *watchCacheEvent) {
-	triggerValues, supported := c.triggerValues(event)
+	// It is safe to call triggerValuesThreadUnsafe here, because at this
+	// point only this thread can access this event (we create a separate
+	// watchCacheEvent for every dispatch).
+	triggerValues, supported := c.triggerValuesThreadUnsafe(event)
 
 	c.Lock()
 	defer c.Unlock()
@@ -875,7 +999,7 @@ func (c *Cacher) finishDispatching() {
 	defer c.Unlock()
 	c.dispatching = false
 	for _, watcher := range c.watchersToStop {
-		watcher.stop()
+		watcher.stopThreadUnsafe()
 	}
 	c.watchersToStop = c.watchersToStop[:0]
 }
@@ -890,7 +1014,7 @@ func (c *Cacher) stopWatcherThreadUnsafe(watcher *cacheWatcher) {
 	if c.dispatching {
 		c.watchersToStop = append(c.watchersToStop, watcher)
 	} else {
-		watcher.stop()
+		watcher.stopThreadUnsafe()
 	}
 }
 
@@ -920,7 +1044,7 @@ func forgetWatcher(c *Cacher, index int, triggerValue string, triggerSupported b
 		defer c.Unlock()
 
 		// It's possible that the watcher is already not in the structure (e.g. in case of
-		// simultaneous Stop() and terminateAllWatchers(), but it is safe to call stop()
+		// simultaneous Stop() and terminateAllWatchers(), but it is safe to call stopThreadUnsafe()
 		// on a watcher multiple times.
 		c.watchers.deleteWatcher(index, triggerValue, triggerSupported, c.stopWatcherThreadUnsafe)
 	}
@@ -1022,8 +1146,8 @@ func (c *errWatcher) Stop() {
 }
 
 // cacheWatcher implements watch.Interface
+// this is not thread-safe
 type cacheWatcher struct {
-	sync.Mutex
 	input     chan *watchCacheEvent
 	result    chan watch.Event
 	done      chan struct{}
@@ -1064,12 +1188,8 @@ func (c *cacheWatcher) Stop() {
 	c.forget()
 }
 
-// TODO(#73958)
-// stop() is protected by Cacher.Lock(), rename it to
-// stopThreadUnsafe and remove the sync.Mutex.
-func (c *cacheWatcher) stop() {
-	c.Lock()
-	defer c.Unlock()
+// we rely on the fact that stopThredUnsafe is actually protected by Cacher.Lock()
+func (c *cacheWatcher) stopThreadUnsafe() {
 	if !c.stopped {
 		c.stopped = true
 		close(c.done)
@@ -1078,7 +1198,6 @@ func (c *cacheWatcher) stop() {
 }
 
 func (c *cacheWatcher) nonblockingAdd(event *watchCacheEvent) bool {
-	// If we can't send it, don't block on it.
 	select {
 	case c.input <- event:
 		return true
@@ -1087,36 +1206,34 @@ func (c *cacheWatcher) nonblockingAdd(event *watchCacheEvent) bool {
 	}
 }
 
-func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer, budget *timeBudget) {
+// Nil timer means that add will not block (if it can't send event immediately, it will break the watcher)
+func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool {
 	// Try to send the event immediately, without blocking.
 	if c.nonblockingAdd(event) {
-		return
+		return true
 	}
 
-	// OK, block sending, but only for up to <timeout>.
-	// cacheWatcher.add is called very often, so arrange
-	// to reuse timers instead of constantly allocating.
-	startTime := time.Now()
-	timeout := budget.takeAvailable()
-
-	timer.Reset(timeout)
-
-	select {
-	case c.input <- event:
-		if !timer.Stop() {
-			// Consume triggered (but not yet received) timer event
-			// so that future reuse does not get a spurious timeout.
-			<-timer.C
-		}
-	case <-timer.C:
+	closeFunc := func() {
 		// This means that we couldn't send event to that watcher.
 		// Since we don't want to block on it infinitely,
 		// we simply terminate it.
-		klog.V(1).Infof("Forcing watcher close due to unresponsiveness: %v", reflect.TypeOf(event.Object).String())
+		klog.V(1).Infof("Forcing watcher close due to unresponsiveness: %v", c.objectType.String())
 		c.forget()
 	}
 
-	budget.returnUnused(timeout - time.Since(startTime))
+	if timer == nil {
+		closeFunc()
+		return false
+	}
+
+	// OK, block sending, but only until timer fires.
+	select {
+	case c.input <- event:
+		return true
+	case <-timer.C:
+		closeFunc()
+		return false
+	}
 }
 
 func (c *cacheWatcher) nextBookmarkTime(now time.Time) (time.Time, bool) {
@@ -1128,6 +1245,25 @@ func (c *cacheWatcher) nextBookmarkTime(now time.Time) (time.Time, bool) {
 	return c.deadline.Add(-2 * time.Second), true
 }
 
+func getEventObject(object runtime.Object) runtime.Object {
+	if _, ok := object.(runtime.CacheableObject); ok {
+		// It is safe to return without deep-copy, because the underlying
+		// object was already deep-copied during construction.
+		return object
+	}
+	return object.DeepCopyObject()
+}
+
+func updateResourceVersionIfNeeded(object runtime.Object, versioner storage.Versioner, resourceVersion uint64) {
+	if _, ok := object.(*cachingObject); ok {
+		// We assume that for cachingObject resourceVersion was already propagated before.
+		return
+	}
+	if err := versioner.UpdateObject(object, resourceVersion); err != nil {
+		utilruntime.HandleError(fmt.Errorf("failure to version api object (%d) %#v: %v", resourceVersion, object, err))
+	}
+}
+
 func (c *cacheWatcher) convertToWatchEvent(event *watchCacheEvent) *watch.Event {
 	if event.Type == watch.Bookmark {
 		return &watch.Event{Type: watch.Bookmark, Object: event.Object.DeepCopyObject()}
@@ -1145,15 +1281,13 @@ func (c *cacheWatcher) convertToWatchEvent(event *watchCacheEvent) *watch.Event
 
 	switch {
 	case curObjPasses && !oldObjPasses:
-		return &watch.Event{Type: watch.Added, Object: event.Object.DeepCopyObject()}
+		return &watch.Event{Type: watch.Added, Object: getEventObject(event.Object)}
 	case curObjPasses && oldObjPasses:
-		return &watch.Event{Type: watch.Modified, Object: event.Object.DeepCopyObject()}
+		return &watch.Event{Type: watch.Modified, Object: getEventObject(event.Object)}
 	case !curObjPasses && oldObjPasses:
 		// return a delete event with the previous object content, but with the event's resource version
-		oldObj := event.PrevObject.DeepCopyObject()
-		if err := c.versioner.UpdateObject(oldObj, event.ResourceVersion); err != nil {
-			utilruntime.HandleError(fmt.Errorf("failure to version api object (%d) %#v: %v", event.ResourceVersion, oldObj, err))
-		}
+		oldObj := getEventObject(event.PrevObject)
+		updateResourceVersionIfNeeded(oldObj, c.versioner, event.ResourceVersion)
 		return &watch.Event{Type: watch.Deleted, Object: oldObj}
 	}
 

vendor/k8s.io/apiserver/pkg/storage/cacher/caching_object.go

@@ -0,0 +1,397 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cacher
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"reflect"
+	"runtime/debug"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/klog"
+)
+
+var _ runtime.CacheableObject = &cachingObject{}
+
+// metaRuntimeInterface implements runtime.Object and
+// metav1.Object interfaces.
+type metaRuntimeInterface interface {
+	runtime.Object
+	metav1.Object
+}
+
+// serializationResult captures a result of serialization.
+type serializationResult struct {
+	// once should be used to ensure serialization is computed once.
+	once sync.Once
+
+	// raw is serialized object.
+	raw []byte
+	// err is error from serialization.
+	err error
+}
+
+// serializationsCache is a type for caching serialization results.
+type serializationsCache map[runtime.Identifier]*serializationResult
+
+// cachingObject is an object that is able to cache its serializations
+// so that each of those is computed exactly once.
+//
+// cachingObject implements the metav1.Object interface (accessors for
+// all metadata fields). However, setters for all fields except from
+// SelfLink (which is set lately in the path) are ignored.
+type cachingObject struct {
+	lock sync.RWMutex
+
+	// Object for which serializations are cached.
+	object metaRuntimeInterface
+
+	// serializations is a cache containing object`s serializations.
+	// The value stored in atomic.Value is of type serializationsCache.
+	// The atomic.Value type is used to allow fast-path.
+	serializations atomic.Value
+}
+
+// newCachingObject performs a deep copy of the given object and wraps it
+// into a cachingObject.
+// An error is returned if it's not possible to cast the object to
+// metav1.Object type.
+func newCachingObject(object runtime.Object) (*cachingObject, error) {
+	if obj, ok := object.(metaRuntimeInterface); ok {
+		result := &cachingObject{object: obj.DeepCopyObject().(metaRuntimeInterface)}
+		result.serializations.Store(make(serializationsCache))
+		return result, nil
+	}
+	return nil, fmt.Errorf("can't cast object to metav1.Object: %#v", object)
+}
+
+func (o *cachingObject) getSerializationResult(id runtime.Identifier) *serializationResult {
+	// Fast-path for getting from cache.
+	serializations := o.serializations.Load().(serializationsCache)
+	if result, exists := serializations[id]; exists {
+		return result
+	}
+
+	// Slow-path (that may require insert).
+	o.lock.Lock()
+	defer o.lock.Unlock()
+
+	serializations = o.serializations.Load().(serializationsCache)
+	// Check if in the meantime it wasn't inserted.
+	if result, exists := serializations[id]; exists {
+		return result
+	}
+
+	// Insert an entry for <id>. This requires copy of existing map.
+	newSerializations := make(serializationsCache)
+	for k, v := range serializations {
+		newSerializations[k] = v
+	}
+	result := &serializationResult{}
+	newSerializations[id] = result
+	o.serializations.Store(newSerializations)
+	return result
+}
+
+// CacheEncode implements runtime.CacheableObject interface.
+// It serializes the object and writes the result to given io.Writer trying
+// to first use the already cached result and falls back to a given encode
+// function in case of cache miss.
+// It assumes that for a given identifier, the encode function always encodes
+// each input object into the same output format.
+func (o *cachingObject) CacheEncode(id runtime.Identifier, encode func(runtime.Object, io.Writer) error, w io.Writer) error {
+	result := o.getSerializationResult(id)
+	result.once.Do(func() {
+		buffer := bytes.NewBuffer(nil)
+		result.err = encode(o.GetObject(), buffer)
+		result.raw = buffer.Bytes()
+	})
+	// Once invoked, fields of serialization will not change.
+	if result.err != nil {
+		return result.err
+	}
+	_, err := w.Write(result.raw)
+	return err
+}
+
+// GetObject implements runtime.CacheableObject interface.
+// It returns deep-copy of the wrapped object to return ownership of it
+// to the called according to the contract of the interface.
+func (o *cachingObject) GetObject() runtime.Object {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.DeepCopyObject().(metaRuntimeInterface)
+}
+
+// GetObjectKind implements runtime.Object interface.
+func (o *cachingObject) GetObjectKind() schema.ObjectKind {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetObjectKind()
+}
+
+// DeepCopyObject implements runtime.Object interface.
+func (o *cachingObject) DeepCopyObject() runtime.Object {
+	// DeepCopyObject on cachingObject is not expected to be called anywhere.
+	// However, to be on the safe-side, we implement it, though given the
+	// cache is only an optimization we ignore copying it.
+	result := &cachingObject{}
+	result.serializations.Store(make(serializationsCache))
+
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	result.object = o.object.DeepCopyObject().(metaRuntimeInterface)
+	return result
+}
+
+var (
+	invalidationCacheTimestampLock sync.Mutex
+	invalidationCacheTimestamp     time.Time
+)
+
+// shouldLogCacheInvalidation allows for logging cache-invalidation
+// at most once per second (to avoid spamming logs in case of issues).
+func shouldLogCacheInvalidation(now time.Time) bool {
+	invalidationCacheTimestampLock.Lock()
+	defer invalidationCacheTimestampLock.Unlock()
+	if invalidationCacheTimestamp.Add(time.Second).Before(now) {
+		invalidationCacheTimestamp = now
+		return true
+	}
+	return false
+}
+
+func (o *cachingObject) invalidateCacheLocked() {
+	if cache, ok := o.serializations.Load().(serializationsCache); ok && len(cache) == 0 {
+		return
+	}
+	// We don't expect cache invalidation to happen - so we want
+	// to log the stacktrace to allow debugging if that will happen.
+	// OTOH, we don't want to spam logs with it.
+	// So we try to log it at most once per second.
+	if shouldLogCacheInvalidation(time.Now()) {
+		klog.Warningf("Unexpected cache invalidation for %#v\n%s", o.object, string(debug.Stack()))
+	}
+	o.serializations.Store(make(serializationsCache))
+}
+
+// The following functions implement metav1.Object interface:
+// - getters simply delegate for the underlying object
+// - setters check if operations isn't noop and if so,
+//   invalidate the cache and delegate for the underlying object
+
+func (o *cachingObject) conditionalSet(isNoop func() bool, set func()) {
+	if fastPath := func() bool {
+		o.lock.RLock()
+		defer o.lock.RUnlock()
+		return isNoop()
+	}(); fastPath {
+		return
+	}
+	o.lock.Lock()
+	defer o.lock.Unlock()
+	if isNoop() {
+		return
+	}
+	o.invalidateCacheLocked()
+	set()
+}
+
+func (o *cachingObject) GetNamespace() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetNamespace()
+}
+func (o *cachingObject) SetNamespace(namespace string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetNamespace() == namespace },
+		func() { o.object.SetNamespace(namespace) },
+	)
+}
+func (o *cachingObject) GetName() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetName()
+}
+func (o *cachingObject) SetName(name string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetName() == name },
+		func() { o.object.SetName(name) },
+	)
+}
+func (o *cachingObject) GetGenerateName() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetGenerateName()
+}
+func (o *cachingObject) SetGenerateName(name string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetGenerateName() == name },
+		func() { o.object.SetGenerateName(name) },
+	)
+}
+func (o *cachingObject) GetUID() types.UID {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetUID()
+}
+func (o *cachingObject) SetUID(uid types.UID) {
+	o.conditionalSet(
+		func() bool { return o.object.GetUID() == uid },
+		func() { o.object.SetUID(uid) },
+	)
+}
+func (o *cachingObject) GetResourceVersion() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetResourceVersion()
+}
+func (o *cachingObject) SetResourceVersion(version string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetResourceVersion() == version },
+		func() { o.object.SetResourceVersion(version) },
+	)
+}
+func (o *cachingObject) GetGeneration() int64 {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetGeneration()
+}
+func (o *cachingObject) SetGeneration(generation int64) {
+	o.conditionalSet(
+		func() bool { return o.object.GetGeneration() == generation },
+		func() { o.object.SetGeneration(generation) },
+	)
+}
+func (o *cachingObject) GetSelfLink() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetSelfLink()
+}
+func (o *cachingObject) SetSelfLink(selfLink string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetSelfLink() == selfLink },
+		func() { o.object.SetSelfLink(selfLink) },
+	)
+}
+func (o *cachingObject) GetCreationTimestamp() metav1.Time {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetCreationTimestamp()
+}
+func (o *cachingObject) SetCreationTimestamp(timestamp metav1.Time) {
+	o.conditionalSet(
+		func() bool { return o.object.GetCreationTimestamp() == timestamp },
+		func() { o.object.SetCreationTimestamp(timestamp) },
+	)
+}
+func (o *cachingObject) GetDeletionTimestamp() *metav1.Time {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetDeletionTimestamp()
+}
+func (o *cachingObject) SetDeletionTimestamp(timestamp *metav1.Time) {
+	o.conditionalSet(
+		func() bool { return o.object.GetDeletionTimestamp() == timestamp },
+		func() { o.object.SetDeletionTimestamp(timestamp) },
+	)
+}
+func (o *cachingObject) GetDeletionGracePeriodSeconds() *int64 {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetDeletionGracePeriodSeconds()
+}
+func (o *cachingObject) SetDeletionGracePeriodSeconds(gracePeriodSeconds *int64) {
+	o.conditionalSet(
+		func() bool { return o.object.GetDeletionGracePeriodSeconds() == gracePeriodSeconds },
+		func() { o.object.SetDeletionGracePeriodSeconds(gracePeriodSeconds) },
+	)
+}
+func (o *cachingObject) GetLabels() map[string]string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetLabels()
+}
+func (o *cachingObject) SetLabels(labels map[string]string) {
+	o.conditionalSet(
+		func() bool { return reflect.DeepEqual(o.object.GetLabels(), labels) },
+		func() { o.object.SetLabels(labels) },
+	)
+}
+func (o *cachingObject) GetAnnotations() map[string]string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetAnnotations()
+}
+func (o *cachingObject) SetAnnotations(annotations map[string]string) {
+	o.conditionalSet(
+		func() bool { return reflect.DeepEqual(o.object.GetAnnotations(), annotations) },
+		func() { o.object.SetAnnotations(annotations) },
+	)
+}
+func (o *cachingObject) GetFinalizers() []string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetFinalizers()
+}
+func (o *cachingObject) SetFinalizers(finalizers []string) {
+	o.conditionalSet(
+		func() bool { return reflect.DeepEqual(o.object.GetFinalizers(), finalizers) },
+		func() { o.object.SetFinalizers(finalizers) },
+	)
+}
+func (o *cachingObject) GetOwnerReferences() []metav1.OwnerReference {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetOwnerReferences()
+}
+func (o *cachingObject) SetOwnerReferences(references []metav1.OwnerReference) {
+	o.conditionalSet(
+		func() bool { return reflect.DeepEqual(o.object.GetOwnerReferences(), references) },
+		func() { o.object.SetOwnerReferences(references) },
+	)
+}
+func (o *cachingObject) GetClusterName() string {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetClusterName()
+}
+func (o *cachingObject) SetClusterName(clusterName string) {
+	o.conditionalSet(
+		func() bool { return o.object.GetClusterName() == clusterName },
+		func() { o.object.SetClusterName(clusterName) },
+	)
+}
+func (o *cachingObject) GetManagedFields() []metav1.ManagedFieldsEntry {
+	o.lock.RLock()
+	defer o.lock.RUnlock()
+	return o.object.GetManagedFields()
+}
+func (o *cachingObject) SetManagedFields(managedFields []metav1.ManagedFieldsEntry) {
+	o.conditionalSet(
+		func() bool { return reflect.DeepEqual(o.object.GetManagedFields(), managedFields) },
+		func() { o.object.SetManagedFields(managedFields) },
+	)
+}

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -40,6 +40,10 @@ const (
 	// before terminating request and returning Timeout error with retry
 	// after suggestion.
 	blockTimeout = 3 * time.Second
+
+	// resourceVersionTooHighRetrySeconds is the seconds before a operation should be retried by the client
+	// after receiving a 'too high resource version' error.
+	resourceVersionTooHighRetrySeconds = 1
 )
 
 // watchCacheEvent is a single "watch event" that is send to users of
@@ -219,7 +223,7 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 		return err
 	}
 
-	watchCacheEvent := &watchCacheEvent{
+	wcEvent := &watchCacheEvent{
 		Type:            event.Type,
 		Object:          elem.Object,
 		ObjLabels:       elem.Labels,
@@ -242,12 +246,12 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 		}
 		if exists {
 			previousElem := previous.(*storeElement)
-			watchCacheEvent.PrevObject = previousElem.Object
-			watchCacheEvent.PrevObjLabels = previousElem.Labels
-			watchCacheEvent.PrevObjFields = previousElem.Fields
+			wcEvent.PrevObject = previousElem.Object
+			wcEvent.PrevObjLabels = previousElem.Labels
+			wcEvent.PrevObjFields = previousElem.Fields
 		}
 
-		w.updateCache(watchCacheEvent)
+		w.updateCache(wcEvent)
 		w.resourceVersion = resourceVersion
 		defer w.cond.Broadcast()
 
@@ -260,7 +264,7 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 	// This is safe as long as there is at most one call to processEvent in flight
 	// at any point in time.
 	if w.eventHandler != nil {
-		w.eventHandler(watchCacheEvent)
+		w.eventHandler(wcEvent)
 	}
 	return nil
 }
@@ -303,8 +307,8 @@ func (w *watchCache) waitUntilFreshAndBlock(resourceVersion uint64, trace *utilt
 	}
 	for w.resourceVersion < resourceVersion {
 		if w.clock.Since(startTime) >= blockTimeout {
-			// Timeout with retry after 1 second.
-			return errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %v, current: %v", resourceVersion, w.resourceVersion), 1)
+			// Request that the client retry after 'resourceVersionTooHighRetrySeconds' seconds.
+			return storage.NewTooLargeResourceVersionError(resourceVersion, w.resourceVersion, resourceVersionTooHighRetrySeconds)
 		}
 		w.cond.Wait()
 	}
@@ -464,7 +468,7 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 		return result, nil
 	}
 	if resourceVersion < oldest-1 {
-		return nil, errors.NewGone(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest-1))
+		return nil, errors.NewResourceExpired(fmt.Sprintf("too old resource version: %d (%d)", resourceVersion, oldest-1))
 	}
 
 	// Binary search the smallest index at which resourceVersion is greater than the given one.

vendor/k8s.io/apiserver/pkg/storage/errors.go

@@ -19,6 +19,8 @@ package storage
 import (
 	"fmt"
 
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 )
 
@@ -168,3 +170,31 @@ func NewInternalError(reason string) InternalError {
 func NewInternalErrorf(format string, a ...interface{}) InternalError {
 	return InternalError{fmt.Sprintf(format, a...)}
 }
+
+var tooLargeResourceVersionCauseMsg = "Too large resource version"
+
+// NewTooLargeResourceVersionError returns a timeout error with the given retrySeconds for a request for
+// a minimum resource version that is larger than the largest currently available resource version for a requested resource.
+func NewTooLargeResourceVersionError(minimumResourceVersion, currentRevision uint64, retrySeconds int) error {
+	err := errors.NewTimeoutError(fmt.Sprintf("Too large resource version: %d, current: %d", minimumResourceVersion, currentRevision), retrySeconds)
+	err.ErrStatus.Details.Causes = []metav1.StatusCause{{Message: tooLargeResourceVersionCauseMsg}}
+	return err
+}
+
+// IsTooLargeResourceVersion returns true if the error is a TooLargeResourceVersion error.
+func IsTooLargeResourceVersion(err error) bool {
+	if !errors.IsTimeout(err) {
+		return false
+	}
+	switch t := err.(type) {
+	case errors.APIStatus:
+		if d := t.Status().Details; d != nil {
+			for _, cause := range d.Causes {
+				if cause.Message == tooLargeResourceVersionCauseMsg {
+					return true
+				}
+			}
+		}
+	}
+	return false
+}

vendor/k8s.io/apiserver/pkg/storage/etcd/OWNERS

@@ -1,27 +0,0 @@
-# See the OWNERS docs at https://go.k8s.io/owners
-
-reviewers:
-- lavalamp
-- smarterclayton
-- wojtek-t
-- deads2k
-- derekwaynecarr
-- caesarxuchao
-- mikedanese
-- liggitt
-- davidopp
-- pmorie
-- luxas
-- janetkuo
-- roberthbailey
-- tallclair
-- timothysc
-- dims
-- hongchaodeng
-- krousey
-- fgrzadkowski
-- resouer
-- pweil-
-- mqliang
-- feihujiang
-- enj

vendor/k8s.io/apiserver/pkg/storage/etcd/metrics/metrics.go

@@ -1,187 +0,0 @@
-/*
-Copyright 2015 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package metrics
-
-import (
-	"sync"
-	"time"
-
-	"github.com/prometheus/client_golang/prometheus"
-)
-
-var (
-	cacheHitCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_hit_total",
-		Help: "Counter of etcd helper cache hits.",
-	}
-	cacheHitCounter      = prometheus.NewCounter(cacheHitCounterOpts)
-	cacheMissCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_miss_total",
-		Help: "Counter of etcd helper cache miss.",
-	}
-	cacheMissCounter      = prometheus.NewCounter(cacheMissCounterOpts)
-	cacheEntryCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_entry_total",
-		Help: "Counter of etcd helper cache entries. This can be different from etcd_helper_cache_miss_count " +
-			"because two concurrent threads can miss the cache and generate the same entry twice.",
-	}
-	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
-	cacheGetLatency   = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Name: "etcd_request_cache_get_duration_seconds",
-			Help: "Latency in seconds of getting an object from etcd cache",
-		},
-	)
-	cacheAddLatency = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Name: "etcd_request_cache_add_duration_seconds",
-			Help: "Latency in seconds of adding an object to etcd cache",
-		},
-	)
-	etcdRequestLatency = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name: "etcd_request_duration_seconds",
-			Help: "Etcd request latency in seconds for each operation and object type.",
-		},
-		[]string{"operation", "type"},
-	)
-	objectCounts = prometheus.NewGaugeVec(
-		prometheus.GaugeOpts{
-			Name: "etcd_object_counts",
-			Help: "Number of stored objects at the time of last check split by kind.",
-		},
-		[]string{"resource"},
-	)
-
-	deprecatedCacheHitCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_hit_count",
-		Help: "(Deprecated) Counter of etcd helper cache hits.",
-	}
-	deprecatedCacheHitCounter      = prometheus.NewCounter(deprecatedCacheHitCounterOpts)
-	deprecatedCacheMissCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_miss_count",
-		Help: "(Deprecated) Counter of etcd helper cache miss.",
-	}
-	deprecatedCacheMissCounter      = prometheus.NewCounter(deprecatedCacheMissCounterOpts)
-	deprecatedCacheEntryCounterOpts = prometheus.CounterOpts{
-		Name: "etcd_helper_cache_entry_count",
-		Help: "(Deprecated) Counter of etcd helper cache entries. This can be different from etcd_helper_cache_miss_count " +
-			"because two concurrent threads can miss the cache and generate the same entry twice.",
-	}
-	deprecatedCacheEntryCounter = prometheus.NewCounter(deprecatedCacheEntryCounterOpts)
-	deprecatedCacheGetLatency   = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_cache_get_latencies_summary",
-			Help: "(Deprecated) Latency in microseconds of getting an object from etcd cache",
-		},
-	)
-	deprecatedCacheAddLatency = prometheus.NewSummary(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_cache_add_latencies_summary",
-			Help: "(Deprecated) Latency in microseconds of adding an object to etcd cache",
-		},
-	)
-	deprecatedEtcdRequestLatenciesSummary = prometheus.NewSummaryVec(
-		prometheus.SummaryOpts{
-			Name: "etcd_request_latencies_summary",
-			Help: "(Deprecated) Etcd request latency summary in microseconds for each operation and object type.",
-		},
-		[]string{"operation", "type"},
-	)
-)
-
-var registerMetrics sync.Once
-
-// Register all metrics.
-func Register() {
-	// Register the metrics.
-	registerMetrics.Do(func() {
-		prometheus.MustRegister(cacheHitCounter)
-		prometheus.MustRegister(cacheMissCounter)
-		prometheus.MustRegister(cacheEntryCounter)
-		prometheus.MustRegister(cacheAddLatency)
-		prometheus.MustRegister(cacheGetLatency)
-		prometheus.MustRegister(etcdRequestLatency)
-		prometheus.MustRegister(objectCounts)
-
-		// TODO(danielqsj): Remove the following metrics, they are deprecated
-		prometheus.MustRegister(deprecatedCacheHitCounter)
-		prometheus.MustRegister(deprecatedCacheMissCounter)
-		prometheus.MustRegister(deprecatedCacheEntryCounter)
-		prometheus.MustRegister(deprecatedCacheAddLatency)
-		prometheus.MustRegister(deprecatedCacheGetLatency)
-		prometheus.MustRegister(deprecatedEtcdRequestLatenciesSummary)
-	})
-}
-
-func UpdateObjectCount(resourcePrefix string, count int64) {
-	objectCounts.WithLabelValues(resourcePrefix).Set(float64(count))
-}
-
-func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
-	etcdRequestLatency.WithLabelValues(verb, resource).Observe(sinceInSeconds(startTime))
-	deprecatedEtcdRequestLatenciesSummary.WithLabelValues(verb, resource).Observe(sinceInMicroseconds(startTime))
-}
-
-func ObserveGetCache(startTime time.Time) {
-	cacheGetLatency.Observe(sinceInSeconds(startTime))
-	deprecatedCacheGetLatency.Observe(sinceInMicroseconds(startTime))
-}
-
-func ObserveAddCache(startTime time.Time) {
-	cacheAddLatency.Observe(sinceInSeconds(startTime))
-	deprecatedCacheAddLatency.Observe(sinceInMicroseconds(startTime))
-}
-
-func ObserveCacheHit() {
-	cacheHitCounter.Inc()
-	deprecatedCacheHitCounter.Inc()
-}
-
-func ObserveCacheMiss() {
-	cacheMissCounter.Inc()
-	deprecatedCacheMissCounter.Inc()
-}
-
-func ObserveNewEntry() {
-	cacheEntryCounter.Inc()
-	deprecatedCacheEntryCounter.Inc()
-}
-
-func Reset() {
-	cacheHitCounter = prometheus.NewCounter(cacheHitCounterOpts)
-	cacheMissCounter = prometheus.NewCounter(cacheMissCounterOpts)
-	cacheEntryCounter = prometheus.NewCounter(cacheEntryCounterOpts)
-	// TODO: Reset cacheAddLatency.
-	// TODO: Reset cacheGetLatency.
-	etcdRequestLatency.Reset()
-
-	deprecatedCacheHitCounter = prometheus.NewCounter(deprecatedCacheHitCounterOpts)
-	deprecatedCacheMissCounter = prometheus.NewCounter(deprecatedCacheMissCounterOpts)
-	deprecatedCacheEntryCounter = prometheus.NewCounter(deprecatedCacheEntryCounterOpts)
-	deprecatedEtcdRequestLatenciesSummary.Reset()
-}
-
-// sinceInMicroseconds gets the time since the specified start in microseconds.
-func sinceInMicroseconds(start time.Time) float64 {
-	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
-}
-
-// sinceInSeconds gets the time since the specified start in seconds.
-func sinceInSeconds(start time.Time) float64 {
-	return time.Since(start).Seconds()
-}

vendor/k8s.io/apiserver/pkg/storage/etcd3/api_object_versioner.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package etcd
+package etcd3
 
 import (
 	"strconv"

vendor/k8s.io/apiserver/pkg/storage/etcd3/compact.go

@@ -22,7 +22,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3"
 	"k8s.io/klog"
 )
 

vendor/k8s.io/apiserver/pkg/storage/etcd3/errors.go

@@ -19,7 +19,7 @@ package etcd3
 import (
 	"k8s.io/apimachinery/pkg/api/errors"
 
-	etcdrpc "github.com/coreos/etcd/etcdserver/api/v3rpc/rpctypes"
+	etcdrpc "go.etcd.io/etcd/etcdserver/api/v3rpc/rpctypes"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 )
 

vendor/k8s.io/apiserver/pkg/storage/etcd3/event.go

@@ -18,8 +18,8 @@ package etcd3
 
 import (
 	"fmt"
-	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/mvcc/mvccpb"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
 )
 
 type event struct {

vendor/k8s.io/apiserver/pkg/storage/etcd3/healthcheck.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package etcd3
+
+import (
+	"encoding/json"
+	"fmt"
+)
+
+// etcdHealth encodes data returned from etcd /healthz handler.
+type etcdHealth struct {
+	// Note this has to be public so the json library can modify it.
+	Health string `json:"health"`
+}
+
+// EtcdHealthCheck decodes data returned from etcd /healthz handler.
+func EtcdHealthCheck(data []byte) error {
+	obj := etcdHealth{}
+	if err := json.Unmarshal(data, &obj); err != nil {
+		return err
+	}
+	if obj.Health != "true" {
+		return fmt.Errorf("Unhealthy status: %s", obj.Health)
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/lease_manager.go

@@ -21,7 +21,7 @@ import (
 	"sync"
 	"time"
 
-	"github.com/coreos/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3"
 )
 
 // leaseManager is used to manage leases requested from etcd. If a new write

vendor/k8s.io/apiserver/pkg/storage/etcd3/logger.go

@@ -19,7 +19,7 @@ package etcd3
 import (
 	"fmt"
 
-	"github.com/coreos/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3"
 	"k8s.io/klog"
 )
 

vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go

@@ -0,0 +1,104 @@
+/*
+Copyright 2015 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"sync"
+	"time"
+
+	compbasemetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	etcdRequestLatency = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Name:           "etcd_request_duration_seconds",
+			Help:           "Etcd request latency in seconds for each operation and object type.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"operation", "type"},
+	)
+	objectCounts = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Name:           "etcd_object_counts",
+			Help:           "Number of stored objects at the time of last check split by kind.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	deprecatedEtcdRequestLatenciesSummary = compbasemetrics.NewSummaryVec(
+		&compbasemetrics.SummaryOpts{
+			Name:              "etcd_request_latencies_summary",
+			Help:              "Etcd request latency summary in microseconds for each operation and object type.",
+			StabilityLevel:    compbasemetrics.ALPHA,
+			DeprecatedVersion: "1.14.0",
+		},
+		[]string{"operation", "type"},
+	)
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	// Register the metrics.
+	registerMetrics.Do(func() {
+		legacyregistry.MustRegister(etcdRequestLatency)
+		legacyregistry.MustRegister(objectCounts)
+
+		// TODO(danielqsj): Remove the following metrics, they are deprecated
+		legacyregistry.MustRegister(deprecatedEtcdRequestLatenciesSummary)
+	})
+}
+
+// UpdateObjectCount sets the etcd_object_counts metric.
+func UpdateObjectCount(resourcePrefix string, count int64) {
+	objectCounts.WithLabelValues(resourcePrefix).Set(float64(count))
+}
+
+// RecordEtcdRequestLatency sets the etcd_request_duration_seconds metrics.
+func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
+	etcdRequestLatency.WithLabelValues(verb, resource).Observe(sinceInSeconds(startTime))
+	deprecatedEtcdRequestLatenciesSummary.WithLabelValues(verb, resource).Observe(sinceInMicroseconds(startTime))
+}
+
+// Reset resets the etcd_request_duration_seconds metric.
+func Reset() {
+	etcdRequestLatency.Reset()
+
+	deprecatedEtcdRequestLatenciesSummary.Reset()
+}
+
+// sinceInMicroseconds gets the time since the specified start in microseconds.
+func sinceInMicroseconds(start time.Time) float64 {
+	return float64(time.Since(start).Nanoseconds() / time.Microsecond.Nanoseconds())
+}
+
+// sinceInSeconds gets the time since the specified start in seconds.
+func sinceInSeconds(start time.Time) float64 {
+	return time.Since(start).Seconds()
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -28,8 +28,7 @@ import (
 	"strings"
 	"time"
 
-	"github.com/coreos/etcd/clientv3"
-	"k8s.io/klog"
+	"go.etcd.io/etcd/clientv3"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -38,10 +37,10 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/storage"
-	"k8s.io/apiserver/pkg/storage/etcd"
-	"k8s.io/apiserver/pkg/storage/etcd/metrics"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/value"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/klog"
 	utiltrace "k8s.io/utils/trace"
 )
 
@@ -89,7 +88,7 @@ func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer val
 }
 
 func newStore(c *clientv3.Client, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
-	versioner := etcd.APIObjectVersioner{}
+	versioner := APIObjectVersioner{}
 	result := &store{
 		client:        c,
 		codec:         codec,
@@ -120,6 +119,9 @@ func (s *store) Get(ctx context.Context, key string, resourceVersion string, out
 	if err != nil {
 		return err
 	}
+	if err = s.ensureMinimumResourceVersion(resourceVersion, uint64(getResp.Header.Revision)); err != nil {
+		return err
+	}
 
 	if len(getResp.Kvs) == 0 {
 		if ignoreNotFound {
@@ -186,7 +188,7 @@ func (s *store) Create(ctx context.Context, key string, obj, out runtime.Object,
 func (s *store) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, validateDeletion storage.ValidateObjectFunc) error {
 	v, err := conversion.EnforcePtr(out)
 	if err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	key = path.Join(s.pathPrefix, key)
 	return s.conditionalDelete(ctx, key, out, v, preconditions, validateDeletion)
@@ -209,7 +211,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 				return err
 			}
 		}
-		if err := validateDeletion(origState.obj); err != nil {
+		if err := validateDeletion(ctx, origState.obj); err != nil {
 			return err
 		}
 		startTime := time.Now()
@@ -237,12 +239,12 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 func (s *store) GuaranteedUpdate(
 	ctx context.Context, key string, out runtime.Object, ignoreNotFound bool,
 	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error {
-	trace := utiltrace.New(fmt.Sprintf("GuaranteedUpdate etcd3: %s", getTypeName(out)))
+	trace := utiltrace.New("GuaranteedUpdate etcd3", utiltrace.Field{"type", getTypeName(out)})
 	defer trace.LogIfLong(500 * time.Millisecond)
 
 	v, err := conversion.EnforcePtr(out)
 	if err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	key = path.Join(s.pathPrefix, key)
 
@@ -377,7 +379,11 @@ func (s *store) GuaranteedUpdate(
 
 // GetToList implements storage.Interface.GetToList.
 func (s *store) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
-	trace := utiltrace.New(fmt.Sprintf("GetToList etcd3: key=%v, resourceVersion=%s, limit: %d, continue: %s", key, resourceVersion, pred.Limit, pred.Continue))
+	trace := utiltrace.New("GetToList etcd3",
+		utiltrace.Field{"key", key},
+		utiltrace.Field{"resourceVersion", resourceVersion},
+		utiltrace.Field{"limit", pred.Limit},
+		utiltrace.Field{"continue", pred.Continue})
 	defer trace.LogIfLong(500 * time.Millisecond)
 	listPtr, err := meta.GetItemsPtr(listObj)
 	if err != nil {
@@ -385,7 +391,7 @@ func (s *store) GetToList(ctx context.Context, key string, resourceVersion strin
 	}
 	v, err := conversion.EnforcePtr(listPtr)
 	if err != nil || v.Kind() != reflect.Slice {
-		panic("need ptr to slice")
+		return fmt.Errorf("need ptr to slice: %v", err)
 	}
 
 	key = path.Join(s.pathPrefix, key)
@@ -395,6 +401,9 @@ func (s *store) GetToList(ctx context.Context, key string, resourceVersion strin
 	if err != nil {
 		return err
 	}
+	if err = s.ensureMinimumResourceVersion(resourceVersion, uint64(getResp.Header.Revision)); err != nil {
+		return err
+	}
 
 	if len(getResp.Kvs) > 0 {
 		data, _, err := s.transformer.TransformFromStorage(getResp.Kvs[0].Value, authenticatedDataString(key))
@@ -482,7 +491,11 @@ func encodeContinue(key, keyPrefix string, resourceVersion int64) (string, error
 
 // List implements storage.Interface.List.
 func (s *store) List(ctx context.Context, key, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
-	trace := utiltrace.New(fmt.Sprintf("List etcd3: key=%v, resourceVersion=%s, limit: %d, continue: %s", key, resourceVersion, pred.Limit, pred.Continue))
+	trace := utiltrace.New("List etcd3",
+		utiltrace.Field{"key", key},
+		utiltrace.Field{"resourceVersion", resourceVersion},
+		utiltrace.Field{"limit", pred.Limit},
+		utiltrace.Field{"continue", pred.Continue})
 	defer trace.LogIfLong(500 * time.Millisecond)
 	listPtr, err := meta.GetItemsPtr(listObj)
 	if err != nil {
@@ -490,7 +503,7 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
 	}
 	v, err := conversion.EnforcePtr(listPtr)
 	if err != nil || v.Kind() != reflect.Slice {
-		panic("need ptr to slice")
+		return fmt.Errorf("need ptr to slice: %v", err)
 	}
 
 	if s.pathPrefix != "" {
@@ -552,17 +565,6 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
 		options = append(options, clientv3.WithRange(rangeEnd))
 
 	default:
-		if len(resourceVersion) > 0 {
-			fromRV, err := s.versioner.ParseResourceVersion(resourceVersion)
-			if err != nil {
-				return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
-			}
-			if fromRV > 0 {
-				options = append(options, clientv3.WithRev(int64(fromRV)))
-			}
-			returnedRV = int64(fromRV)
-		}
-
 		options = append(options, clientv3.WithPrefix())
 	}
 
@@ -577,6 +579,9 @@ func (s *store) List(ctx context.Context, key, resourceVersion string, pred stor
 		if err != nil {
 			return interpretListError(err, len(pred.Continue) > 0, continueKey, keyPrefix)
 		}
+		if err = s.ensureMinimumResourceVersion(resourceVersion, uint64(getResp.Header.Revision)); err != nil {
+			return err
+		}
 		hasMore = getResp.More
 
 		if len(getResp.Kvs) == 0 && getResp.More {
@@ -756,7 +761,9 @@ func (s *store) getStateFromObject(obj runtime.Object) (*objState, error) {
 	if err != nil {
 		return nil, err
 	}
-	s.versioner.UpdateObject(state.obj, uint64(rv))
+	if err := s.versioner.UpdateObject(state.obj, uint64(rv)); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	return state, nil
 }
 
@@ -789,18 +796,38 @@ func (s *store) ttlOpts(ctx context.Context, ttl int64) ([]clientv3.OpOption, er
 	return []clientv3.OpOption{clientv3.WithLease(id)}, nil
 }
 
+// ensureMinimumResourceVersion returns a 'too large resource' version error when the provided minimumResourceVersion is
+// greater than the most recent actualRevision available from storage.
+func (s *store) ensureMinimumResourceVersion(minimumResourceVersion string, actualRevision uint64) error {
+	if minimumResourceVersion == "" {
+		return nil
+	}
+	minimumRV, err := s.versioner.ParseResourceVersion(minimumResourceVersion)
+	if err != nil {
+		return apierrors.NewBadRequest(fmt.Sprintf("invalid resource version: %v", err))
+	}
+	// Enforce the storage.Interface guarantee that the resource version of the returned data
+	// "will be at least 'resourceVersion'".
+	if minimumRV > actualRevision {
+		return storage.NewTooLargeResourceVersionError(minimumRV, actualRevision, 0)
+	}
+	return nil
+}
+
 // decode decodes value of bytes into object. It will also set the object resource version to rev.
 // On success, objPtr would be set to the object.
 func decode(codec runtime.Codec, versioner storage.Versioner, value []byte, objPtr runtime.Object, rev int64) error {
 	if _, err := conversion.EnforcePtr(objPtr); err != nil {
-		panic("unable to convert output object to pointer")
+		return fmt.Errorf("unable to convert output object to pointer: %v", err)
 	}
 	_, _, err := codec.Decode(value, nil, objPtr)
 	if err != nil {
 		return err
 	}
 	// being unable to set the version does not prevent the object from being extracted
-	versioner.UpdateObject(objPtr, uint64(rev))
+	if err := versioner.UpdateObject(objPtr, uint64(rev)); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	return nil
 }
 
@@ -811,7 +838,9 @@ func appendListItem(v reflect.Value, data []byte, rev uint64, pred storage.Selec
 		return err
 	}
 	// being unable to set the version does not prevent the object from being extracted
-	versioner.UpdateObject(obj, rev)
+	if err := versioner.UpdateObject(obj, rev); err != nil {
+		klog.Errorf("failed to update object version: %v", err)
+	}
 	if matched, err := pred.Matches(obj); err == nil && matched {
 		v.Set(reflect.Append(v, reflect.ValueOf(obj).Elem()))
 	}

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -31,7 +31,7 @@ import (
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/value"
 
-	"github.com/coreos/etcd/clientv3"
+	"go.etcd.io/etcd/clientv3"
 	"k8s.io/klog"
 )
 

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -73,16 +73,13 @@ type ResponseMeta struct {
 	ResourceVersion uint64
 }
 
-// MatchValue defines a pair (<index name>, <value for that index>).
-type MatchValue struct {
-	IndexName string
-	Value     string
-}
+// IndexerFunc is a function that for a given object computes
+// <value of an index> for a particular <index>.
+type IndexerFunc func(obj runtime.Object) string
 
-// TriggerPublisherFunc is a function that takes an object, and returns a list of pairs
-// (<index name>, <index value for the given object>) for all indexes known
-// to that function.
-type TriggerPublisherFunc func(obj runtime.Object) []MatchValue
+// IndexerFuncs is a mapping from <index name> to function that
+// for a given object computes <value for that index>.
+type IndexerFuncs map[string]IndexerFunc
 
 // Everything accepts all objects.
 var Everything = SelectionPredicate{
@@ -98,10 +95,10 @@ type UpdateFunc func(input runtime.Object, res ResponseMeta) (output runtime.Obj
 // ValidateObjectFunc is a function to act on a given object. An error may be returned
 // if the hook cannot be completed. The function may NOT transform the provided
 // object.
-type ValidateObjectFunc func(obj runtime.Object) error
+type ValidateObjectFunc func(ctx context.Context, obj runtime.Object) error
 
 // ValidateAllObjectFunc is a "admit everything" instance of ValidateObjectFunc.
-func ValidateAllObjectFunc(obj runtime.Object) error {
+func ValidateAllObjectFunc(ctx context.Context, obj runtime.Object) error {
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/storage/selection_predicate.go

@@ -124,20 +124,6 @@ func (s *SelectionPredicate) MatchesSingle() (string, bool) {
 	return "", false
 }
 
-// For any index defined by IndexFields, if a matcher can match only (a subset)
-// of objects that return <value> for a given index, a pair (<index name>, <value>)
-// wil be returned.
-// TODO: Consider supporting also labels.
-func (s *SelectionPredicate) MatcherIndex() []MatchValue {
-	var result []MatchValue
-	for _, field := range s.IndexFields {
-		if value, ok := s.Field.RequiresExactMatch(field); ok {
-			result = append(result, MatchValue{IndexName: field, Value: value})
-		}
-	}
-	return result
-}
-
 // Empty returns true if the predicate performs no filtering.
 func (s *SelectionPredicate) Empty() bool {
 	return s.Label.Empty() && s.Field.Empty()

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -20,6 +20,7 @@ import (
 	"time"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apiserver/pkg/server/egressselector"
 	"k8s.io/apiserver/pkg/storage/value"
 )
 
@@ -35,9 +36,11 @@ type TransportConfig struct {
 	// ServerList is the list of storage servers to connect with.
 	ServerList []string
 	// TLS credentials
-	KeyFile  string
-	CertFile string
-	CAFile   string
+	KeyFile       string
+	CertFile      string
+	TrustedCAFile string
+	// function to determine the egress dialer. (i.e. konnectivity server dialer)
+	EgressLookup egressselector.Lookup
 }
 
 // Config is configuration for creating a storage backend.

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -19,21 +19,26 @@ package factory
 import (
 	"context"
 	"fmt"
+	"net"
+	"net/url"
 	"path"
 	"sync"
 	"sync/atomic"
 	"time"
 
-	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/pkg/transport"
 	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/pkg/transport"
 	"google.golang.org/grpc"
 
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/egressselector"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	"k8s.io/apiserver/pkg/storage/value"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 // The short keepalive timeout and interval have been chosen to aggressively
@@ -46,6 +51,14 @@ const keepaliveTimeout = 10 * time.Second
 // on heavily loaded arm64 CPUs (issue #64649)
 const dialTimeout = 20 * time.Second
 
+func init() {
+	// grpcprom auto-registers (via an init function) their client metrics, since we are opting out of
+	// using the global prometheus registry and using our own wrapped global registry,
+	// we need to explicitly register these metrics to our global registry here.
+	// For reference: https://github.com/kubernetes/kubernetes/pull/81387
+	legacyregistry.RawMustRegister(grpcprom.DefaultClientMetrics)
+}
+
 func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 	// constructing the etcd v3 client blocks and times out if etcd is not available.
 	// retry in a loop in the background until we successfully create the client, storing the client or error encountered
@@ -73,8 +86,8 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 		client := clientValue.Load().(*clientv3.Client)
 		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
 		defer cancel()
-		// See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118
-		_, err := client.Get(ctx, path.Join(c.Prefix, "health"))
+		// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
+		_, err := client.Get(ctx, path.Join("/", c.Prefix, "health"))
 		if err == nil {
 			return nil
 		}
@@ -84,9 +97,9 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 
 func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) {
 	tlsInfo := transport.TLSInfo{
-		CertFile: c.CertFile,
-		KeyFile:  c.KeyFile,
-		CAFile:   c.CAFile,
+		CertFile:      c.CertFile,
+		KeyFile:       c.KeyFile,
+		TrustedCAFile: c.TrustedCAFile,
 	}
 	tlsConfig, err := tlsInfo.ClientConfig()
 	if err != nil {
@@ -94,19 +107,39 @@ func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error)
 	}
 	// NOTE: Client relies on nil tlsConfig
 	// for non-secure connections, update the implicit variable
-	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.CAFile) == 0 {
+	if len(c.CertFile) == 0 && len(c.KeyFile) == 0 && len(c.TrustedCAFile) == 0 {
 		tlsConfig = nil
 	}
+	networkContext := egressselector.Etcd.AsNetworkContext()
+	var egressDialer utilnet.DialFunc
+	if c.EgressLookup != nil {
+		egressDialer, err = c.EgressLookup(networkContext)
+		if err != nil {
+			return nil, err
+		}
+	}
+	dialOptions := []grpc.DialOption{
+		grpc.WithBlock(), // block until the underlying connection is up
+		grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
+		grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
+	}
+	if egressDialer != nil {
+		dialer := func(ctx context.Context, addr string) (net.Conn, error) {
+			u, err := url.Parse(addr)
+			if err != nil {
+				return nil, err
+			}
+			return egressDialer(ctx, "tcp", u.Host)
+		}
+		dialOptions = append(dialOptions, grpc.WithContextDialer(dialer))
+	}
 	cfg := clientv3.Config{
 		DialTimeout:          dialTimeout,
 		DialKeepAliveTime:    keepaliveTime,
 		DialKeepAliveTimeout: keepaliveTimeout,
-		DialOptions: []grpc.DialOption{
-			grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
-			grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
-		},
-		Endpoints: c.ServerList,
-		TLS:       tlsConfig,
+		DialOptions:          dialOptions,
+		Endpoints:            c.ServerList,
+		TLS:                  tlsConfig,
 	}
 
 	return clientv3.New(cfg)

vendor/k8s.io/apiserver/pkg/storage/util.go

@@ -39,14 +39,6 @@ func EverythingFunc(runtime.Object) bool {
 	return true
 }
 
-func NoTriggerFunc() []MatchValue {
-	return nil
-}
-
-func NoTriggerPublisher(runtime.Object) []MatchValue {
-	return nil
-}
-
 func NamespaceKeyFunc(prefix string, obj runtime.Object) (string, error) {
 	meta, err := meta.Accessor(obj)
 	if err != nil {

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/aes/aes.go

@@ -0,0 +1,152 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package aes transforms values for storage at rest using AES-GCM.
+package aes
+
+import (
+	"bytes"
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"errors"
+	"fmt"
+	"io"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// gcm implements AEAD encryption of the provided values given a cipher.Block algorithm.
+// The authenticated data provided as part of the value.Context method must match when the same
+// value is set to and loaded from storage. In order to ensure that values cannot be copied by
+// an attacker from a location under their control, use characteristics of the storage location
+// (such as the etcd key) as part of the authenticated data.
+//
+// Because this mode requires a generated IV and IV reuse is a known weakness of AES-GCM, keys
+// must be rotated before a birthday attack becomes feasible. NIST SP 800-38D
+// (http://csrc.nist.gov/publications/nistpubs/800-38D/SP-800-38D.pdf) recommends using the same
+// key with random 96-bit nonces (the default nonce length) no more than 2^32 times, and
+// therefore transformers using this implementation *must* ensure they allow for frequent key
+// rotation. Future work should include investigation of AES-GCM-SIV as an alternative to
+// random nonces.
+type gcm struct {
+	block cipher.Block
+}
+
+// NewGCMTransformer takes the given block cipher and performs encryption and decryption on the given
+// data.
+func NewGCMTransformer(block cipher.Block) value.Transformer {
+	return &gcm{block: block}
+}
+
+func (t *gcm) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	aead, err := cipher.NewGCM(t.block)
+	if err != nil {
+		return nil, false, err
+	}
+	nonceSize := aead.NonceSize()
+	if len(data) < nonceSize {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	result, err := aead.Open(nil, data[:nonceSize], data[nonceSize:], context.AuthenticatedData())
+	return result, false, err
+}
+
+func (t *gcm) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	aead, err := cipher.NewGCM(t.block)
+	if err != nil {
+		return nil, err
+	}
+	nonceSize := aead.NonceSize()
+	result := make([]byte, nonceSize+aead.Overhead()+len(data))
+	n, err := rand.Read(result[:nonceSize])
+	if err != nil {
+		return nil, err
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	cipherText := aead.Seal(result[nonceSize:nonceSize], result[:nonceSize], data, context.AuthenticatedData())
+	return result[:nonceSize+len(cipherText)], nil
+}
+
+// cbc implements encryption at rest of the provided values given a cipher.Block algorithm.
+type cbc struct {
+	block cipher.Block
+}
+
+// NewCBCTransformer takes the given block cipher and performs encryption and decryption on the given
+// data.
+func NewCBCTransformer(block cipher.Block) value.Transformer {
+	return &cbc{block: block}
+}
+
+var (
+	errInvalidBlockSize    = fmt.Errorf("the stored data is not a multiple of the block size")
+	errInvalidPKCS7Data    = errors.New("invalid PKCS7 data (empty or not padded)")
+	errInvalidPKCS7Padding = errors.New("invalid padding on input")
+)
+
+func (t *cbc) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	blockSize := aes.BlockSize
+	if len(data) < blockSize {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	iv := data[:blockSize]
+	data = data[blockSize:]
+
+	if len(data)%blockSize != 0 {
+		return nil, false, errInvalidBlockSize
+	}
+
+	result := make([]byte, len(data))
+	copy(result, data)
+	mode := cipher.NewCBCDecrypter(t.block, iv)
+	mode.CryptBlocks(result, result)
+
+	// remove and verify PKCS#7 padding for CBC
+	c := result[len(result)-1]
+	paddingSize := int(c)
+	size := len(result) - paddingSize
+	if paddingSize == 0 || paddingSize > len(result) {
+		return nil, false, errInvalidPKCS7Data
+	}
+	for i := 0; i < paddingSize; i++ {
+		if result[size+i] != c {
+			return nil, false, errInvalidPKCS7Padding
+		}
+	}
+
+	return result[:size], false, nil
+}
+
+func (t *cbc) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	blockSize := aes.BlockSize
+	paddingSize := blockSize - (len(data) % blockSize)
+	result := make([]byte, blockSize+len(data)+paddingSize)
+	iv := result[:blockSize]
+	if _, err := io.ReadFull(rand.Reader, iv); err != nil {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	copy(result[blockSize:], data)
+
+	// add PKCS#7 padding for CBC
+	copy(result[blockSize+len(data):], bytes.Repeat([]byte{byte(paddingSize)}, paddingSize))
+
+	mode := cipher.NewCBCEncrypter(t.block, iv)
+	mode.CryptBlocks(result[blockSize:], result[blockSize:])
+	return result, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/envelope.go

@@ -0,0 +1,180 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package envelope transforms values for storage at rest using a Envelope provider
+package envelope
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/base64"
+	"fmt"
+	"time"
+
+	"k8s.io/apiserver/pkg/storage/value"
+
+	lru "github.com/hashicorp/golang-lru"
+	"golang.org/x/crypto/cryptobyte"
+)
+
+// defaultCacheSize is the number of decrypted DEKs which would be cached by the transformer.
+const defaultCacheSize = 1000
+
+func init() {
+	value.RegisterMetrics()
+}
+
+// Service allows encrypting and decrypting data using an external Key Management Service.
+type Service interface {
+	// Decrypt a given bytearray to obtain the original data as bytes.
+	Decrypt(data []byte) ([]byte, error)
+	// Encrypt bytes to a ciphertext.
+	Encrypt(data []byte) ([]byte, error)
+}
+
+type envelopeTransformer struct {
+	envelopeService Service
+
+	// transformers is a thread-safe LRU cache which caches decrypted DEKs indexed by their encrypted form.
+	transformers *lru.Cache
+
+	// baseTransformerFunc creates a new transformer for encrypting the data with the DEK.
+	baseTransformerFunc func(cipher.Block) value.Transformer
+}
+
+// NewEnvelopeTransformer returns a transformer which implements a KEK-DEK based envelope encryption scheme.
+// It uses envelopeService to encrypt and decrypt DEKs. Respective DEKs (in encrypted form) are prepended to
+// the data items they encrypt. A cache (of size cacheSize) is maintained to store the most recently
+// used decrypted DEKs in memory.
+func NewEnvelopeTransformer(envelopeService Service, cacheSize int, baseTransformerFunc func(cipher.Block) value.Transformer) (value.Transformer, error) {
+	if cacheSize == 0 {
+		cacheSize = defaultCacheSize
+	}
+	cache, err := lru.New(cacheSize)
+	if err != nil {
+		return nil, err
+	}
+	return &envelopeTransformer{
+		envelopeService:     envelopeService,
+		transformers:        cache,
+		baseTransformerFunc: baseTransformerFunc,
+	}, nil
+}
+
+// TransformFromStorage decrypts data encrypted by this transformer using envelope encryption.
+func (t *envelopeTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	// Read the 16 bit length-of-DEK encoded at the start of the encrypted DEK. 16 bits can
+	// represent a maximum key length of 65536 bytes. We are using a 256 bit key, whose
+	// length cannot fit in 8 bits (1 byte). Thus, we use 16 bits (2 bytes) to store the length.
+	var encKey cryptobyte.String
+	s := cryptobyte.String(data)
+	if ok := s.ReadUint16LengthPrefixed(&encKey); !ok {
+		return nil, false, fmt.Errorf("invalid data encountered by envelope transformer: failed to read uint16 length prefixed data")
+	}
+
+	encData := []byte(s)
+
+	// Look up the decrypted DEK from cache or Envelope.
+	transformer := t.getTransformer(encKey)
+	if transformer == nil {
+		value.RecordCacheMiss()
+		key, err := t.envelopeService.Decrypt(encKey)
+		if err != nil {
+			// Do NOT wrap this err using fmt.Errorf() or similar functions
+			// because this gRPC status error has useful error code when
+			// record the metric.
+			return nil, false, err
+		}
+		transformer, err = t.addTransformer(encKey, key)
+		if err != nil {
+			return nil, false, err
+		}
+	}
+
+	return transformer.TransformFromStorage(encData, context)
+}
+
+// TransformToStorage encrypts data to be written to disk using envelope encryption.
+func (t *envelopeTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	newKey, err := generateKey(32)
+	if err != nil {
+		return nil, err
+	}
+
+	encKey, err := t.envelopeService.Encrypt(newKey)
+	if err != nil {
+		// Do NOT wrap this err using fmt.Errorf() or similar functions
+		// because this gRPC status error has useful error code when
+		// record the metric.
+		return nil, err
+	}
+
+	transformer, err := t.addTransformer(encKey, newKey)
+	if err != nil {
+		return nil, err
+	}
+
+	result, err := transformer.TransformToStorage(data, context)
+	if err != nil {
+		return nil, err
+	}
+	// Append the length of the encrypted DEK as the first 2 bytes.
+	b := cryptobyte.NewBuilder(nil)
+	b.AddUint16LengthPrefixed(func(b *cryptobyte.Builder) {
+		b.AddBytes([]byte(encKey))
+	})
+	b.AddBytes(result)
+
+	return b.Bytes()
+}
+
+var _ value.Transformer = &envelopeTransformer{}
+
+// addTransformer inserts a new transformer to the Envelope cache of DEKs for future reads.
+func (t *envelopeTransformer) addTransformer(encKey []byte, key []byte) (value.Transformer, error) {
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return nil, err
+	}
+	transformer := t.baseTransformerFunc(block)
+	// Use base64 of encKey as the key into the cache because hashicorp/golang-lru
+	// cannot hash []uint8.
+	t.transformers.Add(base64.StdEncoding.EncodeToString(encKey), transformer)
+	return transformer, nil
+}
+
+// getTransformer fetches the transformer corresponding to encKey from cache, if it exists.
+func (t *envelopeTransformer) getTransformer(encKey []byte) value.Transformer {
+	_transformer, found := t.transformers.Get(base64.StdEncoding.EncodeToString(encKey))
+	if found {
+		return _transformer.(value.Transformer)
+	}
+	return nil
+}
+
+// generateKey generates a random key using system randomness.
+func generateKey(length int) (key []byte, err error) {
+	defer func(start time.Time) {
+		value.RecordDataKeyGeneration(start, err)
+	}(time.Now())
+	key = make([]byte, length)
+	if _, err = rand.Read(key); err != nil {
+		return nil, err
+	}
+
+	return key, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/grpc_service.go

@@ -0,0 +1,181 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package envelope transforms values for storage at rest using a Envelope provider
+package envelope
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"net/url"
+	"strings"
+	"sync"
+	"time"
+
+	"k8s.io/klog"
+
+	"google.golang.org/grpc"
+
+	kmsapi "k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1"
+)
+
+const (
+	// Now only supported unix domain socket.
+	unixProtocol = "unix"
+
+	// Current version for the protocol interface definition.
+	kmsapiVersion = "v1beta1"
+
+	versionErrorf = "KMS provider api version %s is not supported, only %s is supported now"
+)
+
+// The gRPC implementation for envelope.Service.
+type gRPCService struct {
+	kmsClient      kmsapi.KeyManagementServiceClient
+	connection     *grpc.ClientConn
+	callTimeout    time.Duration
+	mux            sync.RWMutex
+	versionChecked bool
+}
+
+// NewGRPCService returns an envelope.Service which use gRPC to communicate the remote KMS provider.
+func NewGRPCService(endpoint string, callTimeout time.Duration) (Service, error) {
+	klog.V(4).Infof("Configure KMS provider with endpoint: %s", endpoint)
+
+	addr, err := parseEndpoint(endpoint)
+	if err != nil {
+		return nil, err
+	}
+
+	s := &gRPCService{callTimeout: callTimeout}
+	s.connection, err = grpc.Dial(
+		addr,
+		grpc.WithInsecure(),
+		grpc.WithUnaryInterceptor(s.interceptor),
+		grpc.WithDefaultCallOptions(grpc.WaitForReady(true)),
+		grpc.WithContextDialer(
+			func(context.Context, string) (net.Conn, error) {
+				// Ignoring addr and timeout arguments:
+				// addr - comes from the closure
+				c, err := net.DialUnix(unixProtocol, nil, &net.UnixAddr{Name: addr})
+				if err != nil {
+					klog.Errorf("failed to create connection to unix socket: %s, error: %v", addr, err)
+				} else {
+					klog.V(4).Infof("Successfully dialed Unix socket %v", addr)
+				}
+				return c, err
+			}))
+
+	if err != nil {
+		return nil, fmt.Errorf("failed to create connection to %s, error: %v", endpoint, err)
+	}
+
+	s.kmsClient = kmsapi.NewKeyManagementServiceClient(s.connection)
+	return s, nil
+}
+
+// Parse the endpoint to extract schema, host or path.
+func parseEndpoint(endpoint string) (string, error) {
+	if len(endpoint) == 0 {
+		return "", fmt.Errorf("remote KMS provider can't use empty string as endpoint")
+	}
+
+	u, err := url.Parse(endpoint)
+	if err != nil {
+		return "", fmt.Errorf("invalid endpoint %q for remote KMS provider, error: %v", endpoint, err)
+	}
+
+	if u.Scheme != unixProtocol {
+		return "", fmt.Errorf("unsupported scheme %q for remote KMS provider", u.Scheme)
+	}
+
+	// Linux abstract namespace socket - no physical file required
+	// Warning: Linux Abstract sockets have not concept of ACL (unlike traditional file based sockets).
+	// However, Linux Abstract sockets are subject to Linux networking namespace, so will only be accessible to
+	// containers within the same pod (unless host networking is used).
+	if strings.HasPrefix(u.Path, "/@") {
+		return strings.TrimPrefix(u.Path, "/"), nil
+	}
+
+	return u.Path, nil
+}
+
+func (g *gRPCService) checkAPIVersion(ctx context.Context) error {
+	g.mux.Lock()
+	defer g.mux.Unlock()
+
+	if g.versionChecked {
+		return nil
+	}
+
+	request := &kmsapi.VersionRequest{Version: kmsapiVersion}
+	response, err := g.kmsClient.Version(ctx, request)
+	if err != nil {
+		return fmt.Errorf("failed get version from remote KMS provider: %v", err)
+	}
+	if response.Version != kmsapiVersion {
+		return fmt.Errorf(versionErrorf, response.Version, kmsapiVersion)
+	}
+	g.versionChecked = true
+
+	klog.V(4).Infof("Version of KMS provider is %s", response.Version)
+	return nil
+}
+
+// Decrypt a given data string to obtain the original byte data.
+func (g *gRPCService) Decrypt(cipher []byte) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), g.callTimeout)
+	defer cancel()
+
+	request := &kmsapi.DecryptRequest{Cipher: cipher, Version: kmsapiVersion}
+	response, err := g.kmsClient.Decrypt(ctx, request)
+	if err != nil {
+		return nil, err
+	}
+	return response.Plain, nil
+}
+
+// Encrypt bytes to a string ciphertext.
+func (g *gRPCService) Encrypt(plain []byte) ([]byte, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), g.callTimeout)
+	defer cancel()
+
+	request := &kmsapi.EncryptRequest{Plain: plain, Version: kmsapiVersion}
+	response, err := g.kmsClient.Encrypt(ctx, request)
+	if err != nil {
+		return nil, err
+	}
+	return response.Cipher, nil
+}
+
+func (g *gRPCService) interceptor(
+	ctx context.Context,
+	method string,
+	req interface{},
+	reply interface{},
+	cc *grpc.ClientConn,
+	invoker grpc.UnaryInvoker,
+	opts ...grpc.CallOption,
+) error {
+	if !kmsapi.IsVersionCheckMethod(method) {
+		if err := g.checkAPIVersion(ctx); err != nil {
+			return err
+		}
+	}
+
+	return invoker(ctx, method, req, reply, cc, opts...)
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/service.pb.go

@@ -0,0 +1,502 @@
+/*
+Copyright The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by protoc-gen-gogo. DO NOT EDIT.
+// source: service.proto
+
+package v1beta1
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/gogo/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	codes "google.golang.org/grpc/codes"
+	status "google.golang.org/grpc/status"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.GoGoProtoPackageIsVersion2 // please upgrade the proto package
+
+type VersionRequest struct {
+	// Version of the KMS plugin API.
+	Version              string   `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VersionRequest) Reset()         { *m = VersionRequest{} }
+func (m *VersionRequest) String() string { return proto.CompactTextString(m) }
+func (*VersionRequest) ProtoMessage()    {}
+func (*VersionRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{0}
+}
+func (m *VersionRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_VersionRequest.Unmarshal(m, b)
+}
+func (m *VersionRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_VersionRequest.Marshal(b, m, deterministic)
+}
+func (m *VersionRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VersionRequest.Merge(m, src)
+}
+func (m *VersionRequest) XXX_Size() int {
+	return xxx_messageInfo_VersionRequest.Size(m)
+}
+func (m *VersionRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_VersionRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VersionRequest proto.InternalMessageInfo
+
+func (m *VersionRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+type VersionResponse struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// Name of the KMS provider.
+	RuntimeName string `protobuf:"bytes,2,opt,name=runtime_name,json=runtimeName,proto3" json:"runtime_name,omitempty"`
+	// Version of the KMS provider. The string must be semver-compatible.
+	RuntimeVersion       string   `protobuf:"bytes,3,opt,name=runtime_version,json=runtimeVersion,proto3" json:"runtime_version,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *VersionResponse) Reset()         { *m = VersionResponse{} }
+func (m *VersionResponse) String() string { return proto.CompactTextString(m) }
+func (*VersionResponse) ProtoMessage()    {}
+func (*VersionResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{1}
+}
+func (m *VersionResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_VersionResponse.Unmarshal(m, b)
+}
+func (m *VersionResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_VersionResponse.Marshal(b, m, deterministic)
+}
+func (m *VersionResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_VersionResponse.Merge(m, src)
+}
+func (m *VersionResponse) XXX_Size() int {
+	return xxx_messageInfo_VersionResponse.Size(m)
+}
+func (m *VersionResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_VersionResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_VersionResponse proto.InternalMessageInfo
+
+func (m *VersionResponse) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *VersionResponse) GetRuntimeName() string {
+	if m != nil {
+		return m.RuntimeName
+	}
+	return ""
+}
+
+func (m *VersionResponse) GetRuntimeVersion() string {
+	if m != nil {
+		return m.RuntimeVersion
+	}
+	return ""
+}
+
+type DecryptRequest struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// The data to be decrypted.
+	Cipher               []byte   `protobuf:"bytes,2,opt,name=cipher,proto3" json:"cipher,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DecryptRequest) Reset()         { *m = DecryptRequest{} }
+func (m *DecryptRequest) String() string { return proto.CompactTextString(m) }
+func (*DecryptRequest) ProtoMessage()    {}
+func (*DecryptRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{2}
+}
+func (m *DecryptRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DecryptRequest.Unmarshal(m, b)
+}
+func (m *DecryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DecryptRequest.Marshal(b, m, deterministic)
+}
+func (m *DecryptRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DecryptRequest.Merge(m, src)
+}
+func (m *DecryptRequest) XXX_Size() int {
+	return xxx_messageInfo_DecryptRequest.Size(m)
+}
+func (m *DecryptRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_DecryptRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DecryptRequest proto.InternalMessageInfo
+
+func (m *DecryptRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *DecryptRequest) GetCipher() []byte {
+	if m != nil {
+		return m.Cipher
+	}
+	return nil
+}
+
+type DecryptResponse struct {
+	// The decrypted data.
+	Plain                []byte   `protobuf:"bytes,1,opt,name=plain,proto3" json:"plain,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DecryptResponse) Reset()         { *m = DecryptResponse{} }
+func (m *DecryptResponse) String() string { return proto.CompactTextString(m) }
+func (*DecryptResponse) ProtoMessage()    {}
+func (*DecryptResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{3}
+}
+func (m *DecryptResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DecryptResponse.Unmarshal(m, b)
+}
+func (m *DecryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DecryptResponse.Marshal(b, m, deterministic)
+}
+func (m *DecryptResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DecryptResponse.Merge(m, src)
+}
+func (m *DecryptResponse) XXX_Size() int {
+	return xxx_messageInfo_DecryptResponse.Size(m)
+}
+func (m *DecryptResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_DecryptResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DecryptResponse proto.InternalMessageInfo
+
+func (m *DecryptResponse) GetPlain() []byte {
+	if m != nil {
+		return m.Plain
+	}
+	return nil
+}
+
+type EncryptRequest struct {
+	// Version of the KMS plugin API.
+	Version string `protobuf:"bytes,1,opt,name=version,proto3" json:"version,omitempty"`
+	// The data to be encrypted.
+	Plain                []byte   `protobuf:"bytes,2,opt,name=plain,proto3" json:"plain,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *EncryptRequest) Reset()         { *m = EncryptRequest{} }
+func (m *EncryptRequest) String() string { return proto.CompactTextString(m) }
+func (*EncryptRequest) ProtoMessage()    {}
+func (*EncryptRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{4}
+}
+func (m *EncryptRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EncryptRequest.Unmarshal(m, b)
+}
+func (m *EncryptRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EncryptRequest.Marshal(b, m, deterministic)
+}
+func (m *EncryptRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EncryptRequest.Merge(m, src)
+}
+func (m *EncryptRequest) XXX_Size() int {
+	return xxx_messageInfo_EncryptRequest.Size(m)
+}
+func (m *EncryptRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_EncryptRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EncryptRequest proto.InternalMessageInfo
+
+func (m *EncryptRequest) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *EncryptRequest) GetPlain() []byte {
+	if m != nil {
+		return m.Plain
+	}
+	return nil
+}
+
+type EncryptResponse struct {
+	// The encrypted data.
+	Cipher               []byte   `protobuf:"bytes,1,opt,name=cipher,proto3" json:"cipher,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *EncryptResponse) Reset()         { *m = EncryptResponse{} }
+func (m *EncryptResponse) String() string { return proto.CompactTextString(m) }
+func (*EncryptResponse) ProtoMessage()    {}
+func (*EncryptResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_a0b84a42fa06f626, []int{5}
+}
+func (m *EncryptResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_EncryptResponse.Unmarshal(m, b)
+}
+func (m *EncryptResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_EncryptResponse.Marshal(b, m, deterministic)
+}
+func (m *EncryptResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_EncryptResponse.Merge(m, src)
+}
+func (m *EncryptResponse) XXX_Size() int {
+	return xxx_messageInfo_EncryptResponse.Size(m)
+}
+func (m *EncryptResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_EncryptResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_EncryptResponse proto.InternalMessageInfo
+
+func (m *EncryptResponse) GetCipher() []byte {
+	if m != nil {
+		return m.Cipher
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*VersionRequest)(nil), "v1beta1.VersionRequest")
+	proto.RegisterType((*VersionResponse)(nil), "v1beta1.VersionResponse")
+	proto.RegisterType((*DecryptRequest)(nil), "v1beta1.DecryptRequest")
+	proto.RegisterType((*DecryptResponse)(nil), "v1beta1.DecryptResponse")
+	proto.RegisterType((*EncryptRequest)(nil), "v1beta1.EncryptRequest")
+	proto.RegisterType((*EncryptResponse)(nil), "v1beta1.EncryptResponse")
+}
+
+func init() { proto.RegisterFile("service.proto", fileDescriptor_a0b84a42fa06f626) }
+
+var fileDescriptor_a0b84a42fa06f626 = []byte{
+	// 287 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x8c, 0x52, 0xcd, 0x4a, 0xc4, 0x30,
+	0x10, 0xde, 0xae, 0xb8, 0xc5, 0xb1, 0xb6, 0x10, 0x16, 0x2d, 0x9e, 0x34, 0x97, 0x55, 0x0f, 0x85,
+	0xd5, 0xbb, 0x88, 0xe8, 0x49, 0xf4, 0x50, 0xc1, 0xab, 0x64, 0xcb, 0xa0, 0x05, 0x9b, 0xc6, 0x24,
+	0x5b, 0xd9, 0x17, 0xf5, 0x79, 0xc4, 0x66, 0x5a, 0xd3, 0x15, 0x71, 0x8f, 0x33, 0x99, 0xef, 0x6f,
+	0x26, 0xb0, 0x67, 0x50, 0x37, 0x65, 0x81, 0x99, 0xd2, 0xb5, 0xad, 0x59, 0xd8, 0xcc, 0x17, 0x68,
+	0xc5, 0x9c, 0x9f, 0x41, 0xfc, 0x84, 0xda, 0x94, 0xb5, 0xcc, 0xf1, 0x7d, 0x89, 0xc6, 0xb2, 0x14,
+	0xc2, 0xc6, 0x75, 0xd2, 0xe0, 0x28, 0x38, 0xd9, 0xc9, 0xbb, 0x92, 0x7f, 0x40, 0xd2, 0xcf, 0x1a,
+	0x55, 0x4b, 0x83, 0x7f, 0x0f, 0xb3, 0x63, 0x88, 0xf4, 0x52, 0xda, 0xb2, 0xc2, 0x67, 0x29, 0x2a,
+	0x4c, 0xc7, 0xed, 0xf3, 0x2e, 0xf5, 0x1e, 0x44, 0x85, 0x6c, 0x06, 0x49, 0x37, 0xd2, 0x91, 0x6c,
+	0xb5, 0x53, 0x31, 0xb5, 0x49, 0x8d, 0x5f, 0x43, 0x7c, 0x83, 0x85, 0x5e, 0x29, 0xfb, 0xaf, 0x49,
+	0xb6, 0x0f, 0x93, 0xa2, 0x54, 0xaf, 0xa8, 0x5b, 0xc5, 0x28, 0xa7, 0x8a, 0xcf, 0x20, 0xe9, 0x39,
+	0xc8, 0xfc, 0x14, 0xb6, 0xd5, 0x9b, 0x28, 0x1d, 0x45, 0x94, 0xbb, 0x82, 0x5f, 0x41, 0x7c, 0x2b,
+	0x37, 0x14, 0xeb, 0x19, 0xc6, 0x3e, 0xc3, 0x29, 0x24, 0x3d, 0x03, 0x49, 0xfd, 0xb8, 0x0a, 0x7c,
+	0x57, 0xe7, 0x9f, 0x01, 0x4c, 0xef, 0x70, 0x75, 0x2f, 0xa4, 0x78, 0xc1, 0x0a, 0xa5, 0x7d, 0x74,
+	0x67, 0x62, 0x97, 0x10, 0x52, 0x7a, 0x76, 0x90, 0xd1, 0xb1, 0xb2, 0xe1, 0xa5, 0x0e, 0xd3, 0xdf,
+	0x0f, 0x4e, 0x8e, 0x8f, 0xbe, 0xf1, 0x14, 0xd7, 0xc3, 0x0f, 0x97, 0xe8, 0xe1, 0xd7, 0x36, 0xe3,
+	0xf0, 0x94, 0xc1, 0xc3, 0x0f, 0xf7, 0xe2, 0xe1, 0xd7, 0xe2, 0xf2, 0xd1, 0x62, 0xd2, 0xfe, 0xb3,
+	0x8b, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x33, 0x8d, 0x09, 0xe1, 0x78, 0x02, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// KeyManagementServiceClient is the client API for KeyManagementService service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type KeyManagementServiceClient interface {
+	// Version returns the runtime name and runtime version of the KMS provider.
+	Version(ctx context.Context, in *VersionRequest, opts ...grpc.CallOption) (*VersionResponse, error)
+	// Execute decryption operation in KMS provider.
+	Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error)
+	// Execute encryption operation in KMS provider.
+	Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error)
+}
+
+type keyManagementServiceClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewKeyManagementServiceClient(cc *grpc.ClientConn) KeyManagementServiceClient {
+	return &keyManagementServiceClient{cc}
+}
+
+func (c *keyManagementServiceClient) Version(ctx context.Context, in *VersionRequest, opts ...grpc.CallOption) (*VersionResponse, error) {
+	out := new(VersionResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Version", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *keyManagementServiceClient) Decrypt(ctx context.Context, in *DecryptRequest, opts ...grpc.CallOption) (*DecryptResponse, error) {
+	out := new(DecryptResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Decrypt", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *keyManagementServiceClient) Encrypt(ctx context.Context, in *EncryptRequest, opts ...grpc.CallOption) (*EncryptResponse, error) {
+	out := new(EncryptResponse)
+	err := c.cc.Invoke(ctx, "/v1beta1.KeyManagementService/Encrypt", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// KeyManagementServiceServer is the server API for KeyManagementService service.
+type KeyManagementServiceServer interface {
+	// Version returns the runtime name and runtime version of the KMS provider.
+	Version(context.Context, *VersionRequest) (*VersionResponse, error)
+	// Execute decryption operation in KMS provider.
+	Decrypt(context.Context, *DecryptRequest) (*DecryptResponse, error)
+	// Execute encryption operation in KMS provider.
+	Encrypt(context.Context, *EncryptRequest) (*EncryptResponse, error)
+}
+
+// UnimplementedKeyManagementServiceServer can be embedded to have forward compatible implementations.
+type UnimplementedKeyManagementServiceServer struct {
+}
+
+func (*UnimplementedKeyManagementServiceServer) Version(ctx context.Context, req *VersionRequest) (*VersionResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Version not implemented")
+}
+func (*UnimplementedKeyManagementServiceServer) Decrypt(ctx context.Context, req *DecryptRequest) (*DecryptResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Decrypt not implemented")
+}
+func (*UnimplementedKeyManagementServiceServer) Encrypt(ctx context.Context, req *EncryptRequest) (*EncryptResponse, error) {
+	return nil, status.Errorf(codes.Unimplemented, "method Encrypt not implemented")
+}
+
+func RegisterKeyManagementServiceServer(s *grpc.Server, srv KeyManagementServiceServer) {
+	s.RegisterService(&_KeyManagementService_serviceDesc, srv)
+}
+
+func _KeyManagementService_Version_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(VersionRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Version(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Version",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Version(ctx, req.(*VersionRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _KeyManagementService_Decrypt_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DecryptRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Decrypt(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Decrypt",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Decrypt(ctx, req.(*DecryptRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _KeyManagementService_Encrypt_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(EncryptRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(KeyManagementServiceServer).Encrypt(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/v1beta1.KeyManagementService/Encrypt",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(KeyManagementServiceServer).Encrypt(ctx, req.(*EncryptRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _KeyManagementService_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "v1beta1.KeyManagementService",
+	HandlerType: (*KeyManagementServiceServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Version",
+			Handler:    _KeyManagementService_Version_Handler,
+		},
+		{
+			MethodName: "Decrypt",
+			Handler:    _KeyManagementService_Decrypt_Handler,
+		},
+		{
+			MethodName: "Encrypt",
+			Handler:    _KeyManagementService_Encrypt_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "service.proto",
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/service.proto

@@ -0,0 +1,70 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// To regenerate service.pb.go run hack/update-generated-kms.sh
+syntax = "proto3";
+
+package v1beta1;
+
+// This service defines the public APIs for remote KMS provider.
+service KeyManagementService {
+    // Version returns the runtime name and runtime version of the KMS provider.
+    rpc Version(VersionRequest) returns (VersionResponse) {}
+
+    // Execute decryption operation in KMS provider.
+    rpc Decrypt(DecryptRequest) returns (DecryptResponse) {}
+    // Execute encryption operation in KMS provider.
+    rpc Encrypt(EncryptRequest) returns (EncryptResponse) {}
+}
+
+message VersionRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+}
+
+message VersionResponse {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // Name of the KMS provider.
+    string runtime_name = 2;
+    // Version of the KMS provider. The string must be semver-compatible.
+    string runtime_version = 3;
+}
+
+message DecryptRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // The data to be decrypted.
+    bytes cipher = 2;
+}
+
+message DecryptResponse {
+    // The decrypted data.
+    bytes plain = 1;
+}
+
+message EncryptRequest {
+    // Version of the KMS plugin API.
+    string version = 1;
+    // The data to be encrypted.
+    bytes plain = 2;
+}
+
+message EncryptResponse {
+    // The encrypted data.
+    bytes cipher = 1;
+}
+

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/envelope/v1beta1/v1beta1.go

@@ -1,5 +1,5 @@
 /*
-Copyright 2015 The Kubernetes Authors.
+Copyright 2019 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -14,4 +14,10 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package etcd // import "k8s.io/apiserver/pkg/storage/etcd"
+// Package v1beta1 contains definition of kms-plugin's gRPC service.
+package v1beta1
+
+// IsVersionCheckMethod determines whether the supplied method is a version check against kms-plugin.
+func IsVersionCheckMethod(method string) bool {
+	return method == "/v1beta1.KeyManagementService/Version"
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/identity/identity.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package identity
+
+import (
+	"bytes"
+	"fmt"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// identityTransformer performs no transformation on provided data, but validates
+// that the data is not encrypted data during TransformFromStorage
+type identityTransformer struct{}
+
+// NewEncryptCheckTransformer returns an identityTransformer which returns an error
+// on attempts to read encrypted data
+func NewEncryptCheckTransformer() value.Transformer {
+	return identityTransformer{}
+}
+
+// TransformFromStorage returns the input bytes if the data is not encrypted
+func (identityTransformer) TransformFromStorage(b []byte, context value.Context) ([]byte, bool, error) {
+	// identityTransformer has to return an error if the data is encoded using another transformer.
+	// JSON data starts with '{'. Protobuf data has a prefix 'k8s[\x00-\xFF]'.
+	// Prefix 'k8s:enc:' is reserved for encrypted data on disk.
+	if bytes.HasPrefix(b, []byte("k8s:enc:")) {
+		return []byte{}, false, fmt.Errorf("identity transformer tried to read encrypted data")
+	}
+	return b, false, nil
+}
+
+// TransformToStorage implements the Transformer interface for identityTransformer
+func (identityTransformer) TransformToStorage(b []byte, context value.Context) ([]byte, error) {
+	return b, nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/encrypt/secretbox/secretbox.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package secretbox transforms values for storage at rest using XSalsa20 and Poly1305.
+package secretbox
+
+import (
+	"crypto/rand"
+	"fmt"
+
+	"golang.org/x/crypto/nacl/secretbox"
+
+	"k8s.io/apiserver/pkg/storage/value"
+)
+
+// secretbox implements at rest encryption of the provided values given a 32 byte secret key.
+// Uses a standard 24 byte nonce (placed at the beginning of the cipher text) generated
+// from crypto/rand. Does not perform authentication of the data at rest.
+type secretboxTransformer struct {
+	key [32]byte
+}
+
+const nonceSize = 24
+
+// NewSecretboxTransformer takes the given key and performs encryption and decryption on the given
+// data.
+func NewSecretboxTransformer(key [32]byte) value.Transformer {
+	return &secretboxTransformer{key: key}
+}
+
+func (t *secretboxTransformer) TransformFromStorage(data []byte, context value.Context) ([]byte, bool, error) {
+	if len(data) < (secretbox.Overhead + nonceSize) {
+		return nil, false, fmt.Errorf("the stored data was shorter than the required size")
+	}
+	var nonce [nonceSize]byte
+	copy(nonce[:], data[:nonceSize])
+	data = data[nonceSize:]
+	out := make([]byte, 0, len(data)-secretbox.Overhead)
+	result, ok := secretbox.Open(out, data, &nonce, &t.key)
+	if !ok {
+		return nil, false, fmt.Errorf("output array was not large enough for encryption")
+	}
+	return result, false, nil
+}
+
+func (t *secretboxTransformer) TransformToStorage(data []byte, context value.Context) ([]byte, error) {
+	var nonce [nonceSize]byte
+	n, err := rand.Read(nonce[:])
+	if err != nil {
+		return nil, err
+	}
+	if n != nonceSize {
+		return nil, fmt.Errorf("unable to read sufficient random bytes")
+	}
+	return secretbox.Seal(nonce[:], data, &nonce, &t.key), nil
+}

vendor/k8s.io/apiserver/pkg/storage/value/metrics.go

@@ -22,7 +22,8 @@ import (
 
 	"google.golang.org/grpc/status"
 
-	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 const (
@@ -30,85 +31,104 @@ const (
 	subsystem = "storage"
 )
 
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
 var (
-	transformerLatencies = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
+	transformerLatencies = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "transformation_duration_seconds",
 			Help:      "Latencies in seconds of value transformation operations.",
 			// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
 			// external KMS is involved latencies may climb into milliseconds.
-			Buckets: prometheus.ExponentialBuckets(5e-6, 2, 14),
+			Buckets:        metrics.ExponentialBuckets(5e-6, 2, 14),
+			StabilityLevel: metrics.ALPHA,
 		},
 		[]string{"transformation_type"},
 	)
-	deprecatedTransformerLatencies = prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
+	deprecatedTransformerLatencies = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
 			Namespace: namespace,
 			Subsystem: subsystem,
 			Name:      "transformation_latencies_microseconds",
-			Help:      "(Deprecated) Latencies in microseconds of value transformation operations.",
+			Help:      "Latencies in microseconds of value transformation operations.",
 			// In-process transformations (ex. AES CBC) complete on the order of 20 microseconds. However, when
 			// external KMS is involved latencies may climb into milliseconds.
-			Buckets: prometheus.ExponentialBuckets(5, 2, 14),
+			Buckets:           metrics.ExponentialBuckets(5, 2, 14),
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.14.0",
 		},
 		[]string{"transformation_type"},
 	)
 
-	transformerOperationsTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "transformation_operations_total",
-			Help:      "Total number of transformations.",
+	transformerOperationsTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "transformation_operations_total",
+			Help:           "Total number of transformations.",
+			StabilityLevel: metrics.ALPHA,
 		},
-		[]string{"transformation_type", "status"},
+		[]string{"transformation_type", "transformer_prefix", "status"},
 	)
 
-	deprecatedTransformerFailuresTotal = prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "transformation_failures_total",
-			Help:      "(Deprecated) Total number of failed transformation operations.",
+	deprecatedTransformerFailuresTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "transformation_failures_total",
+			Help:              "Total number of failed transformation operations.",
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.15.0",
 		},
 		[]string{"transformation_type"},
 	)
 
-	envelopeTransformationCacheMissTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "envelope_transformation_cache_misses_total",
-			Help:      "Total number of cache misses while accessing key decryption key(KEK).",
+	envelopeTransformationCacheMissTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "envelope_transformation_cache_misses_total",
+			Help:           "Total number of cache misses while accessing key decryption key(KEK).",
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
 
-	dataKeyGenerationLatencies = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "data_key_generation_duration_seconds",
-			Help:      "Latencies in seconds of data encryption key(DEK) generation operations.",
-			Buckets:   prometheus.ExponentialBuckets(5e-6, 2, 14),
+	dataKeyGenerationLatencies = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "data_key_generation_duration_seconds",
+			Help:           "Latencies in seconds of data encryption key(DEK) generation operations.",
+			Buckets:        metrics.ExponentialBuckets(5e-6, 2, 14),
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
-	deprecatedDataKeyGenerationLatencies = prometheus.NewHistogram(
-		prometheus.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "data_key_generation_latencies_microseconds",
-			Help:      "(Deprecated) Latencies in microseconds of data encryption key(DEK) generation operations.",
-			Buckets:   prometheus.ExponentialBuckets(5, 2, 14),
+	deprecatedDataKeyGenerationLatencies = metrics.NewHistogram(
+		&metrics.HistogramOpts{
+			Namespace:         namespace,
+			Subsystem:         subsystem,
+			Name:              "data_key_generation_latencies_microseconds",
+			Help:              "Latencies in microseconds of data encryption key(DEK) generation operations.",
+			Buckets:           metrics.ExponentialBuckets(5, 2, 14),
+			StabilityLevel:    metrics.ALPHA,
+			DeprecatedVersion: "1.14.0",
 		},
 	)
-	dataKeyGenerationFailuresTotal = prometheus.NewCounter(
-		prometheus.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "data_key_generation_failures_total",
-			Help:      "Total number of failed data encryption key(DEK) generation operations.",
+	dataKeyGenerationFailuresTotal = metrics.NewCounter(
+		&metrics.CounterOpts{
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "data_key_generation_failures_total",
+			Help:           "Total number of failed data encryption key(DEK) generation operations.",
+			StabilityLevel: metrics.ALPHA,
 		},
 	)
 )
@@ -117,21 +137,21 @@ var registerMetrics sync.Once
 
 func RegisterMetrics() {
 	registerMetrics.Do(func() {
-		prometheus.MustRegister(transformerLatencies)
-		prometheus.MustRegister(deprecatedTransformerLatencies)
-		prometheus.MustRegister(transformerOperationsTotal)
-		prometheus.MustRegister(deprecatedTransformerFailuresTotal)
-		prometheus.MustRegister(envelopeTransformationCacheMissTotal)
-		prometheus.MustRegister(dataKeyGenerationLatencies)
-		prometheus.MustRegister(deprecatedDataKeyGenerationLatencies)
-		prometheus.MustRegister(dataKeyGenerationFailuresTotal)
+		legacyregistry.MustRegister(transformerLatencies)
+		legacyregistry.MustRegister(deprecatedTransformerLatencies)
+		legacyregistry.MustRegister(transformerOperationsTotal)
+		legacyregistry.MustRegister(deprecatedTransformerFailuresTotal)
+		legacyregistry.MustRegister(envelopeTransformationCacheMissTotal)
+		legacyregistry.MustRegister(dataKeyGenerationLatencies)
+		legacyregistry.MustRegister(deprecatedDataKeyGenerationLatencies)
+		legacyregistry.MustRegister(dataKeyGenerationFailuresTotal)
 	})
 }
 
 // RecordTransformation records latencies and count of TransformFromStorage and TransformToStorage operations.
 // Note that transformation_failures_total metric is deprecated, use transformation_operations_total instead.
-func RecordTransformation(transformationType string, start time.Time, err error) {
-	transformerOperationsTotal.WithLabelValues(transformationType, status.Code(err).String()).Inc()
+func RecordTransformation(transformationType, transformerPrefix string, start time.Time, err error) {
+	transformerOperationsTotal.WithLabelValues(transformationType, transformerPrefix, status.Code(err).String()).Inc()
 
 	switch {
 	case err == nil:

vendor/k8s.io/apiserver/pkg/storage/value/transformer.go

@@ -87,18 +87,12 @@ func (t *MutableTransformer) Set(transformer Transformer) {
 }
 
 func (t *MutableTransformer) TransformFromStorage(data []byte, context Context) (out []byte, stale bool, err error) {
-	defer func(start time.Time) {
-		RecordTransformation("from_storage", start, err)
-	}(time.Now())
 	t.lock.RLock()
 	transformer := t.transformer
 	t.lock.RUnlock()
 	return transformer.TransformFromStorage(data, context)
 }
 func (t *MutableTransformer) TransformToStorage(data []byte, context Context) (out []byte, err error) {
-	defer func(start time.Time) {
-		RecordTransformation("to_storage", start, err)
-	}(time.Now())
 	t.lock.RLock()
 	transformer := t.transformer
 	t.lock.RUnlock()
@@ -136,6 +130,7 @@ func NewPrefixTransformers(err error, transformers ...PrefixTransformer) Transfo
 // the result of transforming the value. It will always mark any transformation as stale that is not using
 // the first transformer.
 func (t *prefixTransformers) TransformFromStorage(data []byte, context Context) ([]byte, bool, error) {
+	start := time.Now()
 	var errs []error
 	for i, transformer := range t.transformers {
 		if bytes.HasPrefix(data, transformer.Prefix) {
@@ -143,10 +138,15 @@ func (t *prefixTransformers) TransformFromStorage(data []byte, context Context)
 			// To migrate away from encryption, user can specify an identity transformer higher up
 			// (in the config file) than the encryption transformer. In that scenario, the identity transformer needs to
 			// identify (during reads from disk) whether the data being read is encrypted or not. If the data is encrypted,
-			// it shall throw an error, but that error should not prevent subsequent transformers from being tried.
+			// it shall throw an error, but that error should not prevent the next subsequent transformer from being tried.
 			if len(transformer.Prefix) == 0 && err != nil {
 				continue
 			}
+			if len(transformer.Prefix) == 0 {
+				RecordTransformation("from_storage", "identity", start, err)
+			} else {
+				RecordTransformation("from_storage", string(transformer.Prefix), start, err)
+			}
 
 			// It is valid to have overlapping prefixes when the same encryption provider
 			// is specified multiple times but with different keys (the first provider is
@@ -189,15 +189,18 @@ func (t *prefixTransformers) TransformFromStorage(data []byte, context Context)
 	if err := errors.Reduce(errors.NewAggregate(errs)); err != nil {
 		return nil, false, err
 	}
+	RecordTransformation("from_storage", "unknown", start, t.err)
 	return nil, false, t.err
 }
 
 // TransformToStorage uses the first transformer and adds its prefix to the data.
 func (t *prefixTransformers) TransformToStorage(data []byte, context Context) ([]byte, error) {
+	start := time.Now()
 	transformer := t.transformers[0]
 	prefixedData := make([]byte, len(transformer.Prefix), len(data)+len(transformer.Prefix))
 	copy(prefixedData, transformer.Prefix)
 	result, err := transformer.Transformer.TransformToStorage(data, context)
+	RecordTransformation("to_storage", string(transformer.Prefix), start, err)
 	if err != nil {
 		return nil, err
 	}