vendor: revendor metrics-server, custom-metrics-apiserver

vendor/k8s.io/apiserver/pkg/admission/config.go

@@ -25,7 +25,7 @@ import (
 	"path"
 	"path/filepath"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	"sigs.k8s.io/yaml"
 
 	"k8s.io/apimachinery/pkg/runtime"

vendor/k8s.io/apiserver/pkg/admission/plugin/namespace/lifecycle/admission.go

@@ -22,7 +22,7 @@ import (
 	"io"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	v1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go

@@ -26,7 +26,7 @@ import (
 	jsonpatch "github.com/evanphx/json-patch"
 
 	apiequality "k8s.io/apimachinery/pkg/api/equality"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	admissionv1 "k8s.io/api/admission/v1"
 	admissionregistrationv1 "k8s.io/api/admissionregistration/v1"
@@ -43,6 +43,7 @@ import (
 	webhookrequest "k8s.io/apiserver/pkg/admission/plugin/webhook/request"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
+	"k8s.io/apiserver/pkg/warning"
 	utiltrace "k8s.io/utils/trace"
 )
 
@@ -267,6 +268,9 @@ func (a *mutatingDispatcher) callAttrMutatingHook(ctx context.Context, h *admiss
 			klog.Warningf("Failed to set admission audit annotation %s to %s for mutating webhook %s: %v", key, v, h.Name, err)
 		}
 	}
+	for _, w := range result.Warnings {
+		warning.AddWarning(ctx, "", w)
+	}
 
 	if !result.Allowed {
 		return false, &webhookutil.ErrWebhookRejection{Status: webhookerrors.ToStatusErr(h.Name, result.Result)}

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/object/matcher.go

@@ -23,7 +23,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/admission/plugin/webhook"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // Matcher decides if a request selected by the ObjectSelector.

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/request/admissionreview.go

@@ -36,6 +36,7 @@ type AdmissionResponse struct {
 	Patch            []byte
 	PatchType        admissionv1.PatchType
 	Result           *metav1.Status
+	Warnings         []string
 }
 
 // VerifyAdmissionResponse checks the validity of the provided admission review object, and returns the
@@ -93,6 +94,7 @@ func VerifyAdmissionResponse(uid types.UID, mutating bool, review runtime.Object
 			Patch:            patch,
 			PatchType:        patchType,
 			Result:           r.Response.Result,
+			Warnings:         r.Response.Warnings,
 		}, nil
 
 	case *admissionv1beta1.AdmissionReview:
@@ -118,6 +120,7 @@ func VerifyAdmissionResponse(uid types.UID, mutating bool, review runtime.Object
 			Patch:            patch,
 			PatchType:        patchType,
 			Result:           r.Response.Result,
+			Warnings:         r.Response.Warnings,
 		}, nil
 
 	default:

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/validating/dispatcher.go

@@ -33,7 +33,8 @@ import (
 	"k8s.io/apiserver/pkg/admission/plugin/webhook/generic"
 	webhookrequest "k8s.io/apiserver/pkg/admission/plugin/webhook/request"
 	webhookutil "k8s.io/apiserver/pkg/util/webhook"
-	"k8s.io/klog"
+	"k8s.io/apiserver/pkg/warning"
+	"k8s.io/klog/v2"
 	utiltrace "k8s.io/utils/trace"
 )
 
@@ -227,6 +228,9 @@ func (d *validatingDispatcher) callHook(ctx context.Context, h *v1.ValidatingWeb
 			klog.Warningf("Failed to set admission audit annotation %s to %s for validating webhook %s: %v", key, v, h.Name, err)
 		}
 	}
+	for _, w := range result.Warnings {
+		warning.AddWarning(ctx, "", w)
+	}
 	if result.Allowed {
 		return nil
 	}

vendor/k8s.io/apiserver/pkg/admission/plugins.go

@@ -26,7 +26,7 @@ import (
 	"strings"
 	"sync"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // Factory is a function that returns an Interface for admission decisions.

vendor/k8s.io/apiserver/pkg/apis/config/validation/validation.go

@@ -31,6 +31,7 @@ const (
 	keyLenErrFmt             = "secret is not of the expected length, got %d, expected one of %v"
 	unsupportedSchemeErrFmt  = "unsupported scheme %q for KMS provider, only unix is supported"
 	atLeastOneRequiredErrFmt = "at least one %s is required"
+	invalidURLErrFmt         = "invalid endpoint for kms provider, error: parse %s: net/url: invalid control character in URL"
 	mandatoryFieldErrFmt     = "%s is a mandatory field for a %s"
 	base64EncodingErr        = "secrets must be base64 encoded"
 	zeroOrNegativeErrFmt     = "%s should be a positive value"

vendor/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go

@@ -380,7 +380,7 @@ var (
 		"global-default", "global-default", 9900,
 		flowcontrol.FlowDistinguisherMethodByUserType,
 		flowcontrol.PolicyRulesWithSubjects{
-			Subjects: groups(serviceaccount.AllServiceAccountsGroup),
+			Subjects: groups(user.AllUnauthenticated, user.AllAuthenticated),
 			ResourceRules: []flowcontrol.ResourcePolicyRule{resourceRule(
 				[]string{flowcontrol.VerbAll},
 				[]string{flowcontrol.APIGroupAll},

vendor/k8s.io/apiserver/pkg/audit/context.go

@@ -0,0 +1,84 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package audit
+
+import (
+	"context"
+
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// The key type is unexported to prevent collisions
+type key int
+
+const (
+	// auditAnnotationsKey is the context key for the audit annotations.
+	auditAnnotationsKey key = iota
+)
+
+// annotations = *[]annotation instead of a map to preserve order of insertions
+type annotation struct {
+	key, value string
+}
+
+// WithAuditAnnotations returns a new context that can store audit annotations
+// via the AddAuditAnnotation function.  This function is meant to be called from
+// an early request handler to allow all later layers to set audit annotations.
+// This is required to support flows where handlers that come before WithAudit
+// (such as WithAuthentication) wish to set audit annotations.
+func WithAuditAnnotations(parent context.Context) context.Context {
+	// this should never really happen, but prevent double registration of this slice
+	if _, ok := parent.Value(auditAnnotationsKey).(*[]annotation); ok {
+		return parent
+	}
+
+	var annotations []annotation // avoid allocations until we actually need it
+	return genericapirequest.WithValue(parent, auditAnnotationsKey, &annotations)
+}
+
+// AddAuditAnnotation sets the audit annotation for the given key, value pair.
+// It is safe to call at most parts of request flow that come after WithAuditAnnotations.
+// The notable exception being that this function must not be called via a
+// defer statement (i.e. after ServeHTTP) in a handler that runs before WithAudit
+// as at that point the audit event has already been sent to the audit sink.
+// Handlers that are unaware of their position in the overall request flow should
+// prefer AddAuditAnnotation over LogAnnotation to avoid dropping annotations.
+func AddAuditAnnotation(ctx context.Context, key, value string) {
+	// use the audit event directly if we have it
+	if ae := genericapirequest.AuditEventFrom(ctx); ae != nil {
+		LogAnnotation(ae, key, value)
+		return
+	}
+
+	annotations, ok := ctx.Value(auditAnnotationsKey).(*[]annotation)
+	if !ok {
+		return // adding audit annotation is not supported at this call site
+	}
+
+	*annotations = append(*annotations, annotation{key: key, value: value})
+}
+
+// This is private to prevent reads/write to the slice from outside of this package.
+// The audit event should be directly read to get access to the annotations.
+func auditAnnotationsFrom(ctx context.Context) []annotation {
+	annotations, ok := ctx.Value(auditAnnotationsKey).(*[]annotation)
+	if !ok {
+		return nil // adding audit annotation is not supported at this call site
+	}
+
+	return *annotations
+}

vendor/k8s.io/apiserver/pkg/audit/event/attributes.go

@@ -1,148 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package event
-
-import (
-	"fmt"
-	"net/url"
-
-	authnv1 "k8s.io/api/authentication/v1"
-	"k8s.io/apiserver/pkg/apis/audit"
-	authuser "k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/apiserver/pkg/authorization/authorizer"
-)
-
-var _ authorizer.Attributes = &attributes{}
-
-// attributes implements the authorizer attributes interface
-// with event data. This is used for enforced audit backends
-type attributes struct {
-	event *audit.Event
-	path  string
-}
-
-// NewAttributes returns a new attributes struct and parsed request uri
-// if needed
-func NewAttributes(event *audit.Event) (authorizer.Attributes, error) {
-	a := attributes{
-		event: event,
-	}
-	if event.ObjectRef == nil {
-		u, err := url.ParseRequestURI(a.event.RequestURI)
-		if err != nil {
-			return nil, fmt.Errorf("could not parse url: %v", err)
-		}
-		a.path = u.Path
-	}
-	return &a, nil
-}
-
-// GetUser returns the user. This is only used for checking audit policy,
-// and the audit policy user check is based off the original user,
-// not the impersonated user.
-func (a *attributes) GetUser() authuser.Info {
-	return user(a.event.User)
-}
-
-// GetVerb returns the verb
-func (a *attributes) GetVerb() string {
-	return a.event.Verb
-}
-
-// IsReadOnly determines if the verb is a read only action
-func (a *attributes) IsReadOnly() bool {
-	return a.event.Verb == "get" || a.event.Verb == "list" || a.event.Verb == "watch"
-}
-
-// GetNamespace returns the object namespace if present
-func (a *attributes) GetNamespace() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.Namespace
-}
-
-// GetResource returns the object resource if present
-func (a *attributes) GetResource() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.Resource
-}
-
-// GetSubresource returns the object subresource if present
-func (a *attributes) GetSubresource() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.Subresource
-}
-
-// GetName returns the object name if present
-func (a *attributes) GetName() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.Name
-}
-
-// GetAPIGroup returns the object api group if present
-func (a *attributes) GetAPIGroup() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.APIGroup
-}
-
-// GetAPIVersion returns the object api version if present
-func (a *attributes) GetAPIVersion() string {
-	if a.event.ObjectRef == nil {
-		return ""
-	}
-	return a.event.ObjectRef.APIVersion
-}
-
-// IsResourceRequest determines if the request was acted on a resource
-func (a *attributes) IsResourceRequest() bool {
-	return a.event.ObjectRef != nil
-}
-
-// GetPath returns the path uri accessed
-func (a *attributes) GetPath() string {
-	return a.path
-}
-
-// user represents the event user
-type user authnv1.UserInfo
-
-// GetName returns the user name
-func (u user) GetName() string { return u.Username }
-
-// GetUID returns the user uid
-func (u user) GetUID() string { return u.UID }
-
-// GetGroups returns the user groups
-func (u user) GetGroups() []string { return u.Groups }
-
-// GetExtra returns the user extra data
-func (u user) GetExtra() map[string][]string {
-	m := map[string][]string{}
-	for k, v := range u.Extra {
-		m[k] = []string(v)
-	}
-	return m
-}

vendor/k8s.io/apiserver/pkg/audit/metrics.go

@@ -22,7 +22,7 @@ import (
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
 	"k8s.io/component-base/metrics"
 	"k8s.io/component-base/metrics/legacyregistry"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const (

vendor/k8s.io/apiserver/pkg/audit/policy/dynamic.go

@@ -1,54 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package policy
-
-import (
-	"k8s.io/api/auditregistration/v1alpha1"
-	"k8s.io/apiserver/pkg/apis/audit"
-	"k8s.io/apiserver/pkg/authorization/authorizer"
-)
-
-// ConvertDynamicPolicyToInternal constructs an internal policy type from a
-// v1alpha1 dynamic type
-func ConvertDynamicPolicyToInternal(p *v1alpha1.Policy) *audit.Policy {
-	stages := make([]audit.Stage, len(p.Stages))
-	for i, stage := range p.Stages {
-		stages[i] = audit.Stage(stage)
-	}
-	return &audit.Policy{
-		Rules: []audit.PolicyRule{
-			{
-				Level: audit.Level(p.Level),
-			},
-		},
-		OmitStages: InvertStages(stages),
-	}
-}
-
-// NewDynamicChecker returns a new dynamic policy checker
-func NewDynamicChecker() Checker {
-	return &dynamicPolicyChecker{}
-}
-
-type dynamicPolicyChecker struct{}
-
-// LevelAndStages returns returns a fixed level of the full event, this is so that the downstream policy
-// can be applied per sink.
-// TODO: this needs benchmarking before the API moves to beta to determine the effect this has on the apiserver
-func (d *dynamicPolicyChecker) LevelAndStages(authorizer.Attributes) (audit.Level, []audit.Stage) {
-	return audit.LevelRequestResponse, []audit.Stage{}
-}

vendor/k8s.io/apiserver/pkg/audit/policy/reader.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/apiserver/pkg/apis/audit/validation"
 	"k8s.io/apiserver/pkg/audit"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var (

vendor/k8s.io/apiserver/pkg/audit/request.go

@@ -24,7 +24,7 @@ import (
 	"time"
 
 	"github.com/google/uuid"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	authnv1 "k8s.io/api/authentication/v1"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -88,6 +88,10 @@ func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs a
 		}
 	}
 
+	for _, kv := range auditAnnotationsFrom(req.Context()) {
+		LogAnnotation(ev, kv.key, kv.value)
+	}
+
 	return ev, nil
 }
 

vendor/k8s.io/apiserver/pkg/audit/util/conversion.go

@@ -1,49 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package util
-
-import (
-	"k8s.io/api/auditregistration/v1alpha1"
-	"k8s.io/apiserver/pkg/util/webhook"
-)
-
-// HookClientConfigForSink constructs a webhook.ClientConfig using a v1alpha1.AuditSink API object.
-// webhook.ClientConfig is used to create a HookClient and the purpose of the config struct is to
-// share that with other packages that need to create a HookClient.
-func HookClientConfigForSink(a *v1alpha1.AuditSink) webhook.ClientConfig {
-	c := a.Spec.Webhook.ClientConfig
-	ret := webhook.ClientConfig{Name: a.Name, CABundle: c.CABundle}
-	if c.URL != nil {
-		ret.URL = *c.URL
-	}
-	if c.Service != nil {
-		ret.Service = &webhook.ClientConfigService{
-			Name:      c.Service.Name,
-			Namespace: c.Service.Namespace,
-		}
-		if c.Service.Port != nil {
-			ret.Service.Port = *c.Service.Port
-		} else {
-			ret.Service.Port = 443
-		}
-
-		if c.Service.Path != nil {
-			ret.Service.Path = *c.Service.Path
-		}
-	}
-	return ret
-}

vendor/k8s.io/apiserver/pkg/authentication/authenticator/interfaces.go

@@ -35,13 +35,6 @@ type Request interface {
 	AuthenticateRequest(req *http.Request) (*Response, bool, error)
 }
 
-// Password checks a username and password against a backing authentication
-// store and returns a Response or an error if the password could not be
-// checked.
-type Password interface {
-	AuthenticatePassword(ctx context.Context, user, password string) (*Response, bool, error)
-}
-
 // TokenFunc is a function that implements the Token interface.
 type TokenFunc func(ctx context.Context, token string) (*Response, bool, error)
 
@@ -58,14 +51,6 @@ func (f RequestFunc) AuthenticateRequest(req *http.Request) (*Response, bool, er
 	return f(req)
 }
 
-// PasswordFunc is a function that implements the Password interface.
-type PasswordFunc func(ctx context.Context, user, password string) (*Response, bool, error)
-
-// AuthenticatePassword implements authenticator.Password.
-func (f PasswordFunc) AuthenticatePassword(ctx context.Context, user, password string) (*Response, bool, error) {
-	return f(ctx, user, password)
-}
-
 // Response is the struct returned by authenticator interfaces upon successful
 // authentication. It contains information about whether the authenticator
 // authenticated the request, information about the context of the

vendor/k8s.io/apiserver/pkg/authentication/request/headerrequest/requestheader_controller.go

@@ -0,0 +1,337 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package headerrequest
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	coreinformers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog/v2"
+	"sync/atomic"
+)
+
+const (
+	authenticationRoleName = "extension-apiserver-authentication-reader"
+)
+
+// RequestHeaderAuthRequestProvider a provider that knows how to dynamically fill parts of RequestHeaderConfig struct
+type RequestHeaderAuthRequestProvider interface {
+	UsernameHeaders() []string
+	GroupHeaders() []string
+	ExtraHeaderPrefixes() []string
+	AllowedClientNames() []string
+}
+
+var _ RequestHeaderAuthRequestProvider = &RequestHeaderAuthRequestController{}
+
+type requestHeaderBundle struct {
+	UsernameHeaders     []string
+	GroupHeaders        []string
+	ExtraHeaderPrefixes []string
+	AllowedClientNames  []string
+}
+
+// RequestHeaderAuthRequestController a controller that exposes a set of methods for dynamically filling parts of RequestHeaderConfig struct.
+// The methods are sourced from the config map which is being monitored by this controller.
+// The controller is primed from the server at the construction time for components that don't want to dynamically react to changes
+// in the config map.
+type RequestHeaderAuthRequestController struct {
+	name string
+
+	configmapName      string
+	configmapNamespace string
+
+	client                  kubernetes.Interface
+	configmapLister         corev1listers.ConfigMapNamespaceLister
+	configmapInformer       cache.SharedIndexInformer
+	configmapInformerSynced cache.InformerSynced
+
+	queue workqueue.RateLimitingInterface
+
+	// exportedRequestHeaderBundle is a requestHeaderBundle that contains the last read, non-zero length content of the configmap
+	exportedRequestHeaderBundle atomic.Value
+
+	usernameHeadersKey     string
+	groupHeadersKey        string
+	extraHeaderPrefixesKey string
+	allowedClientNamesKey  string
+}
+
+// NewRequestHeaderAuthRequestController creates a new controller that implements RequestHeaderAuthRequestController
+func NewRequestHeaderAuthRequestController(
+	cmName string,
+	cmNamespace string,
+	client kubernetes.Interface,
+	usernameHeadersKey, groupHeadersKey, extraHeaderPrefixesKey, allowedClientNamesKey string) *RequestHeaderAuthRequestController {
+	c := &RequestHeaderAuthRequestController{
+		name: "RequestHeaderAuthRequestController",
+
+		client: client,
+
+		configmapName:      cmName,
+		configmapNamespace: cmNamespace,
+
+		usernameHeadersKey:     usernameHeadersKey,
+		groupHeadersKey:        groupHeadersKey,
+		extraHeaderPrefixesKey: extraHeaderPrefixesKey,
+		allowedClientNamesKey:  allowedClientNamesKey,
+
+		queue: workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "RequestHeaderAuthRequestController"),
+	}
+
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	c.configmapInformer = coreinformers.NewFilteredConfigMapInformer(client, c.configmapNamespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *metav1.ListOptions) {
+		listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", c.configmapName).String()
+	})
+
+	c.configmapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+		},
+	})
+
+	c.configmapLister = corev1listers.NewConfigMapLister(c.configmapInformer.GetIndexer()).ConfigMaps(c.configmapNamespace)
+	c.configmapInformerSynced = c.configmapInformer.HasSynced
+
+	return c
+}
+
+func (c *RequestHeaderAuthRequestController) UsernameHeaders() []string {
+	return c.loadRequestHeaderFor(c.usernameHeadersKey)
+}
+
+func (c *RequestHeaderAuthRequestController) GroupHeaders() []string {
+	return c.loadRequestHeaderFor(c.groupHeadersKey)
+}
+
+func (c *RequestHeaderAuthRequestController) ExtraHeaderPrefixes() []string {
+	return c.loadRequestHeaderFor(c.extraHeaderPrefixesKey)
+}
+
+func (c *RequestHeaderAuthRequestController) AllowedClientNames() []string {
+	return c.loadRequestHeaderFor(c.allowedClientNamesKey)
+}
+
+// Run starts RequestHeaderAuthRequestController controller and blocks until stopCh is closed.
+func (c *RequestHeaderAuthRequestController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	go c.configmapInformer.Run(stopCh)
+
+	// wait for caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync(c.name, stopCh, c.configmapInformerSynced) {
+		return
+	}
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	<-stopCh
+}
+
+// // RunOnce runs a single sync loop
+func (c *RequestHeaderAuthRequestController) RunOnce() error {
+	configMap, err := c.client.CoreV1().ConfigMaps(c.configmapNamespace).Get(context.TODO(), c.configmapName, metav1.GetOptions{})
+	switch {
+	case errors.IsNotFound(err):
+		// ignore, authConfigMap is nil now
+		return nil
+	case errors.IsForbidden(err):
+		klog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
+			"'kubectl create rolebinding -n %s ROLEBINDING_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
+			c.configmapName, c.configmapNamespace, c.configmapNamespace, authenticationRoleName)
+		return err
+	case err != nil:
+		return err
+	}
+	return c.syncConfigMap(configMap)
+}
+
+func (c *RequestHeaderAuthRequestController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *RequestHeaderAuthRequestController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.sync()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// sync reads the config and propagates the changes to exportedRequestHeaderBundle
+// which is exposed by the set of methods that are used to fill RequestHeaderConfig struct
+func (c *RequestHeaderAuthRequestController) sync() error {
+	configMap, err := c.configmapLister.Get(c.configmapName)
+	if err != nil {
+		return err
+	}
+	return c.syncConfigMap(configMap)
+}
+
+func (c *RequestHeaderAuthRequestController) syncConfigMap(configMap *corev1.ConfigMap) error {
+	hasChanged, newRequestHeaderBundle, err := c.hasRequestHeaderBundleChanged(configMap)
+	if err != nil {
+		return err
+	}
+	if hasChanged {
+		c.exportedRequestHeaderBundle.Store(newRequestHeaderBundle)
+		klog.V(2).Infof("Loaded a new request header values for %v", c.name)
+	}
+	return nil
+}
+
+func (c *RequestHeaderAuthRequestController) hasRequestHeaderBundleChanged(cm *corev1.ConfigMap) (bool, *requestHeaderBundle, error) {
+	currentHeadersBundle, err := c.getRequestHeaderBundleFromConfigMap(cm)
+	if err != nil {
+		return false, nil, err
+	}
+
+	rawHeaderBundle := c.exportedRequestHeaderBundle.Load()
+	if rawHeaderBundle == nil {
+		return true, currentHeadersBundle, nil
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	loadedHeadersBundle, ok := rawHeaderBundle.(*requestHeaderBundle)
+	if !ok {
+		return true, currentHeadersBundle, nil
+	}
+
+	if !equality.Semantic.DeepEqual(loadedHeadersBundle, currentHeadersBundle) {
+		return true, currentHeadersBundle, nil
+	}
+	return false, nil, nil
+}
+
+func (c *RequestHeaderAuthRequestController) getRequestHeaderBundleFromConfigMap(cm *corev1.ConfigMap) (*requestHeaderBundle, error) {
+	usernameHeaderCurrentValue, err := deserializeStrings(cm.Data[c.usernameHeadersKey])
+	if err != nil {
+		return nil, err
+	}
+
+	groupHeadersCurrentValue, err := deserializeStrings(cm.Data[c.groupHeadersKey])
+	if err != nil {
+		return nil, err
+	}
+
+	extraHeaderPrefixesCurrentValue, err := deserializeStrings(cm.Data[c.extraHeaderPrefixesKey])
+	if err != nil {
+		return nil, err
+
+	}
+
+	allowedClientNamesCurrentValue, err := deserializeStrings(cm.Data[c.allowedClientNamesKey])
+	if err != nil {
+		return nil, err
+	}
+
+	return &requestHeaderBundle{
+		UsernameHeaders:     usernameHeaderCurrentValue,
+		GroupHeaders:        groupHeadersCurrentValue,
+		ExtraHeaderPrefixes: extraHeaderPrefixesCurrentValue,
+		AllowedClientNames:  allowedClientNamesCurrentValue,
+	}, nil
+}
+
+func (c *RequestHeaderAuthRequestController) loadRequestHeaderFor(key string) []string {
+	rawHeaderBundle := c.exportedRequestHeaderBundle.Load()
+	if rawHeaderBundle == nil {
+		return nil // this can happen if we've been unable load data from the apiserver for some reason
+	}
+	headerBundle := rawHeaderBundle.(*requestHeaderBundle)
+
+	switch key {
+	case c.usernameHeadersKey:
+		return headerBundle.UsernameHeaders
+	case c.groupHeadersKey:
+		return headerBundle.GroupHeaders
+	case c.extraHeaderPrefixesKey:
+		return headerBundle.ExtraHeaderPrefixes
+	case c.allowedClientNamesKey:
+		return headerBundle.AllowedClientNames
+	default:
+		return nil
+	}
+}
+
+func (c *RequestHeaderAuthRequestController) keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return c.configmapNamespace + "/" + c.configmapName
+}
+
+func deserializeStrings(in string) ([]string, error) {
+	if len(in) == 0 {
+		return nil, nil
+	}
+	var ret []string
+	if err := json.Unmarshal([]byte(in), &ret); err != nil {
+		return nil, err
+	}
+	return ret, nil
+}

vendor/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go

@@ -34,8 +34,11 @@ import (
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilclock "k8s.io/apimachinery/pkg/util/clock"
+	auditinternal "k8s.io/apiserver/pkg/apis/audit"
+	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
-	"k8s.io/klog"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/klog/v2"
 )
 
 var errAuthnCrash = apierrors.NewInternalError(errors.New("authentication failed unexpectedly"))
@@ -47,6 +50,16 @@ type cacheRecord struct {
 	resp *authenticator.Response
 	ok   bool
 	err  error
+
+	// this cache assumes token authn has no side-effects or temporal dependence.
+	// neither of these are true for audit annotations set via AddAuditAnnotation.
+	//
+	// for audit annotations, the assumption is that for some period of time (cache TTL),
+	// all requests with the same API audiences and the same bearer token result in the
+	// same annotations.  This may not be true if the authenticator sets an annotation
+	// based on the current time, but that may be okay since cache TTLs are generally
+	// small (seconds).
+	annotations map[string]string
 }
 
 type cachedTokenAuthenticator struct {
@@ -109,6 +122,17 @@ func newWithClock(authenticator authenticator.Token, cacheErrs bool, successTTL,
 
 // AuthenticateToken implements authenticator.Token
 func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) {
+	record := a.doAuthenticateToken(ctx, token)
+	if !record.ok || record.err != nil {
+		return nil, false, record.err
+	}
+	for key, value := range record.annotations {
+		audit.AddAuditAnnotation(ctx, key, value)
+	}
+	return record.resp, true, nil
+}
+
+func (a *cachedTokenAuthenticator) doAuthenticateToken(ctx context.Context, token string) *cacheRecord {
 	doneAuthenticating := stats.authenticating()
 
 	auds, audsOk := authenticator.AudiencesFrom(ctx)
@@ -117,7 +141,7 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
 	if record, ok := a.cache.get(key); ok {
 		// Record cache hit
 		doneAuthenticating(true)
-		return record.resp, record.ok, record.err
+		return record
 	}
 
 	// Record cache miss
@@ -125,18 +149,19 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
 	defer doneBlocking()
 	defer doneAuthenticating(false)
 
-	type lookup struct {
-		resp *authenticator.Response
-		ok   bool
-	}
+	c := a.group.DoChan(key, func() (val interface{}, _ error) {
+		// always use one place to read and write the output of AuthenticateToken
+		record := &cacheRecord{}
 
-	c := a.group.DoChan(key, func() (val interface{}, err error) {
 		doneFetching := stats.fetching()
 		// We're leaving the request handling stack so we need to handle crashes
 		// ourselves. Log a stack trace and return a 500 if something panics.
 		defer func() {
 			if r := recover(); r != nil {
-				err = errAuthnCrash
+				// make sure to always return a record
+				record.err = errAuthnCrash
+				val = record
+
 				// Same as stdlib http server code. Manually allocate stack
 				// trace buffer size to prevent excessively large logs
 				const size = 64 << 10
@@ -144,12 +169,12 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
 				buf = buf[:runtime.Stack(buf, false)]
 				klog.Errorf("%v\n%s", r, buf)
 			}
-			doneFetching(err == nil)
+			doneFetching(record.err == nil)
 		}()
 
 		// Check again for a cached record. We may have raced with a fetch.
 		if record, ok := a.cache.get(key); ok {
-			return lookup{record.resp, record.ok}, record.err
+			return record, nil
 		}
 
 		// Detach the context because the lookup may be shared by multiple callers,
@@ -161,29 +186,35 @@ func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token
 			ctx = authenticator.WithAudiences(ctx, auds)
 		}
 
-		resp, ok, err := a.authenticator.AuthenticateToken(ctx, token)
-		if !a.cacheErrs && err != nil {
-			return nil, err
+		// since this is shared work between multiple requests, we have no way of knowing if any
+		// particular request supports audit annotations.  thus we always attempt to record them.
+		ev := &auditinternal.Event{Level: auditinternal.LevelMetadata}
+		ctx = request.WithAuditEvent(ctx, ev)
+
+		record.resp, record.ok, record.err = a.authenticator.AuthenticateToken(ctx, token)
+		record.annotations = ev.Annotations
+
+		if !a.cacheErrs && record.err != nil {
+			return record, nil
 		}
 
 		switch {
-		case ok && a.successTTL > 0:
-			a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.successTTL)
-		case !ok && a.failureTTL > 0:
-			a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.failureTTL)
+		case record.ok && a.successTTL > 0:
+			a.cache.set(key, record, a.successTTL)
+		case !record.ok && a.failureTTL > 0:
+			a.cache.set(key, record, a.failureTTL)
 		}
-		return lookup{resp, ok}, err
+
+		return record, nil
 	})
 
 	select {
 	case result := <-c:
-		if result.Err != nil {
-			return nil, false, result.Err
-		}
-		lookup := result.Val.(lookup)
-		return lookup.resp, lookup.ok, nil
+		// we always set Val and never set Err
+		return result.Val.(*cacheRecord)
 	case <-ctx.Done():
-		return nil, false, ctx.Err()
+		// fake a record on context cancel
+		return &cacheRecord{err: ctx.Err()}
 	}
 }
 

vendor/k8s.io/apiserver/pkg/authentication/token/tokenfile/tokenfile.go

@@ -26,7 +26,7 @@ import (
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 type TokenAuthenticator struct {

vendor/k8s.io/apiserver/pkg/endpoints/deprecation/deprecation.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package deprecation
+
+import (
+	"fmt"
+	"regexp"
+	"strconv"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/version"
+)
+
+type apiLifecycleDeprecated interface {
+	APILifecycleDeprecated() (major, minor int)
+}
+
+type apiLifecycleRemoved interface {
+	APILifecycleRemoved() (major, minor int)
+}
+
+type apiLifecycleReplacement interface {
+	APILifecycleReplacement() schema.GroupVersionKind
+}
+
+// extract all digits at the beginning of the string
+var leadingDigits = regexp.MustCompile(`^(\d+)`)
+
+// MajorMinor parses a numeric major/minor version from the provided version info.
+// The minor version drops all characters after the first non-digit character:
+//   version.Info{Major:"1", Minor:"2+"} -> 1,2
+//   version.Info{Major:"1", Minor:"2.3-build4"} -> 1,2
+func MajorMinor(v version.Info) (int, int, error) {
+	major, err := strconv.Atoi(v.Major)
+	if err != nil {
+		return 0, 0, err
+	}
+	minor, err := strconv.Atoi(leadingDigits.FindString(v.Minor))
+	if err != nil {
+		return 0, 0, err
+	}
+	return major, minor, nil
+}
+
+// IsDeprecated returns true if obj implements APILifecycleDeprecated() and returns
+// a major/minor version that is non-zero and is <= the specified current major/minor version.
+func IsDeprecated(obj runtime.Object, currentMajor, currentMinor int) bool {
+	deprecated, isDeprecated := obj.(apiLifecycleDeprecated)
+	if !isDeprecated {
+		return false
+	}
+
+	deprecatedMajor, deprecatedMinor := deprecated.APILifecycleDeprecated()
+	// no deprecation version expressed
+	if deprecatedMajor == 0 && deprecatedMinor == 0 {
+		return false
+	}
+	// no current version info available
+	if currentMajor == 0 && currentMinor == 0 {
+		return true
+	}
+	// compare deprecation version to current version
+	if deprecatedMajor > currentMajor {
+		return false
+	}
+	if deprecatedMajor == currentMajor && deprecatedMinor > currentMinor {
+		return false
+	}
+	return true
+}
+
+// RemovedRelease returns the major/minor version in which the given object is unavailable (in the form "<major>.<minor>")
+// if the object implements APILifecycleRemoved() to indicate a non-zero removal version, and returns an empty string otherwise.
+func RemovedRelease(obj runtime.Object) string {
+	if removed, hasRemovalInfo := obj.(apiLifecycleRemoved); hasRemovalInfo {
+		removedMajor, removedMinor := removed.APILifecycleRemoved()
+		if removedMajor != 0 || removedMinor != 0 {
+			return fmt.Sprintf("%d.%d", removedMajor, removedMinor)
+		}
+	}
+	return ""
+}
+
+// WarningMessage returns a human-readable deprecation warning if the object implements APILifecycleDeprecated()
+// to indicate a non-zero deprecated major/minor version and has a populated GetObjectKind().GroupVersionKind().
+func WarningMessage(obj runtime.Object) string {
+	deprecated, isDeprecated := obj.(apiLifecycleDeprecated)
+	if !isDeprecated {
+		return ""
+	}
+
+	deprecatedMajor, deprecatedMinor := deprecated.APILifecycleDeprecated()
+	if deprecatedMajor == 0 && deprecatedMinor == 0 {
+		return ""
+	}
+
+	gvk := obj.GetObjectKind().GroupVersionKind()
+	if gvk.Empty() {
+		return ""
+	}
+	deprecationWarning := fmt.Sprintf("%s %s is deprecated in v%d.%d+", gvk.GroupVersion().String(), gvk.Kind, deprecatedMajor, deprecatedMinor)
+
+	if removed, hasRemovalInfo := obj.(apiLifecycleRemoved); hasRemovalInfo {
+		removedMajor, removedMinor := removed.APILifecycleRemoved()
+		if removedMajor != 0 || removedMinor != 0 {
+			deprecationWarning = deprecationWarning + fmt.Sprintf(", unavailable in v%d.%d+", removedMajor, removedMinor)
+		}
+	}
+
+	if replaced, hasReplacement := obj.(apiLifecycleReplacement); hasReplacement {
+		replacement := replaced.APILifecycleReplacement()
+		if !replacement.Empty() {
+			deprecationWarning = deprecationWarning + fmt.Sprintf("; use %s %s", replacement.GroupVersion().String(), replacement.Kind)
+		}
+	}
+
+	return deprecationWarning
+}

vendor/k8s.io/apiserver/pkg/endpoints/discovery/util.go

@@ -23,7 +23,7 @@ import (
 	"io"
 
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const APIGroupPrefix = "/apis"

vendor/k8s.io/apiserver/pkg/endpoints/filters/audit_annotations.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+
+	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/audit/policy"
+)
+
+// WithAuditAnnotations decorates a http.Handler with a []{key, value} that is merged
+// with the audit.Event.Annotations map.  This allows layers that run before WithAudit
+// (such as authentication) to assert annotations.
+// If sink or audit policy is nil, no decoration takes place.
+func WithAuditAnnotations(handler http.Handler, sink audit.Sink, policy policy.Checker) http.Handler {
+	// no need to wrap if auditing is disabled
+	if sink == nil || policy == nil {
+		return handler
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		req = req.WithContext(audit.WithAuditAnnotations(req.Context()))
+		handler.ServeHTTP(w, req)
+	})
+}

vendor/k8s.io/apiserver/pkg/endpoints/filters/authentication.go

@@ -28,7 +28,7 @@ import (
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
 	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // WithAuthentication creates an http handler that tries to authenticate the given request as a user, and then
@@ -71,11 +71,8 @@ func WithAuthentication(handler http.Handler, auth authenticator.Request, failed
 	})
 }
 
-func Unauthorized(s runtime.NegotiatedSerializer, supportsBasicAuth bool) http.Handler {
+func Unauthorized(s runtime.NegotiatedSerializer) http.Handler {
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		if supportsBasicAuth {
-			w.Header().Set("WWW-Authenticate", `Basic realm="kubernetes-master"`)
-		}
 		ctx := req.Context()
 		requestInfo, found := genericapirequest.RequestInfoFrom(ctx)
 		if !found {

vendor/k8s.io/apiserver/pkg/endpoints/filters/authorization.go

@@ -21,7 +21,7 @@ import (
 	"errors"
 	"net/http"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/audit"

vendor/k8s.io/apiserver/pkg/endpoints/filters/impersonation.go

@@ -23,7 +23,7 @@ import (
 	"net/url"
 	"strings"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	authenticationv1 "k8s.io/api/authentication/v1"
 	"k8s.io/api/core/v1"
@@ -117,10 +117,37 @@ func WithImpersonation(handler http.Handler, a authorizer.Authorizer, s runtime.
 			}
 		}
 
-		if !groupsSpecified && username != user.Anonymous {
-			// When impersonating a non-anonymous user, if no groups were specified
-			// include the system:authenticated group in the impersonated user info
-			groups = append(groups, user.AllAuthenticated)
+		if username != user.Anonymous {
+			// When impersonating a non-anonymous user, include the 'system:authenticated' group
+			// in the impersonated user info:
+			// - if no groups were specified
+			// - if a group has been specified other than 'system:authenticated'
+			//
+			// If 'system:unauthenticated' group has been specified we should not include
+			// the 'system:authenticated' group.
+			addAuthenticated := true
+			for _, group := range groups {
+				if group == user.AllAuthenticated || group == user.AllUnauthenticated {
+					addAuthenticated = false
+					break
+				}
+			}
+
+			if addAuthenticated {
+				groups = append(groups, user.AllAuthenticated)
+			}
+		} else {
+			addUnauthenticated := true
+			for _, group := range groups {
+				if group == user.AllUnauthenticated {
+					addUnauthenticated = false
+					break
+				}
+			}
+
+			if addUnauthenticated {
+				groups = append(groups, user.AllUnauthenticated)
+			}
 		}
 
 		newUser := &user.DefaultInfo{

vendor/k8s.io/apiserver/pkg/endpoints/filters/warning.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"fmt"
+	"net/http"
+	"sync"
+	"unicode/utf8"
+
+	"k8s.io/apimachinery/pkg/util/net"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/warning"
+)
+
+// WithWarningRecorder attaches a deduplicating k8s.io/apiserver/pkg/warning#WarningRecorder to the request context.
+func WithWarningRecorder(handler http.Handler) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		recorder := &recorder{writer: w}
+		req = req.WithContext(warning.WithWarningRecorder(req.Context(), recorder))
+		handler.ServeHTTP(w, req)
+	})
+}
+
+var (
+	truncateAtTotalRunes = 4 * 1024
+	truncateItemRunes    = 256
+)
+
+type recordedWarning struct {
+	agent string
+	text  string
+}
+
+type recorder struct {
+	// lock guards calls to AddWarning from multiple threads
+	lock sync.Mutex
+
+	// recorded tracks whether AddWarning was already called with a given text
+	recorded map[string]bool
+
+	// ordered tracks warnings added so they can be replayed and truncated if needed
+	ordered []recordedWarning
+
+	// written tracks how many runes of text have been added as warning headers
+	written int
+
+	// truncating tracks if we have already exceeded truncateAtTotalRunes and are now truncating warning messages as we add them
+	truncating bool
+
+	// writer is the response writer to add warning headers to
+	writer http.ResponseWriter
+}
+
+func (r *recorder) AddWarning(agent, text string) {
+	if len(text) == 0 {
+		return
+	}
+
+	r.lock.Lock()
+	defer r.lock.Unlock()
+
+	// if we've already exceeded our limit and are already truncating, return early
+	if r.written >= truncateAtTotalRunes && r.truncating {
+		return
+	}
+
+	// init if needed
+	if r.recorded == nil {
+		r.recorded = map[string]bool{}
+	}
+
+	// dedupe if already warned
+	if r.recorded[text] {
+		return
+	}
+	r.recorded[text] = true
+	r.ordered = append(r.ordered, recordedWarning{agent: agent, text: text})
+
+	// truncate on a rune boundary, if needed
+	textRuneLength := utf8.RuneCountInString(text)
+	if r.truncating && textRuneLength > truncateItemRunes {
+		text = string([]rune(text)[:truncateItemRunes])
+		textRuneLength = truncateItemRunes
+	}
+
+	// compute the header
+	header, err := net.NewWarningHeader(299, agent, text)
+	if err != nil {
+		return
+	}
+
+	// if this fits within our limit, or we're already truncating, write and return
+	if r.written+textRuneLength <= truncateAtTotalRunes || r.truncating {
+		r.written += textRuneLength
+		r.writer.Header().Add("Warning", header)
+		return
+	}
+
+	// otherwise, enable truncation, reset, and replay the existing items as truncated warnings
+	r.truncating = true
+	r.written = 0
+	r.writer.Header().Del("Warning")
+	utilruntime.HandleError(fmt.Errorf("exceeded max warning header size, truncating"))
+	for _, w := range r.ordered {
+		agent := w.agent
+		text := w.text
+
+		textRuneLength := utf8.RuneCountInString(text)
+		if textRuneLength > truncateItemRunes {
+			text = string([]rune(text)[:truncateItemRunes])
+			textRuneLength = truncateItemRunes
+		}
+		if header, err := net.NewWarningHeader(299, agent, text); err == nil {
+			r.written += textRuneLength
+			r.writer.Header().Add("Warning", header)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go

@@ -25,6 +25,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/errors"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
 	metainternalversionscheme "k8s.io/apimachinery/pkg/apis/meta/internalversion/scheme"
+	metainternalversionvalidation "k8s.io/apimachinery/pkg/apis/meta/internalversion/validation"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -198,6 +199,12 @@ func DeleteCollection(r rest.CollectionDeleter, checkBody bool, scope *RequestSc
 			return
 		}
 
+		if errs := metainternalversionvalidation.ValidateListOptions(&listOptions); len(errs) > 0 {
+			err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "ListOptions"}, "", errs)
+			scope.err(err, w, req)
+			return
+		}
+
 		// transform fields
 		// TODO: DecodeParametersInto should do this.
 		if listOptions.FieldSelector != nil {

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers.go

@@ -23,7 +23,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 type capManagersManager struct {

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go

@@ -26,9 +26,10 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
 
 // DefaultMaxUpdateManagers defines the default maximum retained number of managedFields entries from updates
@@ -78,31 +79,46 @@ func NewFieldManager(f Manager) *FieldManager {
 // NewDefaultFieldManager creates a new FieldManager that merges apply requests
 // and update managed fields for other types of requests.
 func NewDefaultFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion) (*FieldManager, error) {
-	f, err := NewStructuredMergeManager(models, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
+	typeConverter, err := internal.NewTypeConverter(models, false)
+	if err != nil {
+		return nil, err
+	}
+
+	f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, objectCreater, kind), nil
+	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
 }
 
 // NewDefaultCRDFieldManager creates a new FieldManager specifically for
 // CRDs. This allows for the possibility of fields which are not defined
 // in models, as well as having no models defined at all.
 func NewDefaultCRDFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, preserveUnknownFields bool) (_ *FieldManager, err error) {
-	f, err := NewCRDStructuredMergeManager(models, objectConverter, objectDefaulter, kind.GroupVersion(), hub, preserveUnknownFields)
+	var typeConverter internal.TypeConverter = internal.DeducedTypeConverter{}
+	if models != nil {
+		typeConverter, err = internal.NewTypeConverter(models, preserveUnknownFields)
+		if err != nil {
+			return nil, err
+		}
+	}
+	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, preserveUnknownFields)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, objectCreater, kind), nil
+	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
 }
 
 // newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
-func newDefaultFieldManager(f Manager, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind) *FieldManager {
+func newDefaultFieldManager(f Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind) *FieldManager {
 	f = NewStripMetaManager(f)
 	f = NewManagedFieldsUpdater(f)
 	f = NewBuildManagerInfoManager(f, kind.GroupVersion())
 	f = NewCapManagersManager(f, DefaultMaxUpdateManagers)
 	f = NewProbabilisticSkipNonAppliedManager(f, objectCreater, kind, DefaultTrackOnCreateProbability)
+	f = NewLastAppliedManager(f, typeConverter, objectConverter, kind.GroupVersion())
+	f = NewLastAppliedUpdater(f)
+
 	return NewFieldManager(f)
 }
 
@@ -200,7 +216,11 @@ func (f *FieldManager) Apply(liveObj, appliedObj runtime.Object, manager string,
 
 	internal.RemoveObjectManagedFields(liveObj)
 
-	if object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force); err != nil {
+	object, managed, err = f.fieldManager.Apply(liveObj, appliedObj, managed, manager, force)
+	if err != nil {
+		if conflicts, ok := err.(merge.Conflicts); ok {
+			return nil, internal.NewConflictError(conflicts)
+		}
 		return nil, err
 	}
 

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/conflict.go

@@ -25,8 +25,8 @@ import (
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/v3/merge"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
 
 // NewConflictError returns an error including details on the requests apply conflicts

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/fields.go

@@ -21,7 +21,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 // EmptyFields represents a set with no paths

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/gvkparser.go

@@ -22,7 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kube-openapi/pkg/schemaconv"
 	"k8s.io/kube-openapi/pkg/util/proto"
-	"sigs.k8s.io/structured-merge-diff/v3/typed"
+	"sigs.k8s.io/structured-merge-diff/v4/typed"
 )
 
 // groupVersionKindExtensionKey is the key used to lookup the

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/managedfields.go

@@ -24,7 +24,7 @@ import (
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 // ManagedInterface groups a fieldpath.ManagedFields together with the timestamps associated with each operation.

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/pathelement.go

@@ -23,8 +23,8 @@ import (
 	"strconv"
 	"strings"
 
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/v3/value"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/value"
 )
 
 const (

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/typeconverter.go

@@ -23,8 +23,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/kube-openapi/pkg/util/proto"
-	"sigs.k8s.io/structured-merge-diff/v3/typed"
-	"sigs.k8s.io/structured-merge-diff/v3/value"
+	"sigs.k8s.io/structured-merge-diff/v4/typed"
+	"sigs.k8s.io/structured-merge-diff/v4/value"
 )
 
 // TypeConverter allows you to convert from runtime.Object to

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/versionconverter.go

@@ -19,9 +19,9 @@ package internal
 import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/v3/merge"
-	"sigs.k8s.io/structured-merge-diff/v3/typed"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
+	"sigs.k8s.io/structured-merge-diff/v4/typed"
 )
 
 // versionConverter is an implementation of

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager.go

@@ -0,0 +1,173 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fieldmanager
+
+import (
+	"encoding/json"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
+)
+
+type lastAppliedManager struct {
+	fieldManager    Manager
+	typeConverter   internal.TypeConverter
+	objectConverter runtime.ObjectConvertor
+	groupVersion    schema.GroupVersion
+}
+
+var _ Manager = &lastAppliedManager{}
+
+// NewLastAppliedManager converts the client-side apply annotation to
+// server-side apply managed fields
+func NewLastAppliedManager(fieldManager Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, groupVersion schema.GroupVersion) Manager {
+	return &lastAppliedManager{
+		fieldManager:    fieldManager,
+		typeConverter:   typeConverter,
+		objectConverter: objectConverter,
+		groupVersion:    groupVersion,
+	}
+}
+
+// Update implements Manager.
+func (f *lastAppliedManager) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
+	return f.fieldManager.Update(liveObj, newObj, managed, manager)
+}
+
+// Apply will consider the last-applied annotation
+// for upgrading an object managed by client-side apply to server-side apply
+// without conflicts.
+func (f *lastAppliedManager) Apply(liveObj, newObj runtime.Object, managed Managed, manager string, force bool) (runtime.Object, Managed, error) {
+	newLiveObj, newManaged, newErr := f.fieldManager.Apply(liveObj, newObj, managed, manager, force)
+	// Upgrade the client-side apply annotation only from kubectl server-side-apply.
+	// To opt-out of this behavior, users may specify a different field manager.
+	if manager != "kubectl" {
+		return newLiveObj, newManaged, newErr
+	}
+
+	// Check if we have conflicts
+	if newErr == nil {
+		return newLiveObj, newManaged, newErr
+	}
+	conflicts, ok := newErr.(merge.Conflicts)
+	if !ok {
+		return newLiveObj, newManaged, newErr
+	}
+	conflictSet := conflictsToSet(conflicts)
+
+	// Check if conflicts are allowed due to client-side apply,
+	// and if so, then force apply
+	allowedConflictSet, err := f.allowedConflictsFromLastApplied(liveObj)
+	if err != nil {
+		return newLiveObj, newManaged, newErr
+	}
+	if !conflictSet.Difference(allowedConflictSet).Empty() {
+		newConflicts := conflictsDifference(conflicts, allowedConflictSet)
+		return newLiveObj, newManaged, newConflicts
+	}
+
+	return f.fieldManager.Apply(liveObj, newObj, managed, manager, true)
+}
+
+func (f *lastAppliedManager) allowedConflictsFromLastApplied(liveObj runtime.Object) (*fieldpath.Set, error) {
+	var accessor, err = meta.Accessor(liveObj)
+	if err != nil {
+		panic(fmt.Sprintf("couldn't get accessor: %v", err))
+	}
+
+	// If there is no client-side apply annotation, then there is nothing to do
+	var annotations = accessor.GetAnnotations()
+	if annotations == nil {
+		return nil, fmt.Errorf("no last applied annotation")
+	}
+	var lastApplied, ok = annotations[corev1.LastAppliedConfigAnnotation]
+	if !ok || lastApplied == "" {
+		return nil, fmt.Errorf("no last applied annotation")
+	}
+
+	liveObjVersioned, err := f.objectConverter.ConvertToVersion(liveObj, f.groupVersion)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert live obj to versioned: %v", err)
+	}
+
+	liveObjTyped, err := f.typeConverter.ObjectToTyped(liveObjVersioned)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert live obj to typed: %v", err)
+	}
+
+	var lastAppliedObj = &unstructured.Unstructured{Object: map[string]interface{}{}}
+	err = json.Unmarshal([]byte(lastApplied), lastAppliedObj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decode last applied obj: %v in '%s'", err, lastApplied)
+	}
+
+	if lastAppliedObj.GetAPIVersion() != f.groupVersion.String() {
+		return nil, fmt.Errorf("expected version of last applied to match live object '%s', but got '%s': %v", f.groupVersion.String(), lastAppliedObj.GetAPIVersion(), err)
+	}
+
+	lastAppliedObjTyped, err := f.typeConverter.ObjectToTyped(lastAppliedObj)
+	if err != nil {
+		return nil, fmt.Errorf("failed to convert last applied to typed: %v", err)
+	}
+
+	lastAppliedObjFieldSet, err := lastAppliedObjTyped.ToFieldSet()
+	if err != nil {
+		return nil, fmt.Errorf("failed to create fieldset for last applied object: %v", err)
+	}
+
+	comparison, err := lastAppliedObjTyped.Compare(liveObjTyped)
+	if err != nil {
+		return nil, fmt.Errorf("failed to compare last applied object and live object: %v", err)
+	}
+
+	// Remove fields in last applied that are different, added, or missing in
+	// the live object.
+	// Because last-applied fields don't match the live object fields,
+	// then we don't own these fields.
+	lastAppliedObjFieldSet = lastAppliedObjFieldSet.
+		Difference(comparison.Modified).
+		Difference(comparison.Added).
+		Difference(comparison.Removed)
+
+	return lastAppliedObjFieldSet, nil
+}
+
+// TODO: replace with merge.Conflicts.ToSet()
+func conflictsToSet(conflicts merge.Conflicts) *fieldpath.Set {
+	conflictSet := fieldpath.NewSet()
+	for _, conflict := range []merge.Conflict(conflicts) {
+		conflictSet.Insert(conflict.Path)
+	}
+	return conflictSet
+}
+
+func conflictsDifference(conflicts merge.Conflicts, s *fieldpath.Set) merge.Conflicts {
+	newConflicts := []merge.Conflict{}
+	for _, conflict := range []merge.Conflict(conflicts) {
+		if !s.Has(conflict.Path) {
+			newConflicts = append(newConflicts, conflict)
+		}
+	}
+	return newConflicts
+}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedupdater.go

@@ -0,0 +1,117 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fieldmanager
+
+import (
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/meta"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+type lastAppliedUpdater struct {
+	fieldManager Manager
+}
+
+var _ Manager = &lastAppliedUpdater{}
+
+// NewLastAppliedUpdater sets the client-side apply annotation up to date with
+// server-side apply managed fields
+func NewLastAppliedUpdater(fieldManager Manager) Manager {
+	return &lastAppliedUpdater{
+		fieldManager: fieldManager,
+	}
+}
+
+// Update implements Manager.
+func (f *lastAppliedUpdater) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
+	return f.fieldManager.Update(liveObj, newObj, managed, manager)
+}
+
+// server-side apply managed fields
+func (f *lastAppliedUpdater) Apply(liveObj, newObj runtime.Object, managed Managed, manager string, force bool) (runtime.Object, Managed, error) {
+	liveObj, managed, err := f.fieldManager.Apply(liveObj, newObj, managed, manager, force)
+	if err != nil {
+		return liveObj, managed, err
+	}
+
+	// Sync the client-side apply annotation only from kubectl server-side apply.
+	// To opt-out of this behavior, users may specify a different field manager.
+	//
+	// If the client-side apply annotation doesn't exist,
+	// then continue because we have no annotation to update
+	if manager == "kubectl" && hasLastApplied(liveObj) {
+		lastAppliedValue, err := buildLastApplied(newObj)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to build last-applied annotation: %v", err)
+		}
+		err = setLastApplied(liveObj, lastAppliedValue)
+		if err != nil {
+			return nil, nil, fmt.Errorf("failed to set last-applied annotation: %v", err)
+		}
+	}
+	return liveObj, managed, err
+}
+
+func hasLastApplied(obj runtime.Object) bool {
+	var accessor, err = meta.Accessor(obj)
+	if err != nil {
+		panic(fmt.Sprintf("couldn't get accessor: %v", err))
+	}
+	var annotations = accessor.GetAnnotations()
+	if annotations == nil {
+		return false
+	}
+	_, ok := annotations[corev1.LastAppliedConfigAnnotation]
+	return ok
+}
+
+func setLastApplied(obj runtime.Object, value string) error {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		panic(fmt.Sprintf("couldn't get accessor: %v", err))
+	}
+	var annotations = accessor.GetAnnotations()
+	if annotations == nil {
+		annotations = map[string]string{}
+	}
+	annotations[corev1.LastAppliedConfigAnnotation] = value
+	accessor.SetAnnotations(annotations)
+	return nil
+}
+
+func buildLastApplied(obj runtime.Object) (string, error) {
+	obj = obj.DeepCopyObject()
+
+	var accessor, err = meta.Accessor(obj)
+	if err != nil {
+		panic(fmt.Sprintf("couldn't get accessor: %v", err))
+	}
+
+	// Remove the annotation from the object before encoding the object
+	var annotations = accessor.GetAnnotations()
+	delete(annotations, corev1.LastAppliedConfigAnnotation)
+	accessor.SetAnnotations(annotations)
+
+	lastApplied, err := runtime.Encode(unstructured.UnstructuredJSONScheme, obj)
+	if err != nil {
+		return "", fmt.Errorf("couldn't encode object into last applied annotation: %v", err)
+	}
+	return string(lastApplied), nil
+}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater.go

@@ -21,7 +21,7 @@ import (
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 type managedFieldsUpdater struct {

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/stripmeta.go

@@ -20,7 +20,7 @@ import (
 	"fmt"
 
 	"k8s.io/apimachinery/pkg/runtime"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 )
 
 type stripMetaManager struct {

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/structuredmerge.go

@@ -24,9 +24,8 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
-	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
-	"sigs.k8s.io/structured-merge-diff/v3/fieldpath"
-	"sigs.k8s.io/structured-merge-diff/v3/merge"
+	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
+	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
 
 type structuredMergeManager struct {
@@ -42,12 +41,7 @@ var _ Manager = &structuredMergeManager{}
 
 // NewStructuredMergeManager creates a new Manager that merges apply requests
 // and update managed fields for other types of requests.
-func NewStructuredMergeManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
-	typeConverter, err := internal.NewTypeConverter(models, false)
-	if err != nil {
-		return nil, err
-	}
-
+func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
 	return &structuredMergeManager{
 		typeConverter:   typeConverter,
 		objectConverter: objectConverter,
@@ -55,7 +49,7 @@ func NewStructuredMergeManager(models openapiproto.Models, objectConverter runti
 		groupVersion:    gv,
 		hubVersion:      hub,
 		updater: merge.Updater{
-			Converter: internal.NewVersionConverter(typeConverter, objectConverter, hub),
+			Converter: internal.NewVersionConverter(typeConverter, objectConverter, hub), // This is the converter provided to SMD from k8s
 		},
 	}, nil
 }
@@ -63,14 +57,7 @@ func NewStructuredMergeManager(models openapiproto.Models, objectConverter runti
 // NewCRDStructuredMergeManager creates a new Manager specifically for
 // CRDs. This allows for the possibility of fields which are not defined
 // in models, as well as having no models defined at all.
-func NewCRDStructuredMergeManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion, preserveUnknownFields bool) (_ Manager, err error) {
-	var typeConverter internal.TypeConverter = internal.DeducedTypeConverter{}
-	if models != nil {
-		typeConverter, err = internal.NewTypeConverter(models, preserveUnknownFields)
-		if err != nil {
-			return nil, err
-		}
-	}
+func NewCRDStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion, preserveUnknownFields bool) (_ Manager, err error) {
 	return &structuredMergeManager{
 		typeConverter:   typeConverter,
 		objectConverter: objectConverter,
@@ -149,9 +136,6 @@ func (f *structuredMergeManager) Apply(liveObj, patchObj runtime.Object, managed
 	apiVersion := fieldpath.APIVersion(f.groupVersion.String())
 	newObjTyped, managedFields, err := f.updater.Apply(liveObjTyped, patchObjTyped, apiVersion, managed.Fields(), manager, force)
 	if err != nil {
-		if conflicts, ok := err.(merge.Conflicts); ok {
-			return nil, nil, internal.NewConflictError(conflicts)
-		}
 		return nil, nil, err
 	}
 	managed = internal.NewManaged(managedFields, managed.Times())

vendor/k8s.io/apiserver/pkg/endpoints/handlers/get.go

@@ -19,13 +19,15 @@ package handlers
 import (
 	"context"
 	"fmt"
+	metainternalversionvalidation "k8s.io/apimachinery/pkg/apis/meta/internalversion/validation"
+	"k8s.io/apimachinery/pkg/runtime/schema"
 	"math/rand"
 	"net/http"
 	"net/url"
 	"strings"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -198,6 +200,12 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
 			return
 		}
 
+		if errs := metainternalversionvalidation.ValidateListOptions(&opts); len(errs) > 0 {
+			err := errors.NewInvalid(schema.GroupKind{Group: metav1.GroupName, Kind: "ListOptions"}, "", errs)
+			scope.err(err, w, req)
+			return
+		}
+
 		// transform fields
 		// TODO: DecodeParametersInto should do this.
 		if opts.FieldSelector != nil {
@@ -248,7 +256,7 @@ func ListResource(r rest.Lister, rw rest.Watcher, scope *RequestScope, forceWatc
 			if timeout == 0 && minRequestTimeout > 0 {
 				timeout = time.Duration(float64(minRequestTimeout) * (rand.Float64() + 1.0))
 			}
-			klog.V(3).Infof("Starting watch for %s, rv=%s labels=%s fields=%s timeout=%s", req.URL.Path, opts.ResourceVersion, opts.LabelSelector, opts.FieldSelector, timeout)
+			klog.V(3).InfoS("Starting watch", "path", req.URL.Path, "resourceVersion", opts.ResourceVersion, "labels", opts.LabelSelector, "fields", opts.FieldSelector, "timeout", timeout)
 			ctx, cancel := context.WithTimeout(ctx, timeout)
 			defer cancel()
 			watcher, err := rw.Watch(ctx, &opts)

vendor/k8s.io/apiserver/pkg/endpoints/handlers/patch.go

@@ -421,6 +421,7 @@ type applyPatcher struct {
 	creater      runtime.ObjectCreater
 	kind         schema.GroupVersionKind
 	fieldManager *fieldmanager.FieldManager
+	userAgent    string
 }
 
 func (p *applyPatcher) applyPatchToCurrentObject(obj runtime.Object) (runtime.Object, error) {
@@ -569,6 +570,7 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
 			options:      p.options,
 			creater:      p.creater,
 			kind:         p.kind,
+			userAgent:    p.userAgent,
 		}
 		p.forceAllowCreate = true
 	default:

vendor/k8s.io/apiserver/pkg/endpoints/handlers/rest.go

@@ -46,7 +46,7 @@ import (
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // RequestScope encapsulates common fields across all RESTful handler methods.

vendor/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -32,15 +32,19 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/apiserver/pkg/endpoints/deprecation"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
 	"k8s.io/apiserver/pkg/endpoints/handlers"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/endpoints/handlers/negotiation"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
+	utilwarning "k8s.io/apiserver/pkg/endpoints/warning"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	versioninfo "k8s.io/component-base/version"
 )
 
 const (
@@ -381,7 +385,11 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		resourceKind = kind
 	}
 
-	tableProvider, _ := storage.(rest.TableConvertor)
+	tableProvider, isTableProvider := storage.(rest.TableConvertor)
+	if isLister && !isTableProvider {
+		// All listers must implement TableProvider
+		return nil, fmt.Errorf("%q must implement TableConvertor", resource)
+	}
 
 	var apiResource metav1.APIResource
 	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionHash) &&
@@ -617,6 +625,26 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 		verbOverrider, needOverride := storage.(StorageMetricsOverride)
 
+		// accumulate endpoint-level warnings
+		var (
+			enableWarningHeaders = utilfeature.DefaultFeatureGate.Enabled(features.WarningHeaders)
+
+			warnings       []string
+			deprecated     bool
+			removedRelease string
+		)
+
+		{
+			versionedPtrWithGVK := versionedPtr.DeepCopyObject()
+			versionedPtrWithGVK.GetObjectKind().SetGroupVersionKind(fqKindToRegister)
+			currentMajor, currentMinor, _ := deprecation.MajorMinor(versioninfo.Get())
+			deprecated = deprecation.IsDeprecated(versionedPtrWithGVK, currentMajor, currentMinor)
+			if deprecated {
+				removedRelease = deprecation.RemovedRelease(versionedPtrWithGVK)
+				warnings = append(warnings, deprecation.WarningMessage(versionedPtrWithGVK))
+			}
+		}
+
 		switch action.Verb {
 		case "GET": // Get a resource.
 			var handler restful.RouteFunction
@@ -628,9 +656,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 			if needOverride {
 				// need change the reported verb
-				handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)
+				handler = metrics.InstrumentRouteFunc(verbOverrider.OverrideMetricsVerb(action.Verb), group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
 			} else {
-				handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)
+				handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
+			}
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
 			}
 
 			doc := "read the specified " + kind
@@ -661,7 +692,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if isSubresource {
 				doc = "list " + subresource + " of objects of kind " + kind
 			}
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, false, a.minRequestTimeout))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -693,7 +727,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if isSubresource {
 				doc = "replace " + subresource + " of the specified " + kind
 			}
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulUpdateResource(updater, reqScope, admit))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulUpdateResource(updater, reqScope, admit))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.PUT(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -723,7 +760,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
 				supportedTypes = append(supportedTypes, string(types.ApplyPatchType))
 			}
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulPatchResource(patcher, reqScope, admit, supportedTypes))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.PATCH(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -745,7 +785,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			} else {
 				handler = restfulCreateResource(creater, reqScope, admit)
 			}
-			handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, handler)
+			handler = metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, handler)
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			article := GetArticleForNoun(kind, " ")
 			doc := "create" + article + kind
 			if isSubresource {
@@ -778,7 +821,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if deleteReturnsDeletedObject {
 				deleteReturnType = producedObject
 			}
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteResource(gracefulDeleter, isGracefulDeleter, reqScope, admit))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.DELETE(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -801,7 +847,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if isSubresource {
 				doc = "delete collection of " + subresource + " of a " + kind
 			}
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulDeleteCollection(collectionDeleter, isCollectionDeleter, reqScope, admit))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.DELETE(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -816,7 +865,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 					return nil, err
 				}
 			}
-			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
+			if err := AddObjectParams(ws, route, versionedListOptions, "watch", "allowWatchBookmarks"); err != nil {
 				return nil, err
 			}
 			addParams(route, action.Params)
@@ -828,7 +877,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				doc = "watch changes to " + subresource + " of an object of kind " + kind
 			}
 			doc += ". deprecated: use the 'watch' parameter with a list operation instead, filtered to a single item with the 'fieldSelector' parameter."
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -848,7 +900,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				doc = "watch individual changes to a list of " + subresource + " of " + kind
 			}
 			doc += ". deprecated: use the 'watch' parameter with a list operation instead."
-			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
+			handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulListResource(lister, watcher, reqScope, true, a.minRequestTimeout))
+			if enableWarningHeaders {
+				handler = utilwarning.AddWarningsHandler(handler, warnings)
+			}
 			route := ws.GET(action.Path).To(handler).
 				Doc(doc).
 				Param(ws.QueryParameter("pretty", "If 'true', then the output is pretty printed.")).
@@ -871,7 +926,10 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				if isSubresource {
 					doc = "connect " + method + " requests to " + subresource + " of " + kind
 				}
-				handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
+				handler := metrics.InstrumentRouteFunc(action.Verb, group, version, resource, subresource, requestScope, metrics.APIServerComponent, deprecated, removedRelease, restfulConnectResource(connecter, reqScope, admit, path, isSubresource))
+				if enableWarningHeaders {
+					handler = utilwarning.AddWarningsHandler(handler, warnings)
+				}
 				route := ws.Method(method).Path(action.Path).
 					To(handler).
 					Doc(doc).
@@ -958,12 +1016,13 @@ func addParams(route *restful.RouteBuilder, params []*restful.Parameter) {
 // Go JSON behavior for omitting a field) become query parameters. The name of the query parameter is
 // the JSON field name. If a description struct tag is set on the field, that description is used on the
 // query parameter. In essence, it converts a standard JSON top level object into a query param schema.
-func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}) error {
+func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj interface{}, excludedNames ...string) error {
 	sv, err := conversion.EnforcePtr(obj)
 	if err != nil {
 		return err
 	}
 	st := sv.Type()
+	excludedNameSet := sets.NewString(excludedNames...)
 	switch st.Kind() {
 	case reflect.Struct:
 		for i := 0; i < st.NumField(); i++ {
@@ -989,7 +1048,9 @@ func AddObjectParams(ws *restful.WebService, route *restful.RouteBuilder, obj in
 				if len(jsonName) == 0 {
 					continue
 				}
-
+				if excludedNameSet.Has(jsonName) {
+					continue
+				}
 				var desc string
 				if docable, ok := obj.(documentable); ok {
 					desc = docable.SwaggerDoc()[jsonName]
@@ -1017,6 +1078,10 @@ func typeToJSON(typeName string) string {
 		return "string"
 	case "v1.DeletionPropagation", "*v1.DeletionPropagation":
 		return "string"
+	case "v1.ResourceVersionMatch", "*v1.ResourceVersionMatch":
+		return "string"
+	case "v1.IncludeObjectPolicy", "*v1.IncludeObjectPolicy":
+		return "string"
 
 	// TODO: Fix these when go-restful supports a way to specify an array query param:
 	// https://github.com/emicklei/go-restful/issues/225
@@ -1062,7 +1127,7 @@ func splitSubresource(path string) (string, string, error) {
 
 // GetArticleForNoun returns the article needed for the given noun.
 func GetArticleForNoun(noun string, padding string) string {
-	if noun[len(noun)-2:] != "ss" && noun[len(noun)-1:] == "s" {
+	if !strings.HasSuffix(noun, "ss") && strings.HasSuffix(noun, "s") {
 		// Plurals don't have an article.
 		// Don't catch words like class
 		return fmt.Sprintf("%v", padding)

vendor/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -31,6 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/validation"
 	"k8s.io/apimachinery/pkg/types"
 	utilsets "k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -60,6 +61,15 @@ const (
  * the metric stability policy.
  */
 var (
+	deprecatedRequestGauge = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Name:           "apiserver_requested_deprecated_apis",
+			Help:           "Gauge of deprecated APIs that have been requested, broken out by API group, version, resource, subresource, and removed_release.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"group", "version", "resource", "subresource", "removed_release"},
+	)
+
 	// TODO(a-robinson): Add unit tests for the handling of these metrics once
 	// the upstream library supports it.
 	requestCounter = compbasemetrics.NewCounterVec(
@@ -112,7 +122,15 @@ var (
 			Help:           "Number of requests dropped with 'Try again later' response",
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
-		[]string{"requestKind"},
+		[]string{"request_kind"},
+	)
+	// TLSHandshakeErrors is a number of requests dropped with 'TLS handshake error from' error
+	TLSHandshakeErrors = compbasemetrics.NewCounter(
+		&compbasemetrics.CounterOpts{
+			Name:           "apiserver_tls_handshake_errors_total",
+			Help:           "Number of requests dropped with 'TLS handshake error from' error",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
 	)
 	// RegisteredWatchers is a number of currently registered watchers splitted by resource.
 	RegisteredWatchers = compbasemetrics.NewGaugeVec(
@@ -148,7 +166,15 @@ var (
 			Help:           "Maximal number of currently used inflight request limit of this apiserver per request kind in last second.",
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
-		[]string{"requestKind"},
+		[]string{"request_kind"},
+	)
+	currentInqueueRequests = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Name:           "apiserver_current_inqueue_requests",
+			Help:           "Maximal number of queued requests in this apiserver per request kind in last second.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"request_kind"},
 	)
 
 	requestTerminationsTotal = compbasemetrics.NewCounterVec(
@@ -162,15 +188,18 @@ var (
 	kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`)
 
 	metrics = []resettableCollector{
+		deprecatedRequestGauge,
 		requestCounter,
 		longRunningRequestGauge,
 		requestLatencies,
 		responseSizes,
 		DroppedRequests,
+		TLSHandshakeErrors,
 		RegisteredWatchers,
 		WatchEvents,
 		WatchEventsSizes,
 		currentInflightRequests,
+		currentInqueueRequests,
 		requestTerminationsTotal,
 	}
 
@@ -211,6 +240,21 @@ const (
 	ReadOnlyKind = "readOnly"
 	// MutatingKind is a string identifying mutating request kind
 	MutatingKind = "mutating"
+
+	// WaitingPhase is the phase value for a request waiting in a queue
+	WaitingPhase = "waiting"
+	// ExecutingPhase is the phase value for an executing request
+	ExecutingPhase = "executing"
+)
+
+const (
+	// deprecatedAnnotationKey is a key for an audit annotation set to
+	// "true" on requests made to deprecated API versions
+	deprecatedAnnotationKey = "k8s.io/deprecated"
+	// removedReleaseAnnotationKey is a key for an audit annotation set to
+	// the target removal release, in "<major>.<minor>" format,
+	// on requests made to deprecated API versions with a target removal release
+	removedReleaseAnnotationKey = "k8s.io/removed-release"
 )
 
 var registerMetrics sync.Once
@@ -231,9 +275,19 @@ func Reset() {
 	}
 }
 
-func UpdateInflightRequestMetrics(nonmutating, mutating int) {
-	currentInflightRequests.WithLabelValues(ReadOnlyKind).Set(float64(nonmutating))
-	currentInflightRequests.WithLabelValues(MutatingKind).Set(float64(mutating))
+// UpdateInflightRequestMetrics reports concurrency metrics classified by
+// mutating vs Readonly.
+func UpdateInflightRequestMetrics(phase string, nonmutating, mutating int) {
+	for _, kc := range []struct {
+		kind  string
+		count int
+	}{{ReadOnlyKind, nonmutating}, {MutatingKind, mutating}} {
+		if phase == ExecutingPhase {
+			currentInflightRequests.WithLabelValues(kc.kind).Set(float64(kc.count))
+		} else {
+			currentInqueueRequests.WithLabelValues(kc.kind).Set(float64(kc.count))
+		}
+	}
 }
 
 // RecordRequestTermination records that the request was terminated early as part of a resource
@@ -288,12 +342,19 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
 
 // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
 // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
-func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component, contentType string, httpCode, respSize int, elapsed time.Duration) {
+func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, contentType string, httpCode, respSize int, elapsed time.Duration) {
 	reportedVerb := cleanVerb(verb, req)
 	dryRun := cleanDryRun(req.URL)
 	elapsedSeconds := elapsed.Seconds()
 	cleanContentType := cleanContentType(contentType)
 	requestCounter.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, cleanContentType, codeToString(httpCode)).Inc()
+	if deprecated {
+		deprecatedRequestGauge.WithLabelValues(group, version, resource, subresource, removedRelease).Set(1)
+		audit.AddAuditAnnotation(req.Context(), deprecatedAnnotationKey, "true")
+		if len(removedRelease) > 0 {
+			audit.AddAuditAnnotation(req.Context(), removedReleaseAnnotationKey, removedRelease)
+		}
+	}
 	requestLatencies.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component).Observe(elapsedSeconds)
 	// We are only interested in response sizes of read requests.
 	if verb == "GET" || verb == "LIST" {
@@ -303,7 +364,7 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
 
 // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps
 // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information.
-func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, routeFunc restful.RouteFunction) restful.RouteFunction {
+func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, routeFunc restful.RouteFunction) restful.RouteFunction {
 	return restful.RouteFunction(func(request *restful.Request, response *restful.Response) {
 		now := time.Now()
 
@@ -322,12 +383,12 @@ func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, com
 
 		routeFunc(request, response)
 
-		MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
+		MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
 	})
 }
 
 // InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information.
-func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, handler http.HandlerFunc) http.HandlerFunc {
+func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, handler http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		now := time.Now()
 
@@ -344,7 +405,7 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
 
 		handler(w, req)
 
-		MonitorRequest(req, verb, group, version, resource, subresource, scope, component, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
+		MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
 	}
 }
 

vendor/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go

@@ -28,7 +28,7 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/sets"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // LongRunningRequestCheck is a predicate which is true for long-running http requests.

vendor/k8s.io/apiserver/pkg/endpoints/warning/warning.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package warning
+
+import (
+	restful "github.com/emicklei/go-restful"
+
+	"k8s.io/apiserver/pkg/warning"
+)
+
+// AddWarningsHandler returns a handler that adds the provided warnings to all requests,
+// then delegates to the provided handler.
+func AddWarningsHandler(handler restful.RouteFunction, warnings []string) restful.RouteFunction {
+	if len(warnings) == 0 {
+		return handler
+	}
+
+	return func(req *restful.Request, res *restful.Response) {
+		ctx := req.Request.Context()
+		for _, msg := range warnings {
+			warning.AddWarning(ctx, "", msg)
+		}
+		handler(req, res)
+	}
+}

vendor/k8s.io/apiserver/pkg/features/kube_features.go

@@ -59,13 +59,6 @@ const (
 	// audited.
 	AdvancedAuditing featuregate.Feature = "AdvancedAuditing"
 
-	// owner: @pbarker
-	// alpha: v1.13
-	//
-	// DynamicAuditing enables configuration of audit policy and webhook backends through an
-	// AuditSink API object.
-	DynamicAuditing featuregate.Feature = "DynamicAuditing"
-
 	// owner: @ilackams
 	// alpha: v1.7
 	//
@@ -83,6 +76,7 @@ const (
 	// owner: @apelisse
 	// alpha: v1.12
 	// beta: v1.13
+	// stable: v1.18
 	//
 	// Allow requests to be processed but not stored, so that
 	// validation, merging, mutation can be tested without
@@ -111,18 +105,6 @@ const (
 	// document.
 	StorageVersionHash featuregate.Feature = "StorageVersionHash"
 
-	// owner: @ksubrmnn
-	// alpha: v1.14
-	//
-	// Allows kube-proxy to run in Overlay mode for Windows
-	WinOverlay featuregate.Feature = "WinOverlay"
-
-	// owner: @ksubrmnn
-	// alpha: v1.14
-	//
-	// Allows kube-proxy to create DSR loadbalancers for Windows
-	WinDSR featuregate.Feature = "WinDSR"
-
 	// owner: @wojtek-t
 	// alpha: v1.15
 	// beta: v1.16
@@ -144,11 +126,18 @@ const (
 	// Deprecates and removes SelfLink from ObjectMeta and ListMeta.
 	RemoveSelfLink featuregate.Feature = "RemoveSelfLink"
 
-	// owner: @shaloulcy
+	// owner: @shaloulcy, @wojtek-t
 	// alpha: v1.18
+	// beta: v1.19
 	//
 	// Allows label and field based indexes in apiserver watch cache to accelerate list operations.
 	SelectorIndex featuregate.Feature = "SelectorIndex"
+
+	// owner: @liggitt
+	// beta: v1.19
+	//
+	// Allows sending warning headers in API responses.
+	WarningHeaders featuregate.Feature = "WarningHeaders"
 )
 
 func init() {
@@ -162,17 +151,15 @@ var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureS
 	StreamingProxyRedirects: {Default: true, PreRelease: featuregate.Deprecated},
 	ValidateProxyRedirects:  {Default: true, PreRelease: featuregate.Beta},
 	AdvancedAuditing:        {Default: true, PreRelease: featuregate.GA},
-	DynamicAuditing:         {Default: false, PreRelease: featuregate.Alpha},
 	APIResponseCompression:  {Default: true, PreRelease: featuregate.Beta},
 	APIListChunking:         {Default: true, PreRelease: featuregate.Beta},
-	DryRun:                  {Default: true, PreRelease: featuregate.Beta},
+	DryRun:                  {Default: true, PreRelease: featuregate.GA},
 	RemainingItemCount:      {Default: true, PreRelease: featuregate.Beta},
 	ServerSideApply:         {Default: true, PreRelease: featuregate.Beta},
 	StorageVersionHash:      {Default: true, PreRelease: featuregate.Beta},
-	WinOverlay:              {Default: false, PreRelease: featuregate.Alpha},
-	WinDSR:                  {Default: false, PreRelease: featuregate.Alpha},
 	WatchBookmark:           {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
 	APIPriorityAndFairness:  {Default: false, PreRelease: featuregate.Alpha},
 	RemoveSelfLink:          {Default: false, PreRelease: featuregate.Alpha},
-	SelectorIndex:           {Default: false, PreRelease: featuregate.Alpha},
+	SelectorIndex:           {Default: true, PreRelease: featuregate.Beta},
+	WarningHeaders:          {Default: true, PreRelease: featuregate.Beta},
 }

vendor/k8s.io/apiserver/pkg/registry/generic/OWNERS

@@ -18,7 +18,6 @@ reviewers:
 - janetkuo
 - pwittrock
 - ncdc
-- eparis
 - piosz
 - dims
 - hongchaodeng

vendor/k8s.io/apiserver/pkg/registry/generic/registry/dryrun.go

@@ -35,7 +35,7 @@ func (s *DryRunnableStorage) Versioner() storage.Versioner {
 
 func (s *DryRunnableStorage) Create(ctx context.Context, key string, obj, out runtime.Object, ttl uint64, dryRun bool) error {
 	if dryRun {
-		if err := s.Storage.Get(ctx, key, "", out, false); err == nil {
+		if err := s.Storage.Get(ctx, key, storage.GetOptions{}, out); err == nil {
 			return storage.NewKeyExistsError(key, 0)
 		}
 		return s.copyInto(obj, out)
@@ -45,7 +45,7 @@ func (s *DryRunnableStorage) Create(ctx context.Context, key string, obj, out ru
 
 func (s *DryRunnableStorage) Delete(ctx context.Context, key string, out runtime.Object, preconditions *storage.Preconditions, deleteValidation storage.ValidateObjectFunc, dryRun bool) error {
 	if dryRun {
-		if err := s.Storage.Get(ctx, key, "", out, false); err != nil {
+		if err := s.Storage.Get(ctx, key, storage.GetOptions{}, out); err != nil {
 			return err
 		}
 		if err := preconditions.Check(key, out); err != nil {
@@ -56,31 +56,31 @@ func (s *DryRunnableStorage) Delete(ctx context.Context, key string, out runtime
 	return s.Storage.Delete(ctx, key, out, preconditions, deleteValidation)
 }
 
-func (s *DryRunnableStorage) Watch(ctx context.Context, key string, resourceVersion string, p storage.SelectionPredicate) (watch.Interface, error) {
-	return s.Storage.Watch(ctx, key, resourceVersion, p)
+func (s *DryRunnableStorage) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
+	return s.Storage.Watch(ctx, key, opts)
 }
 
-func (s *DryRunnableStorage) WatchList(ctx context.Context, key string, resourceVersion string, p storage.SelectionPredicate) (watch.Interface, error) {
-	return s.Storage.WatchList(ctx, key, resourceVersion, p)
+func (s *DryRunnableStorage) WatchList(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
+	return s.Storage.WatchList(ctx, key, opts)
 }
 
-func (s *DryRunnableStorage) Get(ctx context.Context, key string, resourceVersion string, objPtr runtime.Object, ignoreNotFound bool) error {
-	return s.Storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
+func (s *DryRunnableStorage) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error {
+	return s.Storage.Get(ctx, key, opts, objPtr)
 }
 
-func (s *DryRunnableStorage) GetToList(ctx context.Context, key string, resourceVersion string, p storage.SelectionPredicate, listObj runtime.Object) error {
-	return s.Storage.GetToList(ctx, key, resourceVersion, p, listObj)
+func (s *DryRunnableStorage) GetToList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
+	return s.Storage.GetToList(ctx, key, opts, listObj)
 }
 
-func (s *DryRunnableStorage) List(ctx context.Context, key string, resourceVersion string, p storage.SelectionPredicate, listObj runtime.Object) error {
-	return s.Storage.List(ctx, key, resourceVersion, p, listObj)
+func (s *DryRunnableStorage) List(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
+	return s.Storage.List(ctx, key, opts, listObj)
 }
 
 func (s *DryRunnableStorage) GuaranteedUpdate(
 	ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
 	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, dryRun bool, suggestion ...runtime.Object) error {
 	if dryRun {
-		err := s.Storage.Get(ctx, key, "", ptrToType, ignoreNotFound)
+		err := s.Storage.Get(ctx, key, storage.GetOptions{IgnoreNotFound: ignoreNotFound}, ptrToType)
 		if err != nil {
 			return err
 		}

vendor/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go

@@ -17,10 +17,12 @@ limitations under the License.
 package registry
 
 import (
+	"fmt"
 	"sync"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/storage"
@@ -32,7 +34,7 @@ import (
 )
 
 // Creates a cacher based given storageConfig.
-func StorageWithCacher(capacity int) generic.StorageDecorator {
+func StorageWithCacher() generic.StorageDecorator {
 	return func(
 		storageConfig *storagebackend.Config,
 		resourcePrefix string,
@@ -47,18 +49,11 @@ func StorageWithCacher(capacity int) generic.StorageDecorator {
 		if err != nil {
 			return s, d, err
 		}
-		if capacity <= 0 {
-			klog.V(5).Infof("Storage caching is disabled for %T", newFunc())
-			return s, d, nil
-		}
-		if klog.V(5) {
-			klog.Infof("Storage caching is enabled for %T with capacity %v", newFunc(), capacity)
+		if klog.V(5).Enabled() {
+			klog.Infof("Storage caching is enabled for %s", objectTypeToString(newFunc()))
 		}
 
-		// TODO: we would change this later to make storage always have cacher and hide low level KV layer inside.
-		// Currently it has two layers of same storage interface -- cacher and low level kv.
 		cacherConfig := cacherstorage.Config{
-			CacheCapacity:  capacity,
 			Storage:        s,
 			Versioner:      etcd3.APIObjectVersioner{},
 			ResourcePrefix: resourcePrefix,
@@ -88,6 +83,17 @@ func StorageWithCacher(capacity int) generic.StorageDecorator {
 	}
 }
 
+func objectTypeToString(obj runtime.Object) string {
+	// special-case unstructured objects that tell us their apiVersion/kind
+	if u, isUnstructured := obj.(*unstructured.Unstructured); isUnstructured {
+		if apiVersion, kind := u.GetAPIVersion(), u.GetKind(); len(apiVersion) > 0 && len(kind) > 0 {
+			return fmt.Sprintf("apiVersion=%s, kind=%s", apiVersion, kind)
+		}
+	}
+	// otherwise just return the type
+	return fmt.Sprintf("%T", obj)
+}
+
 // TODO : Remove all the code below when PR
 // https://github.com/kubernetes/kubernetes/pull/50690
 // merges as that shuts down storage properly

vendor/k8s.io/apiserver/pkg/registry/generic/registry/store.go

@@ -47,7 +47,7 @@ import (
 	"k8s.io/apiserver/pkg/util/dryrun"
 	"k8s.io/client-go/tools/cache"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // ObjectFunc is a function to act on a given object. An error may be returned
@@ -322,15 +322,16 @@ func (e *Store) ListPredicate(ctx context.Context, p storage.SelectionPredicate,
 	p.Continue = options.Continue
 	list := e.NewListFunc()
 	qualifiedResource := e.qualifiedResourceFromContext(ctx)
+	storageOpts := storage.ListOptions{ResourceVersion: options.ResourceVersion, ResourceVersionMatch: options.ResourceVersionMatch, Predicate: p}
 	if name, ok := p.MatchesSingle(); ok {
 		if key, err := e.KeyFunc(ctx, name); err == nil {
-			err := e.Storage.GetToList(ctx, key, options.ResourceVersion, p, list)
+			err := e.Storage.GetToList(ctx, key, storageOpts, list)
 			return list, storeerr.InterpretListError(err, qualifiedResource)
 		}
 		// if we cannot extract a key based on the current context, the optimization is skipped
 	}
 
-	err := e.Storage.List(ctx, e.KeyRootFunc(ctx), options.ResourceVersion, p, list)
+	err := e.Storage.List(ctx, e.KeyRootFunc(ctx), storageOpts, list)
 	return list, storeerr.InterpretListError(err, qualifiedResource)
 }
 
@@ -367,7 +368,7 @@ func (e *Store) Create(ctx context.Context, obj runtime.Object, createValidation
 		if !apierrors.IsAlreadyExists(err) {
 			return nil, err
 		}
-		if errGet := e.Storage.Get(ctx, key, "", out, false); errGet != nil {
+		if errGet := e.Storage.Get(ctx, key, storage.GetOptions{}, out); errGet != nil {
 			return nil, err
 		}
 		accessor, errGetAcc := meta.Accessor(out)
@@ -472,6 +473,16 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 	// deleteObj is only used in case a deletion is carried out
 	var deleteObj runtime.Object
 	err = e.Storage.GuaranteedUpdate(ctx, key, out, true, storagePreconditions, func(existing runtime.Object, res storage.ResponseMeta) (runtime.Object, *uint64, error) {
+		existingResourceVersion, err := e.Storage.Versioner().ObjectResourceVersion(existing)
+		if err != nil {
+			return nil, nil, err
+		}
+		if existingResourceVersion == 0 {
+			if !e.UpdateStrategy.AllowCreateOnUpdate() && !forceAllowCreate {
+				return nil, nil, apierrors.NewNotFound(qualifiedResource, name)
+			}
+		}
+
 		// Given the existing object, get the new object
 		obj, err := objInfo.UpdatedObject(ctx, existing)
 		if err != nil {
@@ -482,20 +493,13 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 		// the user does not have a resource version, then we populate it with
 		// the latest version. Else, we check that the version specified by
 		// the user matches the version of latest storage object.
-		resourceVersion, err := e.Storage.Versioner().ObjectResourceVersion(obj)
+		newResourceVersion, err := e.Storage.Versioner().ObjectResourceVersion(obj)
 		if err != nil {
 			return nil, nil, err
 		}
-		doUnconditionalUpdate := resourceVersion == 0 && e.UpdateStrategy.AllowUnconditionalUpdate()
+		doUnconditionalUpdate := newResourceVersion == 0 && e.UpdateStrategy.AllowUnconditionalUpdate()
 
-		version, err := e.Storage.Versioner().ObjectResourceVersion(existing)
-		if err != nil {
-			return nil, nil, err
-		}
-		if version == 0 {
-			if !e.UpdateStrategy.AllowCreateOnUpdate() && !forceAllowCreate {
-				return nil, nil, apierrors.NewNotFound(qualifiedResource, name)
-			}
+		if existingResourceVersion == 0 {
 			creating = true
 			creatingObj = obj
 			if err := rest.BeforeCreate(e.CreateStrategy, ctx, obj); err != nil {
@@ -528,15 +532,15 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 		} else {
 			// Check if the object's resource version matches the latest
 			// resource version.
-			if resourceVersion == 0 {
+			if newResourceVersion == 0 {
 				// TODO: The Invalid error should have a field for Resource.
 				// After that field is added, we should fill the Resource and
 				// leave the Kind field empty. See the discussion in #18526.
 				qualifiedKind := schema.GroupKind{Group: qualifiedResource.Group, Kind: qualifiedResource.Resource}
-				fieldErrList := field.ErrorList{field.Invalid(field.NewPath("metadata").Child("resourceVersion"), resourceVersion, "must be specified for an update")}
+				fieldErrList := field.ErrorList{field.Invalid(field.NewPath("metadata").Child("resourceVersion"), newResourceVersion, "must be specified for an update")}
 				return nil, nil, apierrors.NewInvalid(qualifiedKind, name, fieldErrList)
 			}
-			if resourceVersion != version {
+			if newResourceVersion != existingResourceVersion {
 				return nil, nil, apierrors.NewConflict(qualifiedResource, name, fmt.Errorf(OptimisticLockErrorMsg))
 			}
 		}
@@ -608,7 +612,7 @@ func (e *Store) Get(ctx context.Context, name string, options *metav1.GetOptions
 	if err != nil {
 		return nil, err
 	}
-	if err := e.Storage.Get(ctx, key, options.ResourceVersion, obj, false); err != nil {
+	if err := e.Storage.Get(ctx, key, storage.GetOptions{ResourceVersion: options.ResourceVersion}, obj); err != nil {
 		return nil, storeerr.InterpretGetError(err, e.qualifiedResourceFromContext(ctx), name)
 	}
 	if e.Decorator != nil {
@@ -892,7 +896,7 @@ func (e *Store) Delete(ctx context.Context, name string, deleteValidation rest.V
 	}
 	obj := e.NewFunc()
 	qualifiedResource := e.qualifiedResourceFromContext(ctx)
-	if err = e.Storage.Get(ctx, key, "", obj, false); err != nil {
+	if err = e.Storage.Get(ctx, key, storage.GetOptions{}, obj); err != nil {
 		return nil, false, storeerr.InterpretDeleteError(err, qualifiedResource, name)
 	}
 
@@ -1006,12 +1010,19 @@ func (e *Store) DeleteCollection(ctx context.Context, deleteValidation rest.Vali
 	if err != nil {
 		return nil, err
 	}
+	if len(items) == 0 {
+		// Nothing to delete, return now
+		return listObj, nil
+	}
 	// Spawn a number of goroutines, so that we can issue requests to storage
 	// in parallel to speed up deletion.
-	// TODO: Make this proportional to the number of items to delete, up to
+	// It is proportional to the number of items to delete, up to
 	// DeleteCollectionWorkers (it doesn't make much sense to spawn 16
 	// workers to delete 10 items).
 	workersNumber := e.DeleteCollectionWorkers
+	if workersNumber > len(items) {
+		workersNumber = len(items)
+	}
 	if workersNumber < 1 {
 		workersNumber = 1
 	}
@@ -1119,9 +1130,10 @@ func (e *Store) Watch(ctx context.Context, options *metainternalversion.ListOpti
 
 // WatchPredicate starts a watch for the items that matches.
 func (e *Store) WatchPredicate(ctx context.Context, p storage.SelectionPredicate, resourceVersion string) (watch.Interface, error) {
+	storageOpts := storage.ListOptions{ResourceVersion: resourceVersion, Predicate: p}
 	if name, ok := p.MatchesSingle(); ok {
 		if key, err := e.KeyFunc(ctx, name); err == nil {
-			w, err := e.Storage.Watch(ctx, key, resourceVersion, p)
+			w, err := e.Storage.Watch(ctx, key, storageOpts)
 			if err != nil {
 				return nil, err
 			}
@@ -1134,7 +1146,7 @@ func (e *Store) WatchPredicate(ctx context.Context, p storage.SelectionPredicate
 		// optimization is skipped
 	}
 
-	w, err := e.Storage.WatchList(ctx, e.KeyRootFunc(ctx), resourceVersion, p)
+	w, err := e.Storage.WatchList(ctx, e.KeyRootFunc(ctx), storageOpts)
 	if err != nil {
 		return nil, err
 	}
@@ -1217,6 +1229,10 @@ func (e *Store) CompleteWithOptions(options *generic.StoreOptions) error {
 		return fmt.Errorf("store for %s must set both KeyRootFunc and KeyFunc or neither", e.DefaultQualifiedResource.String())
 	}
 
+	if e.TableConvertor == nil {
+		return fmt.Errorf("store for %s must set TableConvertor; rest.NewDefaultTableConvertor(e.DefaultQualifiedResource) can be used to output just name/creation time", e.DefaultQualifiedResource.String())
+	}
+
 	var isNamespaced bool
 	switch {
 	case e.CreateStrategy != nil:
@@ -1377,7 +1393,7 @@ func (e *Store) ConvertToTable(ctx context.Context, object runtime.Object, table
 	if e.TableConvertor != nil {
 		return e.TableConvertor.ConvertToTable(ctx, object, tableOptions)
 	}
-	return rest.NewDefaultTableConvertor(e.qualifiedResourceFromContext(ctx)).ConvertToTable(ctx, object, tableOptions)
+	return rest.NewDefaultTableConvertor(e.DefaultQualifiedResource).ConvertToTable(ctx, object, tableOptions)
 }
 
 func (e *Store) StorageVersion() runtime.GroupVersioner {

vendor/k8s.io/apiserver/pkg/registry/rest/OWNERS

@@ -14,13 +14,11 @@ reviewers:
 - gmarek
 - justinsb
 - ncdc
-- eparis
 - dims
 - hongchaodeng
 - krousey
 - euank
 - ingvagabund
-- david-mcmahon
 - jianhuiz
 - sdminonne
 - enj

vendor/k8s.io/apiserver/pkg/registry/rest/table.go

@@ -26,15 +26,17 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	genericapirequest "k8s.io/apiserver/pkg/endpoints/request"
 )
 
 type defaultTableConvertor struct {
-	qualifiedResource schema.GroupResource
+	defaultQualifiedResource schema.GroupResource
 }
 
-// NewDefaultTableConvertor creates a default convertor for the provided resource.
-func NewDefaultTableConvertor(resource schema.GroupResource) TableConvertor {
-	return defaultTableConvertor{qualifiedResource: resource}
+// NewDefaultTableConvertor creates a default convertor; the provided resource is used for error messages
+// if no resource info can be determined from the context passed to ConvertToTable.
+func NewDefaultTableConvertor(defaultQualifiedResource schema.GroupResource) TableConvertor {
+	return defaultTableConvertor{defaultQualifiedResource: defaultQualifiedResource}
 }
 
 var swaggerMetadataDescriptions = metav1.ObjectMeta{}.SwaggerDoc()
@@ -44,7 +46,11 @@ func (c defaultTableConvertor) ConvertToTable(ctx context.Context, object runtim
 	fn := func(obj runtime.Object) error {
 		m, err := meta.Accessor(obj)
 		if err != nil {
-			return errNotAcceptable{resource: c.qualifiedResource}
+			resource := c.defaultQualifiedResource
+			if info, ok := genericapirequest.RequestInfoFrom(ctx); ok {
+				resource = schema.GroupResource{Group: info.APIGroup, Resource: info.Resource}
+			}
+			return errNotAcceptable{resource: resource}
 		}
 		table.Rows = append(table.Rows, metav1.TableRow{
 			Cells:  []interface{}{m.GetName(), m.GetCreationTimestamp().Time.UTC().Format(time.RFC3339)},

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -53,6 +53,7 @@ import (
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/features"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	"k8s.io/apiserver/pkg/server/egressselector"
@@ -60,12 +61,14 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/component-base/logs"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
+	utilsnet "k8s.io/utils/net"
 
 	// install apis
 	_ "k8s.io/apiserver/pkg/apis/apiserver/install"
@@ -272,10 +275,6 @@ type AuthenticationInfo struct {
 	APIAudiences authenticator.Audiences
 	// Authenticator determines which subject is making the request
 	Authenticator authenticator.Request
-	// SupportsBasicAuth indicates that's at least one Authenticator supports basic auth
-	// If this is true, a basic auth challenge is returned on authentication failure
-	// TODO(roberthbailey): Remove once the server no longer supports http basic auth.
-	SupportsBasicAuth bool
 }
 
 type AuthorizationInfo struct {
@@ -606,11 +605,18 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 	}
 
 	genericApiServerHookName := "generic-apiserver-start-informers"
-	if c.SharedInformerFactory != nil && !s.isPostStartHookRegistered(genericApiServerHookName) {
-		err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
-			c.SharedInformerFactory.Start(context.StopCh)
-			return nil
-		})
+	if c.SharedInformerFactory != nil {
+		if !s.isPostStartHookRegistered(genericApiServerHookName) {
+			err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
+				c.SharedInformerFactory.Start(context.StopCh)
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+		// TODO: Once we get rid of /healthz consider changing this to post-start-hook.
+		err := s.addReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
 		if err != nil {
 			return nil, err
 		}
@@ -620,6 +626,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 	if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
 	} else if c.FlowControl != nil {
 		err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
+			go c.FlowControl.MaintainObservations(context.StopCh)
 			go c.FlowControl.Run(context.StopCh)
 			return nil
 		})
@@ -670,7 +677,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 	}
 	handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
 	handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
-	failedHandler := genericapifilters.Unauthorized(c.Serializer, c.Authentication.SupportsBasicAuth)
+	failedHandler := genericapifilters.Unauthorized(c.Serializer)
 	failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyChecker)
 	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
 	handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
@@ -680,6 +687,8 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 	if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
 		handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
 	}
+	handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyChecker)
+	handler = genericapifilters.WithWarningRecorder(handler)
 	handler = genericapifilters.WithCacheControl(handler)
 	handler = genericfilters.WithPanicRecovery(handler)
 	return handler
@@ -710,6 +719,9 @@ func installAPI(s *GenericAPIServer, c *Config) {
 	if c.EnableDiscovery {
 		s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
 	}
+	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
+		c.FlowControl.Install(s.Handler.NonGoRestfulMux)
+	}
 }
 
 func NewRequestInfoResolver(c *Config) *apirequest.RequestInfoFactory {
@@ -735,7 +747,7 @@ func (s *SecureServingInfo) HostPort() (string, int, error) {
 	if err != nil {
 		return "", 0, fmt.Errorf("failed to get port from listener address %q: %v", addr, err)
 	}
-	port, err := strconv.Atoi(portStr)
+	port, err := utilsnet.ParsePort(portStr, true)
 	if err != nil {
 		return "", 0, fmt.Errorf("invalid non-numeric port %q", portStr)
 	}

vendor/k8s.io/apiserver/pkg/server/deprecated_insecure_serving.go

@@ -21,7 +21,7 @@ import (
 	"net/http"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/user"

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -33,7 +33,7 @@ import (
 	corev1listers "k8s.io/client-go/listers/core/v1"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/client-go/util/workqueue"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -29,7 +29,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // FileRefreshDuration is exposed so that integration tests can crank up the reload speed.

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -26,7 +26,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/util/workqueue"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // DynamicCertKeyPairContent provides a CertKeyContentProvider that can dynamically react to new file content

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/named_certificates.go

@@ -25,7 +25,7 @@ import (
 
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/validation"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // BuildNamedCertificates returns a map of *tls.Certificate by name. It's

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -31,7 +31,7 @@ import (
 	"k8s.io/client-go/tools/events"
 	"k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/workqueue"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const workItemKey = "key"

vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -34,7 +34,7 @@ import (
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	utiltrace "k8s.io/utils/trace"
 	client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
 )
@@ -199,7 +199,7 @@ func (u *udsGRPCConnector) connect() (proxier, error) {
 		return c, err
 	})
 
-	tunnel, err := client.CreateGrpcTunnel(udsName, dialOption, grpc.WithInsecure())
+	tunnel, err := client.CreateSingleUseGrpcTunnel(udsName, dialOption, grpc.WithInsecure())
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/apiserver/pkg/server/filters/cors.go

@@ -21,7 +21,7 @@ import (
 	"regexp"
 	"strings"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // TODO: use restful.CrossOriginResourceSharing

vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -27,8 +27,9 @@ import (
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -50,13 +51,17 @@ func handleError(w http.ResponseWriter, r *http.Request, err error) {
 	klog.Errorf(err.Error())
 }
 
-// requestWatermark is used to trak maximal usage of inflight requests.
+// requestWatermark is used to track maximal numbers of requests in a particular phase of handling
 type requestWatermark struct {
+	phase                                string
+	readOnlyObserver, mutatingObserver   fcmetrics.TimedObserver
 	lock                                 sync.Mutex
 	readOnlyWatermark, mutatingWatermark int
 }
 
 func (w *requestWatermark) recordMutating(mutatingVal int) {
+	w.mutatingObserver.Set(float64(mutatingVal))
+
 	w.lock.Lock()
 	defer w.lock.Unlock()
 
@@ -66,6 +71,8 @@ func (w *requestWatermark) recordMutating(mutatingVal int) {
 }
 
 func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
+	w.readOnlyObserver.Set(float64(readOnlyVal))
+
 	w.lock.Lock()
 	defer w.lock.Unlock()
 
@@ -74,9 +81,14 @@ func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
 	}
 }
 
-var watermark = &requestWatermark{}
+// watermark tracks requests being executed (not waiting in a queue)
+var watermark = &requestWatermark{
+	phase:            metrics.ExecutingPhase,
+	readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{metrics.ReadOnlyKind}).RequestsExecuting,
+	mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{metrics.MutatingKind}).RequestsExecuting,
+}
 
-func startRecordingUsage() {
+func startRecordingUsage(watermark *requestWatermark) {
 	go func() {
 		wait.Forever(func() {
 			watermark.lock.Lock()
@@ -86,7 +98,7 @@ func startRecordingUsage() {
 			watermark.mutatingWatermark = 0
 			watermark.lock.Unlock()
 
-			metrics.UpdateInflightRequestMetrics(readOnlyWatermark, mutatingWatermark)
+			metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
 		}, inflightUsageMetricUpdatePeriod)
 	}()
 }
@@ -100,7 +112,7 @@ func WithMaxInFlightLimit(
 	mutatingLimit int,
 	longRunningRequestCheck apirequest.LongRunningRequestCheck,
 ) http.Handler {
-	startOnce.Do(startRecordingUsage)
+	startOnce.Do(func() { startRecordingUsage(watermark) })
 	if nonMutatingLimit == 0 && mutatingLimit == 0 {
 		return handler
 	}
@@ -108,9 +120,11 @@ func WithMaxInFlightLimit(
 	var mutatingChan chan bool
 	if nonMutatingLimit != 0 {
 		nonMutatingChan = make(chan bool, nonMutatingLimit)
+		watermark.readOnlyObserver.SetX1(float64(nonMutatingLimit))
 	}
 	if mutatingLimit != 0 {
 		mutatingChan = make(chan bool, mutatingLimit)
+		watermark.mutatingObserver.SetX1(float64(mutatingLimit))
 	}
 
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@@ -141,21 +155,22 @@ func WithMaxInFlightLimit(
 
 			select {
 			case c <- true:
-				var mutatingLen, readOnlyLen int
+				// We note the concurrency level both while the
+				// request is being served and after it is done being
+				// served, because both states contribute to the
+				// sampled stats on concurrency.
 				if isMutatingRequest {
-					mutatingLen = len(mutatingChan)
+					watermark.recordMutating(len(c))
 				} else {
-					readOnlyLen = len(nonMutatingChan)
+					watermark.recordReadOnly(len(c))
 				}
-
 				defer func() {
 					<-c
 					if isMutatingRequest {
-						watermark.recordMutating(mutatingLen)
+						watermark.recordMutating(len(c))
 					} else {
-						watermark.recordReadOnly(readOnlyLen)
+						watermark.recordReadOnly(len(c))
 					}
-
 				}()
 				handler.ServeHTTP(w, r)
 

vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -24,9 +24,11 @@ import (
 
 	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
 	apitypes "k8s.io/apimachinery/pkg/types"
+	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
-	"k8s.io/klog"
+	fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	"k8s.io/klog/v2"
 )
 
 type priorityAndFairnessKeyType int
@@ -53,7 +55,15 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
 	return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
 }
 
-var atomicMutatingLen, atomicNonMutatingLen int32
+// waitingMark tracks requests waiting rather than being executed
+var waitingMark = &requestWatermark{
+	phase:            epmetrics.WaitingPhase,
+	readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.ReadOnlyKind}).RequestsWaiting,
+	mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.MutatingKind}).RequestsWaiting,
+}
+
+var atomicMutatingExecuting, atomicReadOnlyExecuting int32
+var atomicMutatingWaiting, atomicReadOnlyWaiting int32
 
 // WithPriorityAndFairness limits the number of in-flight
 // requests in a fine-grained way.
@@ -66,7 +76,10 @@ func WithPriorityAndFairness(
 		klog.Warningf("priority and fairness support not found, skipping")
 		return handler
 	}
-	startOnce.Do(startRecordingUsage)
+	startOnce.Do(func() {
+		startRecordingUsage(watermark)
+		startRecordingUsage(waitingMark)
+	})
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
@@ -98,22 +111,23 @@ func WithPriorityAndFairness(
 
 		var served bool
 		isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
-		execute := func() {
-			var mutatingLen, readOnlyLen int
+		noteExecutingDelta := func(delta int32) {
 			if isMutatingRequest {
-				mutatingLen = int(atomic.AddInt32(&atomicMutatingLen, 1))
+				watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
 			} else {
-				readOnlyLen = int(atomic.AddInt32(&atomicNonMutatingLen, 1))
+				watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
 			}
-			defer func() {
-				if isMutatingRequest {
-					atomic.AddInt32(&atomicMutatingLen, -11)
-					watermark.recordMutating(mutatingLen)
-				} else {
-					atomic.AddInt32(&atomicNonMutatingLen, -1)
-					watermark.recordReadOnly(readOnlyLen)
-				}
-			}()
+		}
+		noteWaitingDelta := func(delta int32) {
+			if isMutatingRequest {
+				waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
+			} else {
+				waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
+			}
+		}
+		execute := func() {
+			noteExecutingDelta(1)
+			defer noteExecutingDelta(-1)
 			served = true
 			innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
 			innerReq := r.Clone(innerCtx)
@@ -122,10 +136,16 @@ func WithPriorityAndFairness(
 			handler.ServeHTTP(w, innerReq)
 		}
 		digest := utilflowcontrol.RequestDigest{requestInfo, user}
-		fcIfc.Handle(ctx, digest, note, execute)
+		fcIfc.Handle(ctx, digest, note, func(inQueue bool) {
+			if inQueue {
+				noteWaitingDelta(1)
+			} else {
+				noteWaitingDelta(-1)
+			}
+		}, execute)
 		if !served {
+			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
 			tooManyRequests(r, w)
-			return
 		}
 
 	})

vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -19,7 +19,7 @@ package filters
 import (
 	"net/http"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server/httplog"

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -45,7 +45,7 @@ import (
 	"k8s.io/apiserver/pkg/server/routes"
 	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	openapibuilder "k8s.io/kube-openapi/pkg/builder"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/handler"
@@ -330,7 +330,7 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 	}()
 
 	// close socket after delayed stopCh
-	err := s.NonBlockingRun(delayedStopCh)
+	stoppedCh, err := s.NonBlockingRun(delayedStopCh)
 	if err != nil {
 		return err
 	}
@@ -345,6 +345,8 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 
 	// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
 	<-delayedStopCh
+	// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
+	<-stoppedCh
 
 	// Wait for all requests to finish, which are bounded by the RequestTimeout variable.
 	s.HandlerChainWaitGroup.Wait()
@@ -354,7 +356,8 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
 
 // NonBlockingRun spawns the secure http server. An error is
 // returned if the secure port cannot be listened on.
-func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
+// The returned channel is closed when the (asynchronous) termination is finished.
+func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) (<-chan struct{}, error) {
 	// Use an stop channel to allow graceful shutdown without dropping audit events
 	// after http server shutdown.
 	auditStopCh := make(chan struct{})
@@ -363,7 +366,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 	// before http server start serving. Otherwise the Backend.ProcessEvents call might block.
 	if s.AuditBackend != nil {
 		if err := s.AuditBackend.Run(auditStopCh); err != nil {
-			return fmt.Errorf("failed to run the audit backend: %v", err)
+			return nil, fmt.Errorf("failed to run the audit backend: %v", err)
 		}
 	}
 
@@ -376,7 +379,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 		if err != nil {
 			close(internalStopCh)
 			close(auditStopCh)
-			return err
+			return nil, err
 		}
 	}
 
@@ -399,7 +402,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 		klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
 	}
 
-	return nil
+	return stoppedCh, nil
 }
 
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource

vendor/k8s.io/apiserver/pkg/server/handler.go

@@ -25,7 +25,7 @@ import (
 	"strings"
 
 	"github.com/emicklei/go-restful"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"

vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -20,6 +20,7 @@ import (
 	"bytes"
 	"fmt"
 	"net/http"
+	"reflect"
 	"strings"
 	"sync"
 	"sync/atomic"
@@ -29,7 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/httplog"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // HealthChecker is a named healthz checker.
@@ -81,6 +82,43 @@ func (l *log) Check(_ *http.Request) error {
 	return fmt.Errorf("logging blocked")
 }
 
+type cacheSyncWaiter interface {
+	WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
+}
+
+type informerSync struct {
+	cacheSyncWaiter cacheSyncWaiter
+}
+
+var _ HealthChecker = &informerSync{}
+
+// NewInformerSyncHealthz returns a new HealthChecker that will pass only if all informers in the given cacheSyncWaiter sync.
+func NewInformerSyncHealthz(cacheSyncWaiter cacheSyncWaiter) HealthChecker {
+	return &informerSync{
+		cacheSyncWaiter: cacheSyncWaiter,
+	}
+}
+
+func (i *informerSync) Name() string {
+	return "informer-sync"
+}
+
+func (i *informerSync) Check(_ *http.Request) error {
+	stopCh := make(chan struct{})
+	// Close stopCh to force checking if informers are synced now.
+	close(stopCh)
+
+	informersByStarted := make(map[bool][]string)
+	for informerType, started := range i.cacheSyncWaiter.WaitForCacheSync(stopCh) {
+		informersByStarted[started] = append(informersByStarted[started], informerType.String())
+	}
+
+	if notStarted := informersByStarted[false]; len(notStarted) > 0 {
+		return fmt.Errorf("%d informers not started yet: %v", len(notStarted), notStarted)
+	}
+	return nil
+}
+
 // NamedCheck returns a healthz checker for the given name and function.
 func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
 	return &healthzCheck{name, check}
@@ -131,6 +169,8 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
 			/* subresource = */ path,
 			/* scope = */ "",
 			/* component = */ "",
+			/* deprecated */ false,
+			/* removedRelease */ "",
 			handleRootHealthz(checks...)))
 	for _, check := range checks {
 		mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
@@ -170,35 +210,38 @@ func getExcludedChecks(r *http.Request) sets.String {
 // handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
 func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
-		failed := false
 		excluded := getExcludedChecks(r)
-		var verboseOut bytes.Buffer
+		// failedVerboseLogOutput is for output to the log.  It indicates detailed failed output information for the log.
+		var failedVerboseLogOutput bytes.Buffer
+		var failedChecks []string
+		var individualCheckOutput bytes.Buffer
 		for _, check := range checks {
 			// no-op the check if we've specified we want to exclude the check
 			if excluded.Has(check.Name()) {
 				excluded.Delete(check.Name())
-				fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name())
+				fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
 				continue
 			}
 			if err := check.Check(r); err != nil {
 				// don't include the error since this endpoint is public.  If someone wants more detail
 				// they should have explicit permission to the detailed checks.
-				klog.V(4).Infof("healthz check %v failed: %v", check.Name(), err)
-				fmt.Fprintf(&verboseOut, "[-]%v failed: reason withheld\n", check.Name())
-				failed = true
+				fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
+				// but we do want detailed information for our log
+				fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
+				failedChecks = append(failedChecks, check.Name())
 			} else {
-				fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
+				fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
 			}
 		}
 		if excluded.Len() > 0 {
-			fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...))
-			klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v",
+			fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
+			klog.Warningf("cannot exclude some health checks, no health checks are installed matching %s",
 				formatQuoted(excluded.List()...))
 		}
 		// always be verbose on failure
-		if failed {
-			klog.V(2).Infof("%vhealthz check failed", verboseOut.String())
-			http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
+		if len(failedChecks) > 0 {
+			klog.V(2).Infof("healthz check failed: %s\n%v", strings.Join(failedChecks, ","), failedVerboseLogOutput.String())
+			http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%shealthz check failed", individualCheckOutput.String()), http.StatusInternalServerError)
 			return
 		}
 
@@ -209,7 +252,7 @@ func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
 			return
 		}
 
-		verboseOut.WriteTo(w)
+		individualCheckOutput.WriteTo(w)
 		fmt.Fprint(w, "healthz check passed\n")
 	})
 }

vendor/k8s.io/apiserver/pkg/server/hooks.go

@@ -26,7 +26,7 @@ import (
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server/healthz"
 	restclient "k8s.io/client-go/rest"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // PostStartHookFunc is a function that is called after the server has started.

vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -25,7 +25,7 @@ import (
 	"runtime"
 	"time"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // StacktracePred returns true if a stacktrace should be logged for this status.
@@ -85,7 +85,9 @@ func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
 		rl := newLogged(req, w).StacktraceWhen(pred)
 		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
 
-		defer rl.Log()
+		if klog.V(3).Enabled() {
+			defer func() { klog.InfoS("HTTP", rl.LogArgs()...) }()
+		}
 		handler.ServeHTTP(rl, req)
 	})
 }
@@ -153,24 +155,34 @@ func (rl *respLogger) Addf(format string, data ...interface{}) {
 	rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
 }
 
-// Log is intended to be called once at the end of your request handler, via defer
-func (rl *respLogger) Log() {
+func (rl *respLogger) LogArgs() []interface{} {
 	latency := time.Since(rl.startTime)
-	if klog.V(3) {
-		if !rl.hijacked {
-			klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v resp=%v UserAgent=%q srcIP=%q: %v%v",
-				rl.req.Method, rl.req.RequestURI,
-				latency, rl.status,
-				rl.req.UserAgent(), rl.req.RemoteAddr,
-				rl.statusStack, rl.addedInfo,
-			))
-		} else {
-			klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v UserAgent=%q srcIP=%q: hijacked",
-				rl.req.Method, rl.req.RequestURI,
-				latency, rl.req.UserAgent(), rl.req.RemoteAddr,
-			))
+	if rl.hijacked {
+		return []interface{}{
+			"verb", rl.req.Method,
+			"URI", rl.req.RequestURI,
+			"latency", latency,
+			"userAgent", rl.req.UserAgent(),
+			"srcIP", rl.req.RemoteAddr,
+			"hijacked", true,
 		}
 	}
+	args := []interface{}{
+		"verb", rl.req.Method,
+		"URI", rl.req.RequestURI,
+		"latency", latency,
+		"userAgent", rl.req.UserAgent(),
+		"srcIP", rl.req.RemoteAddr,
+		"resp", rl.status,
+	}
+	if len(rl.statusStack) > 0 {
+		args = append(args, "statusStack", rl.statusStack)
+	}
+
+	if len(rl.addedInfo) > 0 {
+		args = append(args, "addedInfo", rl.addedInfo)
+	}
+	return args
 }
 
 // Header implements http.ResponseWriter.
@@ -194,7 +206,7 @@ func (rl *respLogger) Write(b []byte) (int, error) {
 func (rl *respLogger) Flush() {
 	if flusher, ok := rl.w.(http.Flusher); ok {
 		flusher.Flush()
-	} else if klog.V(2) {
+	} else if klog.V(2).Enabled() {
 		klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
 	}
 }

vendor/k8s.io/apiserver/pkg/server/mux/pathrecorder.go

@@ -25,7 +25,7 @@ import (
 	"sync"
 	"sync/atomic"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apimachinery/pkg/util/sets"

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -25,9 +25,8 @@ import (
 
 	"github.com/spf13/pflag"
 	"gopkg.in/natefinch/lumberjack.v2"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
-	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	auditinternal "k8s.io/apiserver/pkg/apis/audit"
@@ -36,20 +35,12 @@ import (
 	auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/audit/policy"
-	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
-	plugindynamic "k8s.io/apiserver/plugin/pkg/audit/dynamic"
-	pluginenforced "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced"
 	pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
 	plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
 	pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
-	"k8s.io/client-go/informers"
-	"k8s.io/client-go/kubernetes"
-	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
-	restclient "k8s.io/client-go/rest"
 )
 
 const (
@@ -80,7 +71,6 @@ type AuditOptions struct {
 	// Plugin options
 	LogOptions     AuditLogOptions
 	WebhookOptions AuditWebhookOptions
-	DynamicOptions AuditDynamicOptions
 }
 
 const (
@@ -180,10 +170,6 @@ func NewAuditOptions() *AuditOptions {
 			TruncateOptions:    NewAuditTruncateOptions(),
 			GroupVersionString: "audit.k8s.io/v1",
 		},
-		DynamicOptions: AuditDynamicOptions{
-			Enabled:     false,
-			BatchConfig: plugindynamic.NewDefaultWebhookBatchConfig(),
-		},
 	}
 }
 
@@ -206,7 +192,6 @@ func (o *AuditOptions) Validate() []error {
 	var allErrors []error
 	allErrors = append(allErrors, o.LogOptions.Validate()...)
 	allErrors = append(allErrors, o.WebhookOptions.Validate()...)
-	allErrors = append(allErrors, o.DynamicOptions.Validate()...)
 
 	return allErrors
 }
@@ -286,15 +271,10 @@ func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) {
 	o.WebhookOptions.AddFlags(fs)
 	o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs)
 	o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs)
-	o.DynamicOptions.AddFlags(fs)
 }
 
 func (o *AuditOptions) ApplyTo(
 	c *server.Config,
-	kubeClientConfig *restclient.Config,
-	informers informers.SharedInformerFactory,
-	processInfo *ProcessInfo,
-	webhookOptions *WebhookOptions,
 ) error {
 	if o == nil {
 		return nil
@@ -347,23 +327,7 @@ func (o *AuditOptions) ApplyTo(
 
 	// 4. Apply dynamic options.
 	var dynamicBackend audit.Backend
-	if o.DynamicOptions.enabled() {
-		// if dynamic is enabled the webhook and log backends need to be wrapped in an enforced backend with the static policy
-		if webhookBackend != nil {
-			webhookBackend = pluginenforced.NewBackend(webhookBackend, checker)
-		}
-		if logBackend != nil {
-			logBackend = pluginenforced.NewBackend(logBackend, checker)
-		}
-		// build dynamic backend
-		dynamicBackend, checker, err = o.DynamicOptions.newBackend(c.ExternalAddress, kubeClientConfig, informers, processInfo, webhookOptions)
-		if err != nil {
-			return err
-		}
-		// union dynamic and webhook backends so that truncate options can be applied to both
-		dynamicBackend = appendBackend(webhookBackend, dynamicBackend)
-		dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(dynamicBackend, groupVersion)
-	} else if webhookBackend != nil {
+	if webhookBackend != nil {
 		// if only webhook is enabled wrap it in the truncate options
 		dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
 	}
@@ -610,66 +574,6 @@ func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc)
 	return webhook, nil
 }
 
-func (o *AuditDynamicOptions) AddFlags(fs *pflag.FlagSet) {
-	fs.BoolVar(&o.Enabled, "audit-dynamic-configuration", o.Enabled,
-		"Enables dynamic audit configuration. This feature also requires the DynamicAuditing feature flag")
-}
-
-func (o *AuditDynamicOptions) enabled() bool {
-	return o.Enabled && utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing)
-}
-
-func (o *AuditDynamicOptions) Validate() []error {
-	var allErrors []error
-	if o.Enabled && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing) {
-		allErrors = append(allErrors, fmt.Errorf("--audit-dynamic-configuration set, but DynamicAuditing feature gate is not enabled"))
-	}
-	return allErrors
-}
-
-func (o *AuditDynamicOptions) newBackend(
-	hostname string,
-	kubeClientConfig *restclient.Config,
-	informers informers.SharedInformerFactory,
-	processInfo *ProcessInfo,
-	webhookOptions *WebhookOptions,
-) (audit.Backend, policy.Checker, error) {
-	if err := validateProcessInfo(processInfo); err != nil {
-		return nil, nil, err
-	}
-	clientset, err := kubernetes.NewForConfig(kubeClientConfig)
-	if err != nil {
-		return nil, nil, err
-	}
-	if webhookOptions == nil {
-		webhookOptions = NewWebhookOptions()
-	}
-	checker := policy.NewDynamicChecker()
-	informer := informers.Auditregistration().V1alpha1().AuditSinks()
-	eventSink := &v1core.EventSinkImpl{Interface: clientset.CoreV1().Events(processInfo.Namespace)}
-
-	dc := &plugindynamic.Config{
-		Informer:       informer,
-		BufferedConfig: o.BatchConfig,
-		EventConfig: plugindynamic.EventConfig{
-			Sink: eventSink,
-			Source: corev1.EventSource{
-				Component: processInfo.Name,
-				Host:      hostname,
-			},
-		},
-		WebhookConfig: plugindynamic.WebhookConfig{
-			AuthInfoResolverWrapper: webhookOptions.AuthInfoResolverWrapper,
-			ServiceResolver:         webhookOptions.ServiceResolver,
-		},
-	}
-	backend, err := plugindynamic.NewBackend(dc)
-	if err != nil {
-		return nil, nil, fmt.Errorf("could not create dynamic audit backend: %v", err)
-	}
-	return backend, checker, nil
-}
-
 // defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend.
 func defaultWebhookBatchConfig() pluginbuffered.BatchConfig {
 	return pluginbuffered.BatchConfig{

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -17,8 +17,6 @@ limitations under the License.
 package options
 
 import (
-	"context"
-	"encoding/json"
 	"fmt"
 	"strings"
 	"time"
@@ -27,7 +25,6 @@ import (
 
 	"github.com/spf13/pflag"
 
-	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
@@ -35,7 +32,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
 
@@ -319,7 +316,6 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
 	if openAPIConfig != nil {
 		openAPIConfig.SecurityDefinitions = securityDefinitions
 	}
-	authenticationInfo.SupportsBasicAuth = false
 
 	return nil
 }
@@ -331,66 +327,28 @@ const (
 	// by --requestheader-username-headers. This is created in the cluster by the kube-apiserver.
 	// "WARNING: generally do not depend on authorization being already done for incoming requests.")
 	authenticationConfigMapName = "extension-apiserver-authentication"
-	authenticationRoleName      = "extension-apiserver-authentication-reader"
 )
 
 func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
-	requestHeaderCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "requestheader-client-ca-file", client)
+	dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client)
 	if err != nil {
 		return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
 	}
 
-	authConfigMap, err := client.CoreV1().ConfigMaps(authenticationConfigMapNamespace).Get(context.TODO(), authenticationConfigMapName, metav1.GetOptions{})
-	switch {
-	case errors.IsNotFound(err):
-		// ignore, authConfigMap is nil now
-		return nil, nil
-	case errors.IsForbidden(err):
-		klog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
-			"'kubectl create rolebinding -n %s ROLEBINDING_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
-			authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
-		return nil, err
-	case err != nil:
-		return nil, err
-	}
-
-	usernameHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-username-headers"])
-	if err != nil {
-		return nil, err
-	}
-	groupHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-group-headers"])
-	if err != nil {
-		return nil, err
-	}
-	extraHeaderPrefixes, err := deserializeStrings(authConfigMap.Data["requestheader-extra-headers-prefix"])
-	if err != nil {
-		return nil, err
-	}
-	allowedNames, err := deserializeStrings(authConfigMap.Data["requestheader-allowed-names"])
-	if err != nil {
+	//  look up authentication configuration in the cluster and in case of an err defer to authentication-tolerate-lookup-failure flag
+	if err := dynamicRequestHeaderProvider.RunOnce(); err != nil {
 		return nil, err
 	}
 
 	return &authenticatorfactory.RequestHeaderConfig{
-		CAContentProvider:   requestHeaderCAProvider,
-		UsernameHeaders:     headerrequest.StaticStringSlice(usernameHeaders),
-		GroupHeaders:        headerrequest.StaticStringSlice(groupHeaders),
-		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(extraHeaderPrefixes),
-		AllowedClientNames:  headerrequest.StaticStringSlice(allowedNames),
+		CAContentProvider:   dynamicRequestHeaderProvider,
+		UsernameHeaders:     headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)),
+		GroupHeaders:        headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)),
+		ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)),
+		AllowedClientNames:  headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)),
 	}, nil
 }
 
-func deserializeStrings(in string) ([]string, error) {
-	if len(in) == 0 {
-		return nil, nil
-	}
-	var ret []string
-	if err := json.Unmarshal([]byte(in), &ret); err != nil {
-		return nil, err
-	}
-	return ret, nil
-}
-
 // getClient returns a Kubernetes clientset. If s.RemoteKubeConfigFileOptional is true, nil will be returned
 // if no kubeconfig is specified by the user and the in-cluster config is not found.
 func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, error) {

vendor/k8s.io/apiserver/pkg/server/options/authentication_dynamic_request_header.go

@@ -0,0 +1,79 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+
+	"k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/client-go/kubernetes"
+)
+
+var _ dynamiccertificates.ControllerRunner = &DynamicRequestHeaderController{}
+var _ dynamiccertificates.Notifier = &DynamicRequestHeaderController{}
+var _ dynamiccertificates.CAContentProvider = &DynamicRequestHeaderController{}
+
+var _ headerrequest.RequestHeaderAuthRequestProvider = &DynamicRequestHeaderController{}
+
+// DynamicRequestHeaderController combines DynamicCAFromConfigMapController and RequestHeaderAuthRequestController
+// into one controller for dynamically filling RequestHeaderConfig struct
+type DynamicRequestHeaderController struct {
+	*dynamiccertificates.ConfigMapCAController
+	*headerrequest.RequestHeaderAuthRequestController
+}
+
+// newDynamicRequestHeaderController creates a new controller that implements DynamicRequestHeaderController
+func newDynamicRequestHeaderController(client kubernetes.Interface) (*DynamicRequestHeaderController, error) {
+	requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
+		"client-ca",
+		authenticationConfigMapNamespace,
+		authenticationConfigMapName,
+		"requestheader-client-ca-file",
+		client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err)
+	}
+
+	requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController(
+		authenticationConfigMapName,
+		authenticationConfigMapNamespace,
+		client,
+		"requestheader-username-headers",
+		"requestheader-group-headers",
+		"requestheader-extra-headers-prefix",
+		"requestheader-allowed-names",
+	)
+	return &DynamicRequestHeaderController{
+		ConfigMapCAController:              requestHeaderCAController,
+		RequestHeaderAuthRequestController: requestHeaderAuthRequestController,
+	}, nil
+}
+
+func (c *DynamicRequestHeaderController) RunOnce() error {
+	errs := []error{}
+	errs = append(errs, c.ConfigMapCAController.RunOnce())
+	errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce())
+	return errors.NewAggregate(errs)
+}
+
+func (c *DynamicRequestHeaderController) Run(workers int, stopCh <-chan struct{}) {
+	go c.ConfigMapCAController.Run(workers, stopCh)
+	go c.RequestHeaderAuthRequestController.Run(workers, stopCh)
+	<-stopCh
+}

vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -21,7 +21,7 @@ import (
 	"time"
 
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -43,7 +43,7 @@ type DeprecatedInsecureServingOptions struct {
 
 	// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
 	// It defaults to options.CreateListener.
-	ListenFunc func(network, addr string) (net.Listener, int, error)
+	ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
 }
 
 // Validate ensures that the insecure port values within the range of the port.
@@ -113,7 +113,7 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure
 			listen = s.ListenFunc
 		}
 		addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
-		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr)
+		s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
 		if err != nil {
 			return fmt.Errorf("failed to create listener: %v", err)
 		}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -161,15 +161,14 @@ func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Tr
 	}
 	defer f.Close()
 
-	result, err := ParseEncryptionConfiguration(f)
+	result, err := parseEncryptionConfiguration(f)
 	if err != nil {
 		return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
 	}
 	return result, nil
 }
 
-// ParseEncryptionConfiguration parses configuration data and returns the transformer overrides
-func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
+func parseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
 	configFileContents, err := ioutil.ReadAll(f)
 	if err != nil {
 		return nil, fmt.Errorf("could not read contents: %v", err)
@@ -184,7 +183,7 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T
 
 	// For each entry in the configuration
 	for _, resourceConfig := range config.Resources {
-		transformers, err := GetPrefixTransformers(&resourceConfig)
+		transformers, err := prefixTransformers(&resourceConfig)
 		if err != nil {
 			return nil, err
 		}
@@ -205,7 +204,6 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T
 
 }
 
-// loadConfig decodes data as a EncryptionConfiguration object.
 func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
 	scheme := runtime.NewScheme()
 	codecs := serializer.NewCodecFactory(scheme)
@@ -227,8 +225,7 @@ func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
 // The factory to create kms service. This is to make writing test easier.
 var envelopeServiceFactory = envelope.NewGRPCService
 
-// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration.
-func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
+func prefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
 	var result []value.PrefixTransformer
 	for _, provider := range config.Providers {
 		var (
@@ -238,18 +235,18 @@ func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]val
 
 		switch {
 		case provider.AESGCM != nil:
-			transformer, err = GetAESPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
+			transformer, err = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
 		case provider.AESCBC != nil:
-			transformer, err = GetAESPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
+			transformer, err = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
 		case provider.Secretbox != nil:
-			transformer, err = GetSecretboxPrefixTransformer(provider.Secretbox)
+			transformer, err = secretboxPrefixTransformer(provider.Secretbox)
 		case provider.KMS != nil:
 			envelopeService, err := envelopeServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration)
 			if err != nil {
 				return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
 			}
 
-			transformer, err = getEnvelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
+			transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
 		case provider.Identity != nil:
 			transformer = value.PrefixTransformer{
 				Transformer: identity.NewEncryptCheckTransformer(),
@@ -267,12 +264,9 @@ func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]val
 	return result, nil
 }
 
-// BlockTransformerFunc takes an AES cipher block and returns a value transformer.
-type BlockTransformerFunc func(cipher.Block) value.Transformer
+type blockTransformerFunc func(cipher.Block) value.Transformer
 
-// GetAESPrefixTransformer returns a prefix transformer from the provided configuration.
-// Returns an AES transformer based on the provided prefix and block transformer.
-func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
+func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
 	var result value.PrefixTransformer
 
 	if len(config.Keys) == 0 {
@@ -319,8 +313,7 @@ func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockT
 	return result, nil
 }
 
-// GetSecretboxPrefixTransformer returns a prefix transformer from the provided configuration
-func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
+func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
 	var result value.PrefixTransformer
 
 	if len(config.Keys) == 0 {
@@ -370,9 +363,7 @@ func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguratio
 	return result, nil
 }
 
-// getEnvelopePrefixTransformer returns a prefix transformer from the provided config.
-// envelopeService is used as the root of trust.
-func getEnvelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
+func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
 	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewCBCTransformer)
 	if err != nil {
 		return value.PrefixTransformer{}, err

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -35,6 +35,7 @@ import (
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
+	"k8s.io/klog/v2"
 )
 
 type EtcdOptions struct {
@@ -176,6 +177,9 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 
 	fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+
 		"Frequency of polling etcd for number of resources per type. 0 disables the metric collection.")
+
+	fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
+		"The interval of requests to poll etcd and update metric. 0 disables the metric collection")
 }
 
 func (s *EtcdOptions) ApplyTo(c *server.Config) error {
@@ -235,12 +239,15 @@ func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource)
 		if err != nil {
 			return generic.RESTOptions{}, err
 		}
-		cacheSize, ok := sizes[resource]
-		if !ok {
-			cacheSize = f.Options.DefaultWatchCacheSize
+		size, ok := sizes[resource]
+		if ok && size > 0 {
+			klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
+		}
+		if ok && size <= 0 {
+			ret.Decorator = generic.UndecoratedStorage
+		} else {
+			ret.Decorator = genericregistry.StorageWithCacher()
 		}
-		// depending on cache size this might return an undecorated storage
-		ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
 	}
 	return ret, nil
 }
@@ -269,12 +276,15 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
 		if err != nil {
 			return generic.RESTOptions{}, err
 		}
-		cacheSize, ok := sizes[resource]
-		if !ok {
-			cacheSize = f.Options.DefaultWatchCacheSize
+		size, ok := sizes[resource]
+		if ok && size > 0 {
+			klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
+		}
+		if ok && size <= 0 {
+			ret.Decorator = generic.UndecoratedStorage
+		} else {
+			ret.Decorator = genericregistry.StorageWithCacher()
 		}
-		// depending on cache size this might return an undecorated storage
-		ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
 	}
 
 	return ret, nil

vendor/k8s.io/apiserver/pkg/server/options/events.go

@@ -1,56 +0,0 @@
-/*
-Copyright 2018 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package options
-
-import (
-	"fmt"
-	"os"
-)
-
-// ProcessInfo holds the apiserver process information used to send events
-type ProcessInfo struct {
-	// Name of the api process to identify events
-	Name string
-
-	// Namespace of the api process to send events
-	Namespace string
-}
-
-// NewProcessInfo returns a new process info with the hostname concatenated to the name given
-func NewProcessInfo(name, namespace string) *ProcessInfo {
-	// try to concat the hostname if available
-	host, _ := os.Hostname()
-	if host != "" {
-		name = fmt.Sprintf("%s-%s", name, host)
-	}
-	return &ProcessInfo{
-		Name:      name,
-		Namespace: namespace,
-	}
-}
-
-// validateProcessInfo checks for a complete process info
-func validateProcessInfo(p *ProcessInfo) error {
-	if p == nil {
-		return fmt.Errorf("ProcessInfo must be set")
-	} else if p.Name == "" {
-		return fmt.Errorf("ProcessInfo name must be set")
-	} else if p.Namespace == "" {
-		return fmt.Errorf("ProcessInfo namespace must be set")
-	}
-	return nil
-}

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -48,14 +48,11 @@ type RecommendedOptions struct {
 	// admission plugin initializers to Admission.ApplyTo.
 	ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
 	Admission                  *AdmissionOptions
-	// ProcessInfo is used to identify events created by the server.
-	ProcessInfo *ProcessInfo
-	Webhook     *WebhookOptions
 	// API Server Egress Selector is used to control outbound traffic from the API Server
 	EgressSelector *EgressSelectorOptions
 }
 
-func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
+func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
 	sso := NewSecureServingOptions()
 
 	// We are composing recommended options for an aggregated api-server,
@@ -78,8 +75,6 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc
 		FeatureGate:                feature.DefaultFeatureGate,
 		ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
 		Admission:                  NewAdmissionOptions(),
-		ProcessInfo:                processInfo,
-		Webhook:                    NewWebhookOptions(),
 		EgressSelector:             NewEgressSelectorOptions(),
 	}
 }
@@ -111,7 +106,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
 		return err
 	}
-	if err := o.Audit.ApplyTo(&config.Config, config.ClientConfig, config.SharedInformerFactory, o.ProcessInfo, o.Webhook); err != nil {
+	if err := o.Audit.ApplyTo(&config.Config); err != nil {
 		return err
 	}
 	if err := o.Features.ApplyTo(&config.Config); err != nil {

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -50,7 +50,6 @@ type ServerRunOptions struct {
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
 	MaxRequestBodyBytes       int64
-	TargetRAMMB               int
 	EnablePriorityAndFairness bool
 }
 
@@ -69,7 +68,7 @@ func NewServerRunOptions() *ServerRunOptions {
 	}
 }
 
-// ApplyOptions applies the run options to the method receiver and returns self
+// ApplyTo applies the run options to the method receiver and returns self
 func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.CorsAllowedOriginList = s.CorsAllowedOriginList
 	c.ExternalAddress = s.ExternalHost
@@ -108,9 +107,6 @@ func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions)
 // Validate checks validation of ServerRunOptions
 func (s *ServerRunOptions) Validate() []error {
 	errors := []error{}
-	if s.TargetRAMMB < 0 {
-		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
-	}
 
 	if s.LivezGracePeriod < 0 {
 		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
@@ -165,8 +161,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"List of allowed origins for CORS, comma separated.  An allowed origin can be a regular "+
 		"expression to support subdomain matching. If this list is empty CORS will not be enabled.")
 
-	fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
-		"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
+	deprecatedTargetRAMMB := 0
+	fs.IntVar(&deprecatedTargetRAMMB, "target-ram-mb", deprecatedTargetRAMMB,
+		"DEPRECATED: Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
+	fs.MarkDeprecated("target-ram-mb", "This flag will be removed in v1.23")
 
 	fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
 		"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
@@ -209,8 +207,8 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
 
 	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
-		"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
-		"returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
+		"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
+		"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
 		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
 
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -17,6 +17,7 @@ limitations under the License.
 package options
 
 import (
+	"context"
 	"fmt"
 	"net"
 	"path"
@@ -24,7 +25,7 @@ import (
 	"strings"
 
 	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/server"
@@ -66,6 +67,10 @@ type SecureServingOptions struct {
 	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
 	// A value of zero means to use the default provided by golang's HTTP/2 support.
 	HTTP2MaxStreamsPerConnection int
+
+	// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
+	// more than one instance to bind on the same address and port.
+	PermitPortSharing bool
 }
 
 type CertKey struct {
@@ -166,11 +171,13 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
 		"File containing the default x509 private key matching --tls-cert-file.")
 
-	tlsCipherPossibleValues := cliflag.TLSCipherPossibleValues()
+	tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
+	tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
 	fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
 		"Comma-separated list of cipher suites for the server. "+
-			"If omitted, the default Go cipher suites will be use.  "+
-			"Possible values: "+strings.Join(tlsCipherPossibleValues, ","))
+			"If omitted, the default Go cipher suites will be used. \n"+
+			"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
+			"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
 
 	tlsPossibleVersions := cliflag.TLSPossibleVersions()
 	fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
@@ -192,6 +199,10 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
 		"The limit that the server gives to clients for "+
 		"the maximum number of streams in an HTTP/2 connection. "+
 		"Zero means to use golang's default.")
+
+	fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
+		"If true, SO_REUSEPORT will be used when binding the port, which allows "+
+			"more than one instance to bind on the same address and port. [default=false]")
 }
 
 // ApplyTo fills up serving information in the server configuration.
@@ -206,7 +217,14 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	if s.Listener == nil {
 		var err error
 		addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
-		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
+
+		c := net.ListenConfig{}
+
+		if s.PermitPortSharing {
+			c.Control = permitPortReuse
+		}
+
+		s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
 		if err != nil {
 			return fmt.Errorf("failed to create listener: %v", err)
 		}
@@ -317,11 +335,12 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 	return nil
 }
 
-func CreateListener(network, addr string) (net.Listener, int, error) {
+func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
 	if len(network) == 0 {
 		network = "tcp"
 	}
-	ln, err := net.Listen(network, addr)
+
+	ln, err := config.Listen(context.TODO(), network, addr)
 	if err != nil {
 		return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
 	}

vendor/k8s.io/apiserver/pkg/server/options/serving_unix.go

@@ -1,5 +1,7 @@
+// +build !windows
+
 /*
-Copyright 2018 The Kubernetes Authors.
+Copyright 2020 The Kubernetes Authors.
 
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@@ -17,18 +19,13 @@ limitations under the License.
 package options
 
 import (
-	utilwebhook "k8s.io/apiserver/pkg/util/webhook"
+	"syscall"
+
+	"golang.org/x/sys/unix"
 )
 
-// WebhookOptions holds the outgoing webhook options
-type WebhookOptions struct {
-	ServiceResolver         utilwebhook.ServiceResolver
-	AuthInfoResolverWrapper utilwebhook.AuthenticationInfoResolverWrapper
-}
-
-// NewWebhookOptions returns the default options for outgoing webhooks
-func NewWebhookOptions() *WebhookOptions {
-	return &WebhookOptions{
-		ServiceResolver: utilwebhook.NewDefaultServiceResolver(),
-	}
+func permitPortReuse(network, addr string, conn syscall.RawConn) error {
+	return conn.Control(func(fd uintptr) {
+		syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
+	})
 }

vendor/k8s.io/apiserver/pkg/server/options/serving_windows.go

@@ -0,0 +1,30 @@
+// +build windows
+
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"syscall"
+)
+
+// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as
+// there is no protection against port hijacking.
+func permitPortReuse(network, address string, c syscall.RawConn) error {
+	return fmt.Errorf("port reuse is not supported on Windows")
+}

vendor/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go

@@ -26,7 +26,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
 	cliflag "k8s.io/component-base/cli/flag"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 // GroupVersionRegistry provides access to registered group versions.

vendor/k8s.io/apiserver/pkg/server/routes/flags.go

@@ -24,7 +24,7 @@ import (
 	"path"
 	"sync"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apiserver/pkg/server/mux"
 )

vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -17,9 +17,6 @@ limitations under the License.
 package routes
 
 import (
-	"io"
-	"net/http"
-
 	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/mux"
 	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
@@ -43,16 +40,7 @@ type MetricsWithReset struct{}
 // Install adds the MetricsWithReset handler
 func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
 	register()
-	defaultMetricsHandler := legacyregistry.Handler().ServeHTTP
-	c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
-		if req.Method == "DELETE" {
-			apimetrics.Reset()
-			etcd3metrics.Reset()
-			io.WriteString(w, "metrics reset\n")
-			return
-		}
-		defaultMetricsHandler(w, req)
-	})
+	c.Handle("/metrics", legacyregistry.HandlerWithReset())
 }
 
 // register apiserver and etcd metrics

vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -19,7 +19,7 @@ package routes
 import (
 	restful "github.com/emicklei/go-restful"
 	"github.com/go-openapi/spec"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apiserver/pkg/server/mux"
 	"k8s.io/kube-openapi/pkg/builder"
@@ -38,9 +38,16 @@ func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) (*hand
 	if err != nil {
 		klog.Fatalf("Failed to build open api spec for root: %v", err)
 	}
-	openAPIVersionedService, err := handler.RegisterOpenAPIVersionedService(spec, "/openapi/v2", mux)
+
+	openAPIVersionedService, err := handler.NewOpenAPIService(spec)
+	if err != nil {
+		klog.Fatalf("Failed to create OpenAPIService: %v", err)
+	}
+
+	err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
 	if err != nil {
 		klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
 	}
+
 	return openAPIVersionedService, spec
 }

vendor/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -20,14 +20,20 @@ import (
 	"context"
 	"crypto/tls"
 	"fmt"
+	"io"
+	"log"
 	"net"
 	"net/http"
+	"os"
+	"strings"
 	"time"
 
 	"golang.org/x/net/http2"
-	"k8s.io/klog"
+	"k8s.io/component-base/cli/flag"
+	"k8s.io/klog/v2"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 )
 
@@ -56,6 +62,14 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
 	}
 	if len(s.CipherSuites) > 0 {
 		tlsConfig.CipherSuites = s.CipherSuites
+		insecureCiphers := flag.InsecureTLSCiphers()
+		for i := 0; i < len(s.CipherSuites); i++ {
+			for cipherName, cipherID := range insecureCiphers {
+				if s.CipherSuites[i] == cipherID {
+					klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
+				}
+			}
+		}
 	}
 
 	if s.ClientCA != nil {
@@ -175,6 +189,11 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
 		}
 	}
 
+	// use tlsHandshakeErrorWriter to handle messages of tls handshake error
+	tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
+	tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
+	secureServer.ErrorLog = tlsErrorLogger
+
 	klog.Infof("Serving securely on %s", secureServer.Addr)
 	return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
 }
@@ -209,7 +228,7 @@ func RunServer(
 		defer utilruntime.HandleCrash()
 
 		var listener net.Listener
-		listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
+		listener = tcpKeepAliveListener{ln}
 		if server.TLSConfig != nil {
 			listener = tls.NewListener(listener, server.TLSConfig)
 		}
@@ -235,15 +254,36 @@ func RunServer(
 //
 // Copied from Go 1.7.2 net/http/server.go
 type tcpKeepAliveListener struct {
-	*net.TCPListener
+	net.Listener
 }
 
 func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
-	tc, err := ln.AcceptTCP()
+	c, err := ln.Listener.Accept()
 	if err != nil {
 		return nil, err
 	}
-	tc.SetKeepAlive(true)
-	tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
-	return tc, nil
+	if tc, ok := c.(*net.TCPConn); ok {
+		tc.SetKeepAlive(true)
+		tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
+	}
+	return c, nil
+}
+
+// tlsHandshakeErrorWriter writes TLS handshake errors to klog with
+// trace level - V(5), to avoid flooding of tls handshake errors.
+type tlsHandshakeErrorWriter struct {
+	out io.Writer
+}
+
+const tlsHandshakeErrorPrefix = "http: TLS handshake error"
+
+func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
+	if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
+		klog.V(5).Info(string(p))
+		metrics.TLSHandshakeErrors.Inc()
+		return len(p), nil
+	}
+
+	// for non tls handshake error, log it as usual
+	return w.out.Write(p)
 }

vendor/k8s.io/apiserver/pkg/server/signal.go

@@ -17,6 +17,7 @@ limitations under the License.
 package server
 
 import (
+	"context"
 	"os"
 	"os/signal"
 )
@@ -27,21 +28,30 @@ var shutdownHandler chan os.Signal
 // SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
 // which is closed on one of these signals. If a second signal is caught, the program
 // is terminated with exit code 1.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
 func SetupSignalHandler() <-chan struct{} {
+	return SetupSignalContext().Done()
+}
+
+// SetupSignalContext is same as SetupSignalHandler, but a context.Context is returned.
+// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
+// be called once.
+func SetupSignalContext() context.Context {
 	close(onlyOneSignalHandler) // panics when called twice
 
 	shutdownHandler = make(chan os.Signal, 2)
 
-	stop := make(chan struct{})
+	ctx, cancel := context.WithCancel(context.Background())
 	signal.Notify(shutdownHandler, shutdownSignals...)
 	go func() {
 		<-shutdownHandler
-		close(stop)
+		cancel()
 		<-shutdownHandler
 		os.Exit(1) // second signal. Exit directly.
 	}()
 
-	return stop
+	return ctx
 }
 
 // RequestShutdown emulates a received event that is considered as shutdown signal (SIGTERM/SIGINT)

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -22,7 +22,7 @@ import (
 	"io/ioutil"
 	"strings"
 
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -15,7 +15,6 @@ reviewers:
 - mikedanese
 - liggitt
 - ncdc
-- tallclair
 - timothysc
 - hongchaodeng
 - krousey

vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -39,29 +39,11 @@ import (
 	"k8s.io/apiserver/pkg/storage"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/component-base/metrics"
-	"k8s.io/component-base/metrics/legacyregistry"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	utiltrace "k8s.io/utils/trace"
 )
 
-/*
- * By default, all the following metrics are defined as falling under
- * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
- *
- * Promoting the stability level of the metric is a responsibility of the component owner, since it
- * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
- * the metric stability policy.
- */
 var (
-	initCounter = metrics.NewCounterVec(
-		&metrics.CounterOpts{
-			Name:           "apiserver_init_events_total",
-			Help:           "Counter of init events processed in watchcache broken by resource type",
-			StabilityLevel: metrics.ALPHA,
-		},
-		[]string{"resource"},
-	)
 	emptyFunc = func() {}
 )
 
@@ -69,17 +51,15 @@ const (
 	// storageWatchListPageSize is the cacher's request chunk size of
 	// initial and resync watch lists to storage.
 	storageWatchListPageSize = int64(10000)
+	// defaultBookmarkFrequency defines how frequently watch bookmarks should be send
+	// in addition to sending a bookmark right before watch deadline.
+	//
+	// NOTE: Update `eventFreshDuration` when changing this value.
+	defaultBookmarkFrequency = time.Minute
 )
 
-func init() {
-	legacyregistry.MustRegister(initCounter)
-}
-
 // Config contains the configuration for a given Cache.
 type Config struct {
-	// Maximum size of the history cached in memory.
-	CacheCapacity int
-
 	// An underlying storage.Interface.
 	Storage storage.Interface
 
@@ -112,6 +92,8 @@ type Config struct {
 	NewListFunc func() runtime.Object
 
 	Codec runtime.Codec
+
+	Clock clock.Clock
 }
 
 type watchersMap map[int]*cacheWatcher
@@ -176,24 +158,26 @@ func (i *indexedWatchers) terminateAll(objectType reflect.Type, done func(*cache
 // second in a bucket, and pop up them once at the timeout. To be more specific,
 // if you set fire time at X, you can get the bookmark within (X-1,X+1) period.
 type watcherBookmarkTimeBuckets struct {
-	lock            sync.Mutex
-	watchersBuckets map[int64][]*cacheWatcher
-	startBucketID   int64
-	clock           clock.Clock
+	lock              sync.Mutex
+	watchersBuckets   map[int64][]*cacheWatcher
+	startBucketID     int64
+	clock             clock.Clock
+	bookmarkFrequency time.Duration
 }
 
-func newTimeBucketWatchers(clock clock.Clock) *watcherBookmarkTimeBuckets {
+func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *watcherBookmarkTimeBuckets {
 	return &watcherBookmarkTimeBuckets{
-		watchersBuckets: make(map[int64][]*cacheWatcher),
-		startBucketID:   clock.Now().Unix(),
-		clock:           clock,
+		watchersBuckets:   make(map[int64][]*cacheWatcher),
+		startBucketID:     clock.Now().Unix(),
+		clock:             clock,
+		bookmarkFrequency: bookmarkFrequency,
 	}
 }
 
 // adds a watcher to the bucket, if the deadline is before the start, it will be
 // added to the first one.
 func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
-	nextTime, ok := w.nextBookmarkTime(t.clock.Now())
+	nextTime, ok := w.nextBookmarkTime(t.clock.Now(), t.bookmarkFrequency)
 	if !ok {
 		return false
 	}
@@ -336,11 +320,14 @@ func NewCacherFromConfig(config Config) (*Cacher, error) {
 		}
 	}
 
-	clock := clock.RealClock{}
+	if config.Clock == nil {
+		config.Clock = clock.RealClock{}
+	}
+	objType := reflect.TypeOf(obj)
 	cacher := &Cacher{
 		ready:          newReady(),
 		storage:        config.Storage,
-		objectType:     reflect.TypeOf(obj),
+		objectType:     objType,
 		versioner:      config.Versioner,
 		newFunc:        config.NewFunc,
 		indexedTrigger: indexedTrigger,
@@ -358,9 +345,9 @@ func NewCacherFromConfig(config Config) (*Cacher, error) {
 		// and there are no guarantees on the order that they will stop.
 		// So we will be simply closing the channel, and synchronizing on the WaitGroup.
 		stopCh:           stopCh,
-		clock:            clock,
+		clock:            config.Clock,
 		timer:            time.NewTimer(time.Duration(0)),
-		bookmarkWatchers: newTimeBucketWatchers(clock),
+		bookmarkWatchers: newTimeBucketWatchers(config.Clock, defaultBookmarkFrequency),
 	}
 
 	// Ensure that timer is stopped.
@@ -371,7 +358,7 @@ func NewCacherFromConfig(config Config) (*Cacher, error) {
 	}
 
 	watchCache := newWatchCache(
-		config.CacheCapacity, config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers)
+		config.KeyFunc, cacher.processEvent, config.GetAttrsFunc, config.Versioner, config.Indexers, config.Clock, objType)
 	listerWatcher := NewCacherListerWatcher(config.Storage, config.ResourcePrefix, config.NewListFunc)
 	reflectorName := "storage/cacher.go:" + config.ResourcePrefix
 
@@ -412,6 +399,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
 	c.watchCache.SetOnReplace(func() {
 		successfulList = true
 		c.ready.set(true)
+		klog.V(1).Infof("cacher (%v): initialized", c.objectType.String())
 	})
 	defer func() {
 		if successfulList {
@@ -425,7 +413,7 @@ func (c *Cacher) startCaching(stopChannel <-chan struct{}) {
 	// Also note that startCaching is called in a loop, so there's no need
 	// to have another loop here.
 	if err := c.reflector.ListAndWatch(stopChannel); err != nil {
-		klog.Errorf("unexpected ListAndWatch error: %v", err)
+		klog.Errorf("cacher (%v): unexpected ListAndWatch error: %v; reinitializing...", c.objectType.String(), err)
 	}
 }
 
@@ -445,8 +433,9 @@ func (c *Cacher) Delete(ctx context.Context, key string, out runtime.Object, pre
 }
 
 // Watch implements storage.Interface.
-func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
-	watchRV, err := c.versioner.ParseResourceVersion(resourceVersion)
+func (c *Cacher) Watch(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
+	pred := opts.Predicate
+	watchRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion)
 	if err != nil {
 		return nil, err
 	}
@@ -529,22 +518,22 @@ func (c *Cacher) Watch(ctx context.Context, key string, resourceVersion string,
 }
 
 // WatchList implements storage.Interface.
-func (c *Cacher) WatchList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate) (watch.Interface, error) {
-	return c.Watch(ctx, key, resourceVersion, pred)
+func (c *Cacher) WatchList(ctx context.Context, key string, opts storage.ListOptions) (watch.Interface, error) {
+	return c.Watch(ctx, key, opts)
 }
 
 // Get implements storage.Interface.
-func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, objPtr runtime.Object, ignoreNotFound bool) error {
-	if resourceVersion == "" {
+func (c *Cacher) Get(ctx context.Context, key string, opts storage.GetOptions, objPtr runtime.Object) error {
+	if opts.ResourceVersion == "" {
 		// If resourceVersion is not specified, serve it from underlying
 		// storage (for backward compatibility).
-		return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
+		return c.storage.Get(ctx, key, opts, objPtr)
 	}
 
 	// If resourceVersion is specified, serve it from cache.
 	// It's guaranteed that the returned value is at least that
 	// fresh as the given resourceVersion.
-	getRV, err := c.versioner.ParseResourceVersion(resourceVersion)
+	getRV, err := c.versioner.ParseResourceVersion(opts.ResourceVersion)
 	if err != nil {
 		return err
 	}
@@ -552,7 +541,7 @@ func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, ob
 	if getRV == 0 && !c.ready.check() {
 		// If Cacher is not yet initialized and we don't require any specific
 		// minimal resource version, simply forward the request to storage.
-		return c.storage.Get(ctx, key, resourceVersion, objPtr, ignoreNotFound)
+		return c.storage.Get(ctx, key, opts, objPtr)
 	}
 
 	// Do not create a trace - it's not for free and there are tons
@@ -577,7 +566,7 @@ func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, ob
 		objVal.Set(reflect.ValueOf(elem.Object).Elem())
 	} else {
 		objVal.Set(reflect.Zero(objVal.Type()))
-		if !ignoreNotFound {
+		if !opts.IgnoreNotFound {
 			return storage.NewKeyNotFoundError(key, int64(readResourceVersion))
 		}
 	}
@@ -585,18 +574,20 @@ func (c *Cacher) Get(ctx context.Context, key string, resourceVersion string, ob
 }
 
 // GetToList implements storage.Interface.
-func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+func (c *Cacher) GetToList(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
+	resourceVersion := opts.ResourceVersion
+	pred := opts.Predicate
 	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	hasContinuation := pagingEnabled && len(pred.Continue) > 0
 	hasLimit := pagingEnabled && pred.Limit > 0 && resourceVersion != "0"
-	if resourceVersion == "" || hasContinuation || hasLimit {
+	if resourceVersion == "" || hasContinuation || hasLimit || opts.ResourceVersionMatch == metav1.ResourceVersionMatchExact {
 		// If resourceVersion is not specified, serve it from underlying
 		// storage (for backward compatibility). If a continuation is
 		// requested, serve it from the underlying storage as well.
 		// Limits are only sent to storage when resourceVersion is non-zero
 		// since the watch cache isn't able to perform continuations, and
 		// limits are ignored when resource version is zero
-		return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
+		return c.storage.GetToList(ctx, key, opts, listObj)
 	}
 
 	// If resourceVersion is specified, serve it from cache.
@@ -610,7 +601,7 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 	if listRV == 0 && !c.ready.check() {
 		// If Cacher is not yet initialized and we don't require any specific
 		// minimal resource version, simply forward the request to storage.
-		return c.storage.GetToList(ctx, key, resourceVersion, pred, listObj)
+		return c.storage.GetToList(ctx, key, opts, listObj)
 	}
 
 	trace := utiltrace.New("cacher list", utiltrace.Field{"type", c.objectType.String()})
@@ -657,18 +648,20 @@ func (c *Cacher) GetToList(ctx context.Context, key string, resourceVersion stri
 }
 
 // List implements storage.Interface.
-func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, pred storage.SelectionPredicate, listObj runtime.Object) error {
+func (c *Cacher) List(ctx context.Context, key string, opts storage.ListOptions, listObj runtime.Object) error {
+	resourceVersion := opts.ResourceVersion
+	pred := opts.Predicate
 	pagingEnabled := utilfeature.DefaultFeatureGate.Enabled(features.APIListChunking)
 	hasContinuation := pagingEnabled && len(pred.Continue) > 0
 	hasLimit := pagingEnabled && pred.Limit > 0 && resourceVersion != "0"
-	if resourceVersion == "" || hasContinuation || hasLimit {
+	if resourceVersion == "" || hasContinuation || hasLimit || opts.ResourceVersionMatch == metav1.ResourceVersionMatchExact {
 		// If resourceVersion is not specified, serve it from underlying
 		// storage (for backward compatibility). If a continuation is
 		// requested, serve it from the underlying storage as well.
 		// Limits are only sent to storage when resourceVersion is non-zero
 		// since the watch cache isn't able to perform continuations, and
 		// limits are ignored when resource version is zero.
-		return c.storage.List(ctx, key, resourceVersion, pred, listObj)
+		return c.storage.List(ctx, key, opts, listObj)
 	}
 
 	// If resourceVersion is specified, serve it from cache.
@@ -682,7 +675,7 @@ func (c *Cacher) List(ctx context.Context, key string, resourceVersion string, p
 	if listRV == 0 && !c.ready.check() {
 		// If Cacher is not yet initialized and we don't require any specific
 		// minimal resource version, simply forward the request to storage.
-		return c.storage.List(ctx, key, resourceVersion, pred, listObj)
+		return c.storage.List(ctx, key, opts, listObj)
 	}
 
 	trace := utiltrace.New("cacher list", utiltrace.Field{"type", c.objectType.String()})
@@ -935,9 +928,8 @@ func (c *Cacher) startDispatchingBookmarkEvents() {
 				continue
 			}
 			c.watchersBuffer = append(c.watchersBuffer, watcher)
-			// Given that we send bookmark event once at deadline-2s, never push again
-			// after the watcher pops up from the buckets. Once we decide to change the
-			// strategy to more sophisticated, we may need it here.
+			// Requeue the watcher for the next bookmark if needed.
+			c.bookmarkWatchers.addWatcher(watcher)
 		}
 	}
 }
@@ -1098,7 +1090,7 @@ func (lw *cacherListerWatcher) List(options metav1.ListOptions) (runtime.Object,
 		Continue: options.Continue,
 	}
 
-	if err := lw.storage.List(context.TODO(), lw.resourcePrefix, "", pred, list); err != nil {
+	if err := lw.storage.List(context.TODO(), lw.resourcePrefix, storage.ListOptions{ResourceVersionMatch: options.ResourceVersionMatch, Predicate: pred}, list); err != nil {
 		return nil, err
 	}
 	return list, nil
@@ -1106,7 +1098,7 @@ func (lw *cacherListerWatcher) List(options metav1.ListOptions) (runtime.Object,
 
 // Implements cache.ListerWatcher interface.
 func (lw *cacherListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
-	return lw.storage.WatchList(context.TODO(), lw.resourcePrefix, options.ResourceVersion, storage.Everything)
+	return lw.storage.WatchList(context.TODO(), lw.resourcePrefix, storage.ListOptions{ResourceVersion: options.ResourceVersion, Predicate: storage.Everything})
 }
 
 // errWatcher implements watch.Interface to return a single error
@@ -1240,13 +1232,28 @@ func (c *cacheWatcher) add(event *watchCacheEvent, timer *time.Timer) bool {
 	}
 }
 
-func (c *cacheWatcher) nextBookmarkTime(now time.Time) (time.Time, bool) {
-	// For now we return 2s before deadline (and maybe +infinity is now already passed this time)
-	// but it gives us extensibility for the future(false when deadline is not set).
+func (c *cacheWatcher) nextBookmarkTime(now time.Time, bookmarkFrequency time.Duration) (time.Time, bool) {
+	// We try to send bookmarks:
+	// (a) roughly every minute
+	// (b) right before the watcher timeout - for now we simply set it 2s before
+	//     the deadline
+	// The former gives us periodicity if the watch breaks due to unexpected
+	// conditions, the later ensures that on timeout the watcher is as close to
+	// now as possible - this covers 99% of cases.
+	heartbeatTime := now.Add(bookmarkFrequency)
 	if c.deadline.IsZero() {
-		return c.deadline, false
+		// Timeout is set by our client libraries (e.g. reflector) as well as defaulted by
+		// apiserver if properly configured. So this shoudln't happen in practice.
+		return heartbeatTime, true
 	}
-	return c.deadline.Add(-2 * time.Second), true
+	if pretimeoutTime := c.deadline.Add(-2 * time.Second); pretimeoutTime.Before(heartbeatTime) {
+		heartbeatTime = pretimeoutTime
+	}
+
+	if heartbeatTime.Before(now) {
+		return time.Time{}, false
+	}
+	return heartbeatTime, true
 }
 
 func getEventObject(object runtime.Object) runtime.Object {

vendor/k8s.io/apiserver/pkg/storage/cacher/caching_object.go

@@ -30,7 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 )
 
 var _ runtime.CacheableObject = &cachingObject{}

vendor/k8s.io/apiserver/pkg/storage/cacher/metrics.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cacher
+
+import (
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+/*
+ * By default, all the following metrics are defined as falling under
+ * ALPHA stability level https://github.com/kubernetes/enhancements/blob/master/keps/sig-instrumentation/20190404-kubernetes-control-plane-metrics-stability.md#stability-classes)
+ *
+ * Promoting the stability level of the metric is a responsibility of the component owner, since it
+ * involves explicitly acknowledging support for the metric across multiple releases, in accordance with
+ * the metric stability policy.
+ */
+var (
+	initCounter = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "apiserver_init_events_total",
+			Help:           "Counter of init events processed in watchcache broken by resource type.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	watchCacheCapacityIncreaseTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "watch_cache_capacity_increase_total",
+			Help:           "Total number of watch cache capacity increase events broken by resource type.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+
+	watchCacheCapacityDecreaseTotal = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Name:           "watch_cache_capacity_decrease_total",
+			Help:           "Total number of watch cache capacity decrease events broken by resource type.",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
+)
+
+func init() {
+	legacyregistry.MustRegister(initCounter)
+	legacyregistry.MustRegister(watchCacheCapacityIncreaseTotal)
+	legacyregistry.MustRegister(watchCacheCapacityDecreaseTotal)
+}
+
+// recordsWatchCacheCapacityChange record watchCache capacity resize(increase or decrease) operations.
+func recordsWatchCacheCapacityChange(objType string, old, new int) {
+	if old < new {
+		watchCacheCapacityIncreaseTotal.WithLabelValues(objType).Inc()
+		return
+	}
+	watchCacheCapacityDecreaseTotal.WithLabelValues(objType).Inc()
+}

vendor/k8s.io/apiserver/pkg/storage/cacher/util.go

@@ -44,3 +44,17 @@ func hasPathPrefix(s, pathPrefix string) bool {
 	}
 	return false
 }
+
+func max(a, b int) int {
+	if a > b {
+		return a
+	}
+	return b
+}
+
+func min(a, b int) int {
+	if a < b {
+		return a
+	}
+	return b
+}

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -18,6 +18,7 @@ package cacher
 
 import (
 	"fmt"
+	"reflect"
 	"sort"
 	"sync"
 	"time"
@@ -30,7 +31,7 @@ import (
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
 	utiltrace "k8s.io/utils/trace"
 )
 
@@ -44,6 +45,19 @@ const (
 	// resourceVersionTooHighRetrySeconds is the seconds before a operation should be retried by the client
 	// after receiving a 'too high resource version' error.
 	resourceVersionTooHighRetrySeconds = 1
+
+	// eventFreshDuration is time duration of events we want to keep.
+	// We set it to `defaultBookmarkFrequency` plus epsilon to maximize
+	// chances that last bookmark was sent within kept history, at the
+	// same time, minimizing the needed memory usage.
+	eventFreshDuration = 75 * time.Second
+
+	// defaultLowerBoundCapacity is a default value for event cache capacity's lower bound.
+	// TODO: Figure out, to what value we can decreased it.
+	defaultLowerBoundCapacity = 100
+
+	// defaultUpperBoundCapacity  should be able to keep eventFreshDuration of history.
+	defaultUpperBoundCapacity = 100 * 1024
 )
 
 // watchCacheEvent is a single "watch event" that is send to users of
@@ -60,6 +74,7 @@ type watchCacheEvent struct {
 	PrevObjFields   fields.Set
 	Key             string
 	ResourceVersion uint64
+	RecordTime      time.Time
 }
 
 // Computing a key of an object is generally non-trivial (it performs
@@ -126,6 +141,12 @@ type watchCache struct {
 	// Maximum size of history window.
 	capacity int
 
+	// upper bound of capacity since event cache has a dynamic size.
+	upperBoundCapacity int
+
+	// lower bound of capacity since event cache has a dynamic size.
+	lowerBoundCapacity int
+
 	// keyFunc is used to get a key in the underlying storage for a given object.
 	keyFunc func(runtime.Object) (string, error)
 
@@ -165,28 +186,35 @@ type watchCache struct {
 
 	// An underlying storage.Versioner.
 	versioner storage.Versioner
+
+	// cacher's objectType.
+	objectType reflect.Type
 }
 
 func newWatchCache(
-	capacity int,
 	keyFunc func(runtime.Object) (string, error),
 	eventHandler func(*watchCacheEvent),
 	getAttrsFunc func(runtime.Object) (labels.Set, fields.Set, error),
 	versioner storage.Versioner,
-	indexers *cache.Indexers) *watchCache {
+	indexers *cache.Indexers,
+	clock clock.Clock,
+	objectType reflect.Type) *watchCache {
 	wc := &watchCache{
-		capacity:            capacity,
+		capacity:            defaultLowerBoundCapacity,
 		keyFunc:             keyFunc,
 		getAttrsFunc:        getAttrsFunc,
-		cache:               make([]*watchCacheEvent, capacity),
+		cache:               make([]*watchCacheEvent, defaultLowerBoundCapacity),
+		lowerBoundCapacity:  defaultLowerBoundCapacity,
+		upperBoundCapacity:  defaultUpperBoundCapacity,
 		startIndex:          0,
 		endIndex:            0,
 		store:               cache.NewIndexer(storeElementKey, storeElementIndexers(indexers)),
 		resourceVersion:     0,
 		listResourceVersion: 0,
 		eventHandler:        eventHandler,
-		clock:               clock.RealClock{},
+		clock:               clock,
 		versioner:           versioner,
+		objectType:          objectType,
 	}
 	wc.cond = sync.NewCond(wc.RLocker())
 	return wc
@@ -260,6 +288,7 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 		ObjFields:       elem.Fields,
 		Key:             key,
 		ResourceVersion: resourceVersion,
+		RecordTime:      w.clock.Now(),
 	}
 
 	if err := func() error {
@@ -301,7 +330,8 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 
 // Assumes that lock is already held for write.
 func (w *watchCache) updateCache(event *watchCacheEvent) {
-	if w.endIndex == w.startIndex+w.capacity {
+	w.resizeCacheLocked(event.RecordTime)
+	if w.isCacheFullLocked() {
 		// Cache is full - remove the oldest element.
 		w.startIndex++
 	}
@@ -309,6 +339,48 @@ func (w *watchCache) updateCache(event *watchCacheEvent) {
 	w.endIndex++
 }
 
+// resizeCacheLocked resizes the cache if necessary:
+// - increases capacity by 2x if cache is full and all cached events occurred within last eventFreshDuration.
+// - decreases capacity by 2x when recent quarter of events occurred outside of eventFreshDuration(protect watchCache from flapping).
+func (w *watchCache) resizeCacheLocked(eventTime time.Time) {
+	if w.isCacheFullLocked() && eventTime.Sub(w.cache[w.startIndex%w.capacity].RecordTime) < eventFreshDuration {
+		capacity := min(w.capacity*2, w.upperBoundCapacity)
+		if capacity > w.capacity {
+			w.doCacheResizeLocked(capacity)
+		}
+		return
+	}
+	if w.isCacheFullLocked() && eventTime.Sub(w.cache[(w.endIndex-w.capacity/4)%w.capacity].RecordTime) > eventFreshDuration {
+		capacity := max(w.capacity/2, w.lowerBoundCapacity)
+		if capacity < w.capacity {
+			w.doCacheResizeLocked(capacity)
+		}
+		return
+	}
+}
+
+// isCacheFullLocked used to judge whether watchCacheEvent is full.
+// Assumes that lock is already held for write.
+func (w *watchCache) isCacheFullLocked() bool {
+	return w.endIndex == w.startIndex+w.capacity
+}
+
+// doCacheResizeLocked resize watchCache's event array with different capacity.
+// Assumes that lock is already held for write.
+func (w *watchCache) doCacheResizeLocked(capacity int) {
+	newCache := make([]*watchCacheEvent, capacity)
+	if capacity < w.capacity {
+		// adjust startIndex if cache capacity shrink.
+		w.startIndex = w.endIndex - capacity
+	}
+	for i := w.startIndex; i < w.endIndex; i++ {
+		newCache[i%capacity] = w.cache[i%w.capacity]
+	}
+	w.cache = newCache
+	recordsWatchCacheCapacityChange(w.objectType.String(), w.capacity, capacity)
+	w.capacity = capacity
+}
+
 // List returns list of pointers to <storeElement> objects.
 func (w *watchCache) List() []interface{} {
 	return w.store.List()
@@ -460,19 +532,16 @@ func (w *watchCache) GetAllEventsSinceThreadUnsafe(resourceVersion uint64) ([]*w
 	size := w.endIndex - w.startIndex
 	var oldest uint64
 	switch {
-	case size >= w.capacity:
-		// Once the watch event buffer is full, the oldest watch event we can deliver
-		// is the first one in the buffer.
-		oldest = w.cache[w.startIndex%w.capacity].ResourceVersion
-	case w.listResourceVersion > 0:
-		// If the watch event buffer isn't full, the oldest watch event we can deliver
-		// is one greater than the resource version of the last full list.
+	case w.listResourceVersion > 0 && w.startIndex == 0:
+		// If no event was removed from the buffer since last relist, the oldest watch
+		// event we can deliver is one greater than the resource version of the list.
 		oldest = w.listResourceVersion + 1
 	case size > 0:
-		// If we've never completed a list, use the resourceVersion of the oldest event
-		// in the buffer.
-		// This should only happen in unit tests that populate the buffer without
-		// performing list/replace operations.
+		// If the previous condition is not satisfied: either some event was already
+		// removed from the buffer or we've never completed a list (the latter can
+		// only happen in unit tests that populate the buffer without performing
+		// list/replace operations), the oldest watch event we can deliver is the first
+		// one in the buffer.
 		oldest = w.cache[w.startIndex%w.capacity].ResourceVersion
 	default:
 		return nil, fmt.Errorf("watch cache isn't correctly initialized")