vendor: revendor

vendor/k8s.io/apiserver/pkg/admission/initializer/interfaces.go

@@ -19,6 +19,7 @@ package initializer
 import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
+	quota "k8s.io/apiserver/pkg/quota/v1"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/component-base/featuregate"
@@ -42,6 +43,12 @@ type WantsAuthorizer interface {
 	admission.InitializationValidator
 }
 
+// WantsQuotaConfiguration defines a function which sets quota configuration for admission plugins that need it.
+type WantsQuotaConfiguration interface {
+	SetQuotaConfiguration(quota.Configuration)
+	admission.InitializationValidator
+}
+
 // WantsFeatureGate defines a function which passes the featureGates for inspection by an admission plugin.
 // Admission plugins should not hold a reference to the featureGates.  Instead, they should query a particular one
 // and assign it to a simple bool in the admission plugin struct.

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/generic/webhook.go

@@ -141,7 +141,18 @@ func (a *Webhook) ValidateInitialization() error {
 // ShouldCallHook returns invocation details if the webhook should be called, nil if the webhook should not be called,
 // or an error if an error was encountered during evaluation.
 func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attributes, o admission.ObjectInterfaces) (*WebhookInvocation, *apierrors.StatusError) {
-	var err *apierrors.StatusError
+	matches, matchNsErr := a.namespaceMatcher.MatchNamespaceSelector(h, attr)
+	// Should not return an error here for webhooks which do not apply to the request, even if err is an unexpected scenario.
+	if !matches && matchNsErr == nil {
+		return nil, nil
+	}
+
+	// Should not return an error here for webhooks which do not apply to the request, even if err is an unexpected scenario.
+	matches, matchObjErr := a.objectMatcher.MatchObjectSelector(h, attr)
+	if !matches && matchObjErr == nil {
+		return nil, nil
+	}
+
 	var invocation *WebhookInvocation
 	for _, r := range h.GetRules() {
 		m := rules.Matcher{Rule: r, Attr: attr}
@@ -189,15 +200,11 @@ func (a *Webhook) ShouldCallHook(h webhook.WebhookAccessor, attr admission.Attri
 	if invocation == nil {
 		return nil, nil
 	}
-
-	matches, err := a.namespaceMatcher.MatchNamespaceSelector(h, attr)
-	if !matches || err != nil {
-		return nil, err
+	if matchNsErr != nil {
+		return nil, matchNsErr
 	}
-
-	matches, err = a.objectMatcher.MatchObjectSelector(h, attr)
-	if !matches || err != nil {
-		return nil, err
+	if matchObjErr != nil {
+		return nil, matchObjErr
 	}
 
 	return invocation, nil

vendor/k8s.io/apiserver/pkg/admission/plugin/webhook/mutating/dispatcher.go

@@ -57,7 +57,7 @@ const (
 	MutationAuditAnnotationPrefix = "mutation.webhook.admission.k8s.io/"
 )
 
-var encodingjson = json.CaseSensitiveJsonIterator()
+var encodingjson = json.CaseSensitiveJSONIterator()
 
 type mutatingDispatcher struct {
 	cm     *webhookutil.ClientManager

vendor/k8s.io/apiserver/pkg/apis/apiserver/types.go

@@ -62,7 +62,8 @@ type EgressSelectorConfiguration struct {
 // EgressSelection provides the configuration for a single egress selection client.
 type EgressSelection struct {
 	// Name is the name of the egress selection.
-	// Currently supported values are "Master", "Etcd" and "Cluster"
+	// Currently supported values are "controlplane", "master", "etcd" and "cluster"
+	// The "master" egress selector is deprecated in favor of "controlplane"
 	Name string
 
 	// Connection is the exact information used to configure the egress selection

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1alpha1/types.go

@@ -62,7 +62,8 @@ type EgressSelectorConfiguration struct {
 // EgressSelection provides the configuration for a single egress selection client.
 type EgressSelection struct {
 	// name is the name of the egress selection.
-	// Currently supported values are "Master", "Etcd" and "Cluster"
+	// Currently supported values are "controlplane", "master", "etcd" and "cluster"
+	// The "master" egress selector is deprecated in favor of "controlplane"
 	Name string `json:"name"`
 
 	// connection is the exact information used to configure the egress selection

vendor/k8s.io/apiserver/pkg/apis/apiserver/v1beta1/types.go

@@ -33,7 +33,8 @@ type EgressSelectorConfiguration struct {
 // EgressSelection provides the configuration for a single egress selection client.
 type EgressSelection struct {
 	// name is the name of the egress selection.
-	// Currently supported values are "Master", "Etcd" and "Cluster"
+	// Currently supported values are "controlplane", "master", "etcd" and "cluster"
+	// The "master" egress selector is deprecated in favor of "controlplane"
 	Name string `json:"name"`
 
 	// connection is the exact information used to configure the egress selection

vendor/k8s.io/apiserver/pkg/apis/audit/v1/generated.proto

@@ -17,7 +17,7 @@ limitations under the License.
 
 // This file was autogenerated by go-to-protobuf. Do not edit it manually!
 
-syntax = 'proto2';
+syntax = "proto2";
 
 package k8s.io.apiserver.pkg.apis.audit.v1;
 

vendor/k8s.io/apiserver/pkg/apis/audit/v1alpha1/generated.proto

@@ -17,7 +17,7 @@ limitations under the License.
 
 // This file was autogenerated by go-to-protobuf. Do not edit it manually!
 
-syntax = 'proto2';
+syntax = "proto2";
 
 package k8s.io.apiserver.pkg.apis.audit.v1alpha1;
 

vendor/k8s.io/apiserver/pkg/apis/audit/v1beta1/generated.proto

@@ -17,7 +17,7 @@ limitations under the License.
 
 // This file was autogenerated by go-to-protobuf. Do not edit it manually!
 
-syntax = 'proto2';
+syntax = "proto2";
 
 package k8s.io.apiserver.pkg.apis.audit.v1beta1;
 

vendor/k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap/default.go

@@ -19,7 +19,7 @@ package bootstrap
 import (
 	coordinationv1 "k8s.io/api/coordination/v1"
 	corev1 "k8s.io/api/core/v1"
-	flowcontrol "k8s.io/api/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/apiserver/pkg/authentication/user"
@@ -32,8 +32,8 @@ import (
 // because that would require importing k8s.io/kubernetes).
 var (
 	MandatoryPriorityLevelConfigurations = []*flowcontrol.PriorityLevelConfiguration{
-		MandatoryPriorityLevelConfigurationExempt,
 		MandatoryPriorityLevelConfigurationCatchAll,
+		MandatoryPriorityLevelConfigurationExempt,
 	}
 	MandatoryFlowSchemas = []*flowcontrol.FlowSchema{
 		MandatoryFlowSchemaExempt,
@@ -87,7 +87,7 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				AssuredConcurrencyShares: 1,
+				AssuredConcurrencyShares: 5,
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeReject,
 				},
@@ -210,7 +210,7 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				AssuredConcurrencyShares: 20,
+				AssuredConcurrencyShares: 100,
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{
@@ -227,7 +227,7 @@ var (
 		flowcontrol.PriorityLevelConfigurationSpec{
 			Type: flowcontrol.PriorityLevelEnablementLimited,
 			Limited: &flowcontrol.LimitedPriorityLevelConfiguration{
-				AssuredConcurrencyShares: 100,
+				AssuredConcurrencyShares: 20,
 				LimitResponse: flowcontrol.LimitResponse{
 					Type: flowcontrol.LimitResponseTypeQueue,
 					Queuing: &flowcontrol.QueuingConfiguration{

vendor/k8s.io/apiserver/pkg/audit/request.go

@@ -43,9 +43,9 @@ const (
 	userAgentTruncateSuffix = "...TRUNCATED"
 )
 
-func NewEventFromRequest(req *http.Request, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
+func NewEventFromRequest(req *http.Request, requestReceivedTimestamp time.Time, level auditinternal.Level, attribs authorizer.Attributes) (*auditinternal.Event, error) {
 	ev := &auditinternal.Event{
-		RequestReceivedTimestamp: metav1.NewMicroTime(time.Now()),
+		RequestReceivedTimestamp: metav1.NewMicroTime(requestReceivedTimestamp),
 		Verb:                     attribs.GetVerb(),
 		RequestURI:               req.URL.RequestURI(),
 		UserAgent:                maybeTruncateUserAgent(req),

vendor/k8s.io/apiserver/pkg/authentication/authenticatorfactory/delegating.go

@@ -22,6 +22,7 @@ import (
 
 	"github.com/go-openapi/spec"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
 	"k8s.io/apiserver/pkg/authentication/group"
 	"k8s.io/apiserver/pkg/authentication/request/anonymous"
@@ -43,6 +44,11 @@ type DelegatingAuthenticatorConfig struct {
 	// TokenAccessReviewClient is a client to do token review. It can be nil. Then every token is ignored.
 	TokenAccessReviewClient authenticationclient.TokenReviewInterface
 
+	// WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
 	// CacheTTL is the length of time that a token authentication answer will be cached.
 	CacheTTL time.Duration
 
@@ -79,7 +85,10 @@ func (c DelegatingAuthenticatorConfig) New() (authenticator.Request, *spec.Secur
 	}
 
 	if c.TokenAccessReviewClient != nil {
-		tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences)
+		if c.WebhookRetryBackoff == nil {
+			return nil, nil, errors.New("retry backoff parameters for delegating authentication webhook has not been specified")
+		}
+		tokenAuth, err := webhooktoken.NewFromInterface(c.TokenAccessReviewClient, c.APIAudiences, *c.WebhookRetryBackoff)
 		if err != nil {
 			return nil, nil, err
 		}

vendor/k8s.io/apiserver/pkg/authentication/request/bearertoken/bearertoken.go

@@ -39,7 +39,7 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.R
 	if auth == "" {
 		return nil, false, nil
 	}
-	parts := strings.Split(auth, " ")
+	parts := strings.SplitN(auth, " ", 3)
 	if len(parts) < 2 || strings.ToLower(parts[0]) != "bearer" {
 		return nil, false, nil
 	}

vendor/k8s.io/apiserver/pkg/authentication/request/x509/x509.go

@@ -19,8 +19,10 @@ package x509
 import (
 	"crypto/x509"
 	"crypto/x509/pkix"
+	"encoding/hex"
 	"fmt"
 	"net/http"
+	"strings"
 	"time"
 
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
@@ -82,6 +84,27 @@ func (f UserConversionFunc) User(chain []*x509.Certificate) (*authenticator.Resp
 	return f(chain)
 }
 
+func columnSeparatedHex(d []byte) string {
+	h := strings.ToUpper(hex.EncodeToString(d))
+	var sb strings.Builder
+	for i, r := range h {
+		sb.WriteRune(r)
+		if i%2 == 1 && i != len(h)-1 {
+			sb.WriteRune(':')
+		}
+	}
+	return sb.String()
+}
+
+func certificateIdentifier(c *x509.Certificate) string {
+	return fmt.Sprintf(
+		"SN=%d, SKID=%s, AKID=%s",
+		c.SerialNumber,
+		columnSeparatedHex(c.SubjectKeyId),
+		columnSeparatedHex(c.AuthorityKeyId),
+	)
+}
+
 // VerifyOptionFunc is function which provides a shallow copy of the VerifyOptions to the authenticator.  This allows
 // for cases where the options (particularly the CAs) can change.  If the bool is false, then the returned VerifyOptions
 // are ignored and the authenticator will express "no opinion".  This allows a clear signal for cases where a CertPool
@@ -129,7 +152,11 @@ func (a *Authenticator) AuthenticateRequest(req *http.Request) (*authenticator.R
 	clientCertificateExpirationHistogram.Observe(remaining.Seconds())
 	chains, err := req.TLS.PeerCertificates[0].Verify(optsCopy)
 	if err != nil {
-		return nil, false, err
+		return nil, false, fmt.Errorf(
+			"verifying certificate %s failed: %w",
+			certificateIdentifier(req.TLS.PeerCertificates[0]),
+			err,
+		)
 	}
 
 	var errlist []error

vendor/k8s.io/apiserver/pkg/authentication/serviceaccount/util.go

@@ -17,10 +17,18 @@ limitations under the License.
 package serviceaccount
 
 import (
+	"context"
 	"fmt"
 	"strings"
 
+	v1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	apimachineryvalidation "k8s.io/apimachinery/pkg/api/validation"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apiserver/pkg/authentication/user"
+	v1core "k8s.io/client-go/kubernetes/typed/core/v1"
+
+	"k8s.io/klog/v2"
 )
 
 const (
@@ -28,6 +36,12 @@ const (
 	ServiceAccountUsernameSeparator = ":"
 	ServiceAccountGroupPrefix       = "system:serviceaccounts:"
 	AllServiceAccountsGroup         = "system:serviceaccounts"
+	// PodNameKey is the key used in a user's "extra" to specify the pod name of
+	// the authenticating request.
+	PodNameKey = "authentication.kubernetes.io/pod-name"
+	// PodUIDKey is the key used in a user's "extra" to specify the pod UID of
+	// the authenticating request.
+	PodUIDKey = "authentication.kubernetes.io/pod-uid"
 )
 
 // MakeUsername generates a username from the given namespace and ServiceAccount name.
@@ -92,3 +106,78 @@ func MakeGroupNames(namespace string) []string {
 func MakeNamespaceGroupName(namespace string) string {
 	return ServiceAccountGroupPrefix + namespace
 }
+
+// UserInfo returns a user.Info interface for the given namespace, service account name and UID
+func UserInfo(namespace, name, uid string) user.Info {
+	return (&ServiceAccountInfo{
+		Name:      name,
+		Namespace: namespace,
+		UID:       uid,
+	}).UserInfo()
+}
+
+type ServiceAccountInfo struct {
+	Name, Namespace, UID string
+	PodName, PodUID      string
+}
+
+func (sa *ServiceAccountInfo) UserInfo() user.Info {
+	info := &user.DefaultInfo{
+		Name:   MakeUsername(sa.Namespace, sa.Name),
+		UID:    sa.UID,
+		Groups: MakeGroupNames(sa.Namespace),
+	}
+	if sa.PodName != "" && sa.PodUID != "" {
+		info.Extra = map[string][]string{
+			PodNameKey: {sa.PodName},
+			PodUIDKey:  {sa.PodUID},
+		}
+	}
+	return info
+}
+
+// IsServiceAccountToken returns true if the secret is a valid api token for the service account
+func IsServiceAccountToken(secret *v1.Secret, sa *v1.ServiceAccount) bool {
+	if secret.Type != v1.SecretTypeServiceAccountToken {
+		return false
+	}
+
+	name := secret.Annotations[v1.ServiceAccountNameKey]
+	uid := secret.Annotations[v1.ServiceAccountUIDKey]
+	if name != sa.Name {
+		// Name must match
+		return false
+	}
+	if len(uid) > 0 && uid != string(sa.UID) {
+		// If UID is specified, it must match
+		return false
+	}
+
+	return true
+}
+
+func GetOrCreateServiceAccount(coreClient v1core.CoreV1Interface, namespace, name string) (*v1.ServiceAccount, error) {
+	sa, err := coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	if err == nil {
+		return sa, nil
+	}
+	if !apierrors.IsNotFound(err) {
+		return nil, err
+	}
+
+	// Create the namespace if we can't verify it exists.
+	// Tolerate errors, since we don't know whether this component has namespace creation permissions.
+	if _, err := coreClient.Namespaces().Get(context.TODO(), namespace, metav1.GetOptions{}); apierrors.IsNotFound(err) {
+		if _, err = coreClient.Namespaces().Create(context.TODO(), &v1.Namespace{ObjectMeta: metav1.ObjectMeta{Name: namespace}}, metav1.CreateOptions{}); err != nil && !apierrors.IsAlreadyExists(err) {
+			klog.Warningf("create non-exist namespace %s failed:%v", namespace, err)
+		}
+	}
+
+	// Create the service account
+	sa, err = coreClient.ServiceAccounts(namespace).Create(context.TODO(), &v1.ServiceAccount{ObjectMeta: metav1.ObjectMeta{Namespace: namespace, Name: name}}, metav1.CreateOptions{})
+	if apierrors.IsAlreadyExists(err) {
+		// If we're racing to init and someone else already created it, re-fetch
+		return coreClient.ServiceAccounts(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+	}
+	return sa, err
+}

vendor/k8s.io/apiserver/pkg/authentication/user/user.go

@@ -70,6 +70,7 @@ func (i *DefaultInfo) GetExtra() map[string][]string {
 const (
 	SystemPrivilegedGroup = "system:masters"
 	NodesGroup            = "system:nodes"
+	MonitoringGroup       = "system:monitoring"
 	AllUnauthenticated    = "system:unauthenticated"
 	AllAuthenticated      = "system:authenticated"
 

vendor/k8s.io/apiserver/pkg/authorization/authorizerfactory/delegating.go

@@ -17,8 +17,10 @@ limitations under the License.
 package authorizerfactory
 
 import (
+	"errors"
 	"time"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/plugin/pkg/authorizer/webhook"
 	authorizationclient "k8s.io/client-go/kubernetes/typed/authorization/v1"
@@ -35,12 +37,22 @@ type DelegatingAuthorizerConfig struct {
 	// DenyCacheTTL is the length of time that an unsuccessful authorization response will be cached.
 	// You generally want more responsive, "deny, try again" flows.
 	DenyCacheTTL time.Duration
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
 }
 
 func (c DelegatingAuthorizerConfig) New() (authorizer.Authorizer, error) {
+	if c.WebhookRetryBackoff == nil {
+		return nil, errors.New("retry backoff parameters for delegating authorization webhook has not been specified")
+	}
+
 	return webhook.NewFromInterface(
 		c.SubjectAccessReviewClient,
 		c.AllowCacheTTL,
 		c.DenyCacheTTL,
+		*c.WebhookRetryBackoff,
 	)
 }

vendor/k8s.io/apiserver/pkg/authorization/union/union.go

@@ -88,7 +88,7 @@ func (authzHandler unionAuthzRulesHandler) RulesFor(user user.Info, namespace st
 	for _, currAuthzHandler := range authzHandler {
 		resourceRules, nonResourceRules, incomplete, err := currAuthzHandler.RulesFor(user, namespace)
 
-		if incomplete == true {
+		if incomplete {
 			incompleteStatus = true
 		}
 		if err != nil {

vendor/k8s.io/apiserver/pkg/endpoints/filterlatency/filterlatency.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filterlatency
+
+import (
+	"context"
+	"net/http"
+	"time"
+
+	utilclock "k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	apirequest "k8s.io/apiserver/pkg/endpoints/request"
+)
+
+type requestFilterRecordKeyType int
+
+// requestFilterRecordKey is the context key for a request filter record struct.
+const requestFilterRecordKey requestFilterRecordKeyType = iota
+
+type requestFilterRecord struct {
+	name             string
+	startedTimestamp time.Time
+}
+
+// withRequestFilterRecord attaches the given request filter record to the parent context.
+func withRequestFilterRecord(parent context.Context, fr *requestFilterRecord) context.Context {
+	return apirequest.WithValue(parent, requestFilterRecordKey, fr)
+}
+
+// requestFilterRecordFrom returns the request filter record from the given context.
+func requestFilterRecordFrom(ctx context.Context) *requestFilterRecord {
+	fr, _ := ctx.Value(requestFilterRecordKey).(*requestFilterRecord)
+	return fr
+}
+
+// TrackStarted measures the timestamp the given handler has started execution
+// by attaching a handler to the chain.
+func TrackStarted(handler http.Handler, name string) http.Handler {
+	return trackStarted(handler, name, utilclock.RealClock{})
+}
+
+// TrackCompleted measures the timestamp the given handler has completed execution and then
+// it updates the corresponding metric with the filter latency duration.
+func TrackCompleted(handler http.Handler) http.Handler {
+	return trackCompleted(handler, utilclock.RealClock{}, func(fr *requestFilterRecord, completedAt time.Time) {
+		metrics.RecordFilterLatency(fr.name, completedAt.Sub(fr.startedTimestamp))
+	})
+}
+
+func trackStarted(handler http.Handler, name string, clock utilclock.PassiveClock) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		ctx := r.Context()
+		if fr := requestFilterRecordFrom(ctx); fr != nil {
+			fr.name = name
+			fr.startedTimestamp = clock.Now()
+
+			handler.ServeHTTP(w, r)
+			return
+		}
+
+		fr := &requestFilterRecord{
+			name:             name,
+			startedTimestamp: clock.Now(),
+		}
+		r = r.WithContext(withRequestFilterRecord(ctx, fr))
+		handler.ServeHTTP(w, r)
+	})
+}
+
+func trackCompleted(handler http.Handler, clock utilclock.PassiveClock, action func(*requestFilterRecord, time.Time)) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		// The previous filter has just completed.
+		completedAt := clock.Now()
+
+		defer handler.ServeHTTP(w, r)
+
+		ctx := r.Context()
+		if fr := requestFilterRecordFrom(ctx); fr != nil {
+			action(fr, completedAt)
+		}
+	})
+}

vendor/k8s.io/apiserver/pkg/endpoints/filters/audit.go

@@ -131,7 +131,11 @@ func createAuditEventAndAttachToContext(req *http.Request, policy policy.Checker
 		return req, nil, nil, nil
 	}
 
-	ev, err := audit.NewEventFromRequest(req, level, attribs)
+	requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
+	if !ok {
+		requestReceivedTimestamp = time.Now()
+	}
+	ev, err := audit.NewEventFromRequest(req, requestReceivedTimestamp, level, attribs)
 	if err != nil {
 		return req, nil, nil, fmt.Errorf("failed to complete audit event from request: %v", err)
 	}

vendor/k8s.io/apiserver/pkg/endpoints/filters/request_received_time.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"net/http"
+
+	utilclock "k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// WithRequestReceivedTimestamp attaches the ReceivedTimestamp (the time the request reached
+// the apiserver) to the context.
+func WithRequestReceivedTimestamp(handler http.Handler) http.Handler {
+	return withRequestReceivedTimestampWithClock(handler, utilclock.RealClock{})
+}
+
+// The clock is passed as a parameter, handy for unit testing.
+func withRequestReceivedTimestampWithClock(handler http.Handler, clock utilclock.PassiveClock) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		req = req.WithContext(request.WithReceivedTimestamp(ctx, clock.Now()))
+
+		handler.ServeHTTP(w, req)
+	})
+}

vendor/k8s.io/apiserver/pkg/endpoints/filters/storageversion.go

@@ -0,0 +1,121 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package filters
+
+import (
+	"errors"
+	"fmt"
+	"net/http"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/storageversion"
+	_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
+	"k8s.io/klog/v2"
+)
+
+// WithStorageVersionPrecondition checks if the storage version barrier has
+// completed, if not, it only passes the following API requests:
+// 1. non-resource requests,
+// 2. read requests,
+// 3. write requests to the storageversion API,
+// 4. create requests to the namespace API sent by apiserver itself,
+// 5. write requests to the lease API in kube-system namespace,
+// 6. resources whose StorageVersion is not pending update, including non-persisted resources.
+func WithStorageVersionPrecondition(handler http.Handler, svm storageversion.Manager, s runtime.NegotiatedSerializer) http.Handler {
+	if svm == nil {
+		// TODO(roycaihw): switch to warning after the feature graduate to beta/GA
+		klog.V(2).Infof("Storage Version barrier is disabled")
+		return handler
+	}
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		if svm.Completed() {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		ctx := req.Context()
+		requestInfo, found := request.RequestInfoFrom(ctx)
+		if !found {
+			responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
+			return
+		}
+		// Allow non-resource requests
+		if !requestInfo.IsResourceRequest {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// Allow read requests
+		if requestInfo.Verb == "get" || requestInfo.Verb == "list" || requestInfo.Verb == "watch" {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// Allow writes to the storage version API
+		if requestInfo.APIGroup == "internal.apiserver.k8s.io" && requestInfo.Resource == "storageversions" {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// The system namespace is required for apiserver-identity lease to exist. Allow the apiserver
+		// itself to create namespaces.
+		// NOTE: with this exception, if the bootstrap client writes namespaces with a new version,
+		// and the upgraded apiserver dies before updating the StorageVersion for namespaces, the
+		// storage migrator won't be able to tell these namespaces are stored in a different version in etcd.
+		// Because the bootstrap client only creates system namespace and doesn't update them, this can
+		// only happen if the upgraded apiserver is the first apiserver that kicks off namespace creation,
+		// or if an upgraded server that joins an existing cluster has new system namespaces (other
+		// than kube-system, kube-public, kube-node-lease) that need to be created.
+		u, hasUser := request.UserFrom(ctx)
+		if requestInfo.APIGroup == "" && requestInfo.Resource == "namespaces" &&
+			requestInfo.Verb == "create" && hasUser &&
+			u.GetName() == user.APIServerUser && contains(u.GetGroups(), user.SystemPrivilegedGroup) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// Allow writes to the lease API in kube-system. The storage version API depends on the
+		// apiserver-identity leases to operate. Leases in kube-system are either apiserver-identity
+		// lease (which gets garbage collected when stale) or leader-election leases (which gets
+		// periodically updated by system components). Both types of leases won't be stale in etcd.
+		if requestInfo.APIGroup == "coordination.k8s.io" && requestInfo.Resource == "leases" &&
+			requestInfo.Namespace == metav1.NamespaceSystem {
+			handler.ServeHTTP(w, req)
+			return
+		}
+		// If the resource's StorageVersion is not in the to-be-updated list, let it pass.
+		// Non-persisted resources are not in the to-be-updated list, so they will pass.
+		gr := schema.GroupResource{requestInfo.APIGroup, requestInfo.Resource}
+		if !svm.PendingUpdate(gr) {
+			handler.ServeHTTP(w, req)
+			return
+		}
+
+		gv := schema.GroupVersion{requestInfo.APIGroup, requestInfo.APIVersion}
+		responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable(fmt.Sprintf("wait for storage version registration to complete for resource: %v, last seen error: %v", gr, svm.LastUpdateError(gr))), s, gv, w, req)
+	})
+}
+
+func contains(s []string, e string) bool {
+	for _, a := range s {
+		if a == e {
+			return true
+		}
+	}
+	return false
+}

vendor/k8s.io/apiserver/pkg/endpoints/groupversion.go

@@ -30,7 +30,9 @@ import (
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
 	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/storageversion"
 	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
 )
 
@@ -71,6 +73,7 @@ type APIGroupVersion struct {
 	Defaulter       runtime.ObjectDefaulter
 	Linker          runtime.SelfLinker
 	UnsafeConvertor runtime.ObjectConvertor
+	TypeConverter   fieldmanager.TypeConverter
 
 	EquivalentResourceRegistry runtime.EquivalentResourceRegistry
 
@@ -94,7 +97,7 @@ type APIGroupVersion struct {
 // InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
 // It is expected that the provided path root prefix will serve all operations. Root MUST NOT end
 // in a slash.
-func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
+func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]*storageversion.ResourceInfo, error) {
 	prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version)
 	installer := &APIInstaller{
 		group:             g,
@@ -102,11 +105,24 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
 		minRequestTimeout: g.MinRequestTimeout,
 	}
 
-	apiResources, ws, registrationErrors := installer.Install()
+	apiResources, resourceInfos, ws, registrationErrors := installer.Install()
 	versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, staticLister{apiResources})
 	versionDiscoveryHandler.AddToWebService(ws)
 	container.Add(ws)
-	return utilerrors.NewAggregate(registrationErrors)
+	return removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors)
+}
+
+func removeNonPersistedResources(infos []*storageversion.ResourceInfo) []*storageversion.ResourceInfo {
+	var filtered []*storageversion.ResourceInfo
+	for _, info := range infos {
+		// if EncodingVersion is empty, then the apiserver does not
+		// need to register this resource via the storage version API,
+		// thus we can remove it.
+		if info != nil && len(info.EncodingVersion) > 0 {
+			filtered = append(filtered, info)
+		}
+	}
+	return filtered
 }
 
 // staticLister implements the APIResourceLister interface

vendor/k8s.io/apiserver/pkg/endpoints/handlers/create.go

@@ -44,6 +44,8 @@ import (
 	utiltrace "k8s.io/utils/trace"
 )
 
+var namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}
+
 func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
 		// For performance tracking purposes.
@@ -76,7 +78,6 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 
 		ctx, cancel := context.WithTimeout(req.Context(), timeout)
 		defer cancel()
-		ctx = request.WithNamespace(ctx, namespace)
 		outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
 		if err != nil {
 			scope.err(err, w, req)
@@ -128,17 +129,21 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 		}
 		trace.Step("Conversion done")
 
+		// On create, get name from new object if unset
+		if len(name) == 0 {
+			_, name, _ = scope.Namer.ObjectName(obj)
+		}
+		if len(namespace) == 0 && *gvk == namespaceGVK {
+			namespace = name
+		}
+		ctx = request.WithNamespace(ctx, namespace)
+
 		ae := request.AuditEventFrom(ctx)
 		admit = admission.WithAudit(admit, ae)
 		audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
 
 		userInfo, _ := request.UserFrom(ctx)
 
-		// On create, get name from new object if unset
-		if len(name) == 0 {
-			_, name, _ = scope.Namer.ObjectName(obj)
-		}
-
 		trace.Step("About to store object in database")
 		admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
 		requestFunc := func() (runtime.Object, error) {
@@ -150,6 +155,8 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 				options,
 			)
 		}
+		// Dedup owner references before updating managed fields
+		dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
 		result, err := finishRequest(timeout, func() (runtime.Object, error) {
 			if scope.FieldManager != nil {
 				liveObj, err := scope.Creater.New(scope.Kind)
@@ -163,6 +170,8 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
 					return nil, err
 				}
 			}
+			// Dedup owner references again after mutating admission happens
+			dedupOwnerReferencesAndAddWarning(obj, req.Context(), true)
 			result, err := requestFunc()
 			// If the object wasn't committed to storage because it's serialized size was too large,
 			// it is safe to remove managedFields (which can be large) and try again.

vendor/k8s.io/apiserver/pkg/endpoints/handlers/delete.go

@@ -141,7 +141,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
 		// that will break existing clients.
 		// Other cases where resource is not instantly deleted are: namespace deletion
 		// and pod graceful deletion.
-		if !wasDeleted && options.OrphanDependents != nil && *options.OrphanDependents == false {
+		if !wasDeleted && options.OrphanDependents != nil && !*options.OrphanDependents {
 			status = http.StatusAccepted
 		}
 		// if the rest.Deleter returns a nil object, fill out a status. Callers may return a valid

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/capmanagers.go

@@ -67,7 +67,7 @@ func (f *capManagersManager) capUpdateManagers(managed Managed) (newManaged Mana
 	// Gather all entries from updates
 	updaters := []string{}
 	for manager, fields := range managed.Fields() {
-		if fields.Applied() == false {
+		if !fields.Applied() {
 			updaters = append(updaters, manager)
 		}
 	}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/fieldmanager.go

@@ -27,7 +27,6 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"k8s.io/klog/v2"
-	openapiproto "k8s.io/kube-openapi/pkg/util/proto"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
@@ -67,50 +66,39 @@ type Manager interface {
 // FieldManager updates the managed fields and merge applied
 // configurations.
 type FieldManager struct {
-	fieldManager Manager
+	fieldManager                         Manager
+	ignoreManagedFieldsFromRequestObject bool
 }
 
 // NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
 // on update and apply requests.
-func NewFieldManager(f Manager) *FieldManager {
-	return &FieldManager{f}
+func NewFieldManager(f Manager, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
+	return &FieldManager{fieldManager: f, ignoreManagedFieldsFromRequestObject: ignoreManagedFieldsFromRequestObject}
 }
 
 // NewDefaultFieldManager creates a new FieldManager that merges apply requests
 // and update managed fields for other types of requests.
-func NewDefaultFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion) (*FieldManager, error) {
-	typeConverter, err := internal.NewTypeConverter(models, false)
-	if err != nil {
-		return nil, err
-	}
-
+func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool) (*FieldManager, error) {
 	f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
+	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
 }
 
 // NewDefaultCRDFieldManager creates a new FieldManager specifically for
 // CRDs. This allows for the possibility of fields which are not defined
 // in models, as well as having no models defined at all.
-func NewDefaultCRDFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, preserveUnknownFields bool) (_ *FieldManager, err error) {
-	var typeConverter internal.TypeConverter = internal.DeducedTypeConverter{}
-	if models != nil {
-		typeConverter, err = internal.NewTypeConverter(models, preserveUnknownFields)
-		if err != nil {
-			return nil, err
-		}
-	}
-	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, preserveUnknownFields)
+func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool) (_ *FieldManager, err error) {
+	f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
 	if err != nil {
 		return nil, fmt.Errorf("failed to create field manager: %v", err)
 	}
-	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
+	return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
 }
 
 // newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
-func newDefaultFieldManager(f Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind) *FieldManager {
+func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
 	f = NewStripMetaManager(f)
 	f = NewManagedFieldsUpdater(f)
 	f = NewBuildManagerInfoManager(f, kind.GroupVersion())
@@ -119,36 +107,59 @@ func newDefaultFieldManager(f Manager, typeConverter internal.TypeConverter, obj
 	f = NewLastAppliedManager(f, typeConverter, objectConverter, kind.GroupVersion())
 	f = NewLastAppliedUpdater(f)
 
-	return NewFieldManager(f)
+	return NewFieldManager(f, ignoreManagedFieldsFromRequestObject)
+}
+
+func decodeLiveManagedFields(liveObj runtime.Object) (Managed, error) {
+	liveAccessor, err := meta.Accessor(liveObj)
+	if err != nil {
+		return nil, err
+	}
+	managed, err := internal.DecodeObjectManagedFields(liveAccessor.GetManagedFields())
+	if err != nil {
+		return internal.NewEmptyManaged(), nil
+	}
+	return managed, nil
+}
+
+func decodeManagedFields(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
+	// We take the managedFields of the live object in case the request tries to
+	// manually set managedFields via a subresource.
+	if ignoreManagedFieldsFromRequestObject {
+		return decodeLiveManagedFields(liveObj)
+	}
+
+	// If the object doesn't have metadata, we should just return without trying to
+	// set the managedFields at all, so creates/updates/patches will work normally.
+	newAccessor, err := meta.Accessor(newObj)
+	if err != nil {
+		return nil, err
+	}
+
+	if isResetManagedFields(newAccessor.GetManagedFields()) {
+		return internal.NewEmptyManaged(), nil
+	}
+
+	managed, err := internal.DecodeObjectManagedFields(newAccessor.GetManagedFields())
+	// If the managed field is empty or we failed to decode it,
+	// let's try the live object. This is to prevent clients who
+	// don't understand managedFields from deleting it accidentally.
+	if err != nil || len(managed.Fields()) == 0 {
+		return decodeLiveManagedFields(liveObj)
+	}
+
+	return managed, nil
 }
 
 // Update is used when the object has already been merged (non-apply
 // use-case), and simply updates the managed fields in the output
 // object.
 func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
-	// If the object doesn't have metadata, we should just return without trying to
-	// set the managedFields at all, so creates/updates/patches will work normally.
-	newAccessor, err := meta.Accessor(newObj)
-	if err != nil {
-		return newObj, nil
-	}
-
 	// First try to decode the managed fields provided in the update,
 	// This is necessary to allow directly updating managed fields.
-	var managed Managed
-	if isResetManagedFields(newAccessor.GetManagedFields()) {
-		managed = internal.NewEmptyManaged()
-	} else if managed, err = internal.DecodeObjectManagedFields(newAccessor.GetManagedFields()); err != nil || len(managed.Fields()) == 0 {
-		liveAccessor, err := meta.Accessor(liveObj)
-		if err != nil {
-			return newObj, nil
-		}
-		// If the managed field is empty or we failed to decode it,
-		// let's try the live object. This is to prevent clients who
-		// don't understand managedFields from deleting it accidentally.
-		if managed, err = internal.DecodeObjectManagedFields(liveAccessor.GetManagedFields()); err != nil {
-			managed = internal.NewEmptyManaged()
-		}
+	managed, err := decodeManagedFields(liveObj, newObj, f.ignoreManagedFieldsFromRequestObject)
+	if err != nil {
+		return newObj, nil
 	}
 
 	internal.RemoveObjectManagedFields(liveObj)

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal/gvkparser.go

@@ -30,12 +30,15 @@ import (
 // definition's "extensions" map.
 const groupVersionKindExtensionKey = "x-kubernetes-group-version-kind"
 
-type gvkParser struct {
+// GvkParser contains a Parser that allows introspecting the schema.
+type GvkParser struct {
 	gvks   map[schema.GroupVersionKind]string
 	parser typed.Parser
 }
 
-func (p *gvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
+// Type returns a helper which can produce objects of the given type. Any
+// errors are deferred until a further function is called.
+func (p *GvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
 	typeName, ok := p.gvks[gvk]
 	if !ok {
 		return nil
@@ -44,12 +47,15 @@ func (p *gvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
 	return &t
 }
 
-func newGVKParser(models proto.Models, preserveUnknownFields bool) (*gvkParser, error) {
+// NewGVKParser builds a GVKParser from a proto.Models. This
+// will automatically find the proper version of the object, and the
+// corresponding schema information.
+func NewGVKParser(models proto.Models, preserveUnknownFields bool) (*GvkParser, error) {
 	typeSchema, err := schemaconv.ToSchemaWithPreserveUnknownFields(models, preserveUnknownFields)
 	if err != nil {
 		return nil, fmt.Errorf("failed to convert models to schema: %v", err)
 	}
-	parser := gvkParser{
+	parser := GvkParser{
 		gvks: map[schema.GroupVersionKind]string{},
 	}
 	parser.parser = typed.Parser{Schema: *typeSchema}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/lastappliedmanager.go

@@ -25,14 +25,13 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
-	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
 	"sigs.k8s.io/structured-merge-diff/v4/merge"
 )
 
 type lastAppliedManager struct {
 	fieldManager    Manager
-	typeConverter   internal.TypeConverter
+	typeConverter   TypeConverter
 	objectConverter runtime.ObjectConvertor
 	groupVersion    schema.GroupVersion
 }
@@ -41,7 +40,7 @@ var _ Manager = &lastAppliedManager{}
 
 // NewLastAppliedManager converts the client-side apply annotation to
 // server-side apply managed fields
-func NewLastAppliedManager(fieldManager Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, groupVersion schema.GroupVersion) Manager {
+func NewLastAppliedManager(fieldManager Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, groupVersion schema.GroupVersion) Manager {
 	return &lastAppliedManager{
 		fieldManager:    fieldManager,
 		typeConverter:   typeConverter,

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/managedfieldsupdater.go

@@ -44,6 +44,7 @@ func NewManagedFieldsUpdater(fieldManager Manager) Manager {
 // Update implements Manager.
 func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
 	self := "current-operation"
+	formerSet := managed.Fields()[manager]
 	object, managed, err := f.fieldManager.Update(liveObj, newObj, managed, self)
 	if err != nil {
 		return object, managed, err
@@ -54,12 +55,15 @@ func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Ma
 	if vs, ok := managed.Fields()[self]; ok {
 		delete(managed.Fields(), self)
 
-		managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
 		if previous, ok := managed.Fields()[manager]; ok {
 			managed.Fields()[manager] = fieldpath.NewVersionedSet(vs.Set().Union(previous.Set()), vs.APIVersion(), vs.Applied())
 		} else {
 			managed.Fields()[manager] = vs
 		}
+		// Update the time only if the manager's fieldSet has changed.
+		if formerSet == nil || !managed.Fields()[manager].Set().Equals(formerSet.Set()) {
+			managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
+		}
 	}
 
 	return object, managed, nil

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/structuredmerge.go

@@ -29,7 +29,7 @@ import (
 )
 
 type structuredMergeManager struct {
-	typeConverter   internal.TypeConverter
+	typeConverter   TypeConverter
 	objectConverter runtime.ObjectConvertor
 	objectDefaulter runtime.ObjectDefaulter
 	groupVersion    schema.GroupVersion
@@ -41,7 +41,7 @@ var _ Manager = &structuredMergeManager{}
 
 // NewStructuredMergeManager creates a new Manager that merges apply requests
 // and update managed fields for other types of requests.
-func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
+func NewStructuredMergeManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
 	return &structuredMergeManager{
 		typeConverter:   typeConverter,
 		objectConverter: objectConverter,
@@ -49,7 +49,7 @@ func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConve
 		groupVersion:    gv,
 		hubVersion:      hub,
 		updater: merge.Updater{
-			Converter: internal.NewVersionConverter(typeConverter, objectConverter, hub), // This is the converter provided to SMD from k8s
+			Converter: newVersionConverter(typeConverter, objectConverter, hub), // This is the converter provided to SMD from k8s
 		},
 	}, nil
 }
@@ -57,7 +57,7 @@ func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConve
 // NewCRDStructuredMergeManager creates a new Manager specifically for
 // CRDs. This allows for the possibility of fields which are not defined
 // in models, as well as having no models defined at all.
-func NewCRDStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion, preserveUnknownFields bool) (_ Manager, err error) {
+func NewCRDStructuredMergeManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (_ Manager, err error) {
 	return &structuredMergeManager{
 		typeConverter:   typeConverter,
 		objectConverter: objectConverter,
@@ -65,7 +65,7 @@ func NewCRDStructuredMergeManager(typeConverter internal.TypeConverter, objectCo
 		groupVersion:    gv,
 		hubVersion:      hub,
 		updater: merge.Updater{
-			Converter: internal.NewCRDVersionConverter(typeConverter, objectConverter, hub),
+			Converter: newCRDVersionConverter(typeConverter, objectConverter, hub),
 		},
 	}, nil
 }

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/typeconverter.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package internal
+package fieldmanager
 
 import (
 	"fmt"
@@ -22,6 +22,7 @@ import (
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
 	"k8s.io/kube-openapi/pkg/util/proto"
 	"sigs.k8s.io/structured-merge-diff/v4/typed"
 	"sigs.k8s.io/structured-merge-diff/v4/value"
@@ -64,7 +65,7 @@ func (DeducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Obje
 }
 
 type typeConverter struct {
-	parser *gvkParser
+	parser *internal.GvkParser
 }
 
 var _ TypeConverter = &typeConverter{}
@@ -73,7 +74,7 @@ var _ TypeConverter = &typeConverter{}
 // will automatically find the proper version of the object, and the
 // corresponding schema information.
 func NewTypeConverter(models proto.Models, preserveUnknownFields bool) (TypeConverter, error) {
-	parser, err := newGVKParser(models, preserveUnknownFields)
+	parser, err := internal.NewGVKParser(models, preserveUnknownFields)
 	if err != nil {
 		return nil, err
 	}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/versionconverter.go

@@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package internal
+package fieldmanager
 
 import (
 	"k8s.io/apimachinery/pkg/runtime"
@@ -35,7 +35,7 @@ type versionConverter struct {
 var _ merge.Converter = &versionConverter{}
 
 // NewVersionConverter builds a VersionConverter from a TypeConverter and an ObjectConvertor.
-func NewVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
+func newVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
 	return &versionConverter{
 		typeConverter:   t,
 		objectConvertor: o,
@@ -49,7 +49,7 @@ func NewVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.Gr
 }
 
 // NewCRDVersionConverter builds a VersionConverter for CRDs from a TypeConverter and an ObjectConvertor.
-func NewCRDVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
+func newCRDVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
 	return &versionConverter{
 		typeConverter:   t,
 		objectConvertor: o,

vendor/k8s.io/apiserver/pkg/endpoints/handlers/patch.go

@@ -38,6 +38,7 @@ import (
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/strategicpatch"
 	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apimachinery/pkg/util/yaml"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/audit"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
@@ -49,7 +50,6 @@ import (
 	"k8s.io/apiserver/pkg/util/dryrun"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utiltrace "k8s.io/utils/trace"
-	"sigs.k8s.io/yaml"
 )
 
 const (
@@ -576,9 +576,14 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
 	default:
 		return nil, false, fmt.Errorf("%v: unimplemented patch type", p.patchType)
 	}
+	dedupOwnerReferencesTransformer := func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
+		// Dedup owner references after mutating admission happens
+		dedupOwnerReferencesAndAddWarning(obj, ctx, true)
+		return obj, nil
+	}
 
 	wasCreated := false
-	p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission)
+	p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission, dedupOwnerReferencesTransformer)
 	requestFunc := func() (runtime.Object, error) {
 		// Pass in UpdateOptions to override UpdateStrategy.AllowUpdateOnCreate
 		options := patchToUpdateOptions(p.options)
@@ -592,11 +597,15 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
 		// it is safe to remove managedFields (which can be large) and try again.
 		if isTooLargeError(err) && p.patchType != types.ApplyPatchType {
 			if _, accessorErr := meta.Accessor(p.restPatcher.New()); accessorErr == nil {
-				p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission, func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
-					accessor, _ := meta.Accessor(obj)
-					accessor.SetManagedFields(nil)
-					return obj, nil
-				})
+				p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil,
+					p.applyPatch,
+					p.applyAdmission,
+					dedupOwnerReferencesTransformer,
+					func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
+						accessor, _ := meta.Accessor(obj)
+						accessor.SetManagedFields(nil)
+						return obj, nil
+					})
 				result, err = requestFunc()
 			}
 		}

vendor/k8s.io/apiserver/pkg/endpoints/handlers/responsewriters/writers.go

@@ -96,9 +96,11 @@ func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.Response
 	err := encoder.Encode(object, w)
 	if err == nil {
 		err = w.Close()
-		if err == nil {
-			return
+		if err != nil {
+			// we cannot write an error to the writer anymore as the Encode call was successful.
+			utilruntime.HandleError(fmt.Errorf("apiserver was unable to close cleanly the response writer: %v", err))
 		}
+		return
 	}
 
 	// make a best effort to write the object if a failure is detected

vendor/k8s.io/apiserver/pkg/endpoints/handlers/rest.go

@@ -31,12 +31,14 @@ import (
 	grpccodes "google.golang.org/grpc/codes"
 	grpcstatus "google.golang.org/grpc/status"
 
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/types"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
@@ -46,9 +48,21 @@ import (
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/rest"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
+	"k8s.io/apiserver/pkg/warning"
 	"k8s.io/klog/v2"
 )
 
+const (
+	// DuplicateOwnerReferencesWarningFormat is the warning that a client receives when a create/update request contains
+	// duplicate owner reference entries.
+	DuplicateOwnerReferencesWarningFormat = ".metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: %v"
+	// DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat indicates the duplication was observed
+	// after mutating admission.
+	// NOTE: For CREATE and UPDATE requests the API server dedups both before and after mutating admission.
+	// For PATCH request the API server only dedups after mutating admission.
+	DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat = ".metadata.ownerReferences contains duplicate entries after mutating admission happens; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: %v"
+)
+
 // RequestScope encapsulates common fields across all RESTful handler methods.
 type RequestScope struct {
 	Namer ScopeNamer
@@ -323,11 +337,79 @@ func checkName(obj runtime.Object, name, namespace string, namer ScopeNamer) err
 	return nil
 }
 
+// dedupOwnerReferences dedups owner references over the entire entry.
+// NOTE: We don't know enough about the existing cases of owner references
+// sharing the same UID but different fields. Nor do we know what might break.
+// In the future we may just dedup/reject owner references with the same UID.
+func dedupOwnerReferences(refs []metav1.OwnerReference) ([]metav1.OwnerReference, []string) {
+	var result []metav1.OwnerReference
+	var duplicates []string
+	seen := make(map[types.UID]struct{})
+	for _, ref := range refs {
+		_, ok := seen[ref.UID]
+		// Short-circuit if we haven't seen the UID before. Otherwise
+		// check the entire list we have so far.
+		if !ok || !hasOwnerReference(result, ref) {
+			seen[ref.UID] = struct{}{}
+			result = append(result, ref)
+		} else {
+			duplicates = append(duplicates, string(ref.UID))
+		}
+	}
+	return result, duplicates
+}
+
+// hasOwnerReference returns true if refs has an item equal to ref. The function
+// focuses on semantic equality instead of memory equality, to catch duplicates
+// with different pointer addresses. The function uses apiequality.Semantic
+// instead of implementing its own comparison, to tolerate API changes to
+// metav1.OwnerReference.
+// NOTE: This is expensive, but we accept it because we've made sure it only
+// happens to owner references containing duplicate UIDs, plus typically the
+// number of items in the list should be small.
+func hasOwnerReference(refs []metav1.OwnerReference, ref metav1.OwnerReference) bool {
+	for _, r := range refs {
+		if apiequality.Semantic.DeepEqual(r, ref) {
+			return true
+		}
+	}
+	return false
+}
+
+// dedupOwnerReferencesAndAddWarning dedups owner references in the object metadata.
+// If duplicates are found, the function records a warning to the provided context.
+func dedupOwnerReferencesAndAddWarning(obj runtime.Object, requestContext context.Context, afterMutatingAdmission bool) {
+	accessor, err := meta.Accessor(obj)
+	if err != nil {
+		// The object doesn't have metadata. Nothing we need to do here.
+		return
+	}
+	refs := accessor.GetOwnerReferences()
+	deduped, duplicates := dedupOwnerReferences(refs)
+	if len(duplicates) > 0 {
+		// NOTE: For CREATE and UPDATE requests the API server dedups both before and after mutating admission.
+		// For PATCH request the API server only dedups after mutating admission.
+		format := DuplicateOwnerReferencesWarningFormat
+		if afterMutatingAdmission {
+			format = DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat
+		}
+		warning.AddWarning(requestContext, "", fmt.Sprintf(format,
+			strings.Join(duplicates, ", ")))
+		accessor.SetOwnerReferences(deduped)
+	}
+}
+
 // setObjectSelfLink sets the self link of an object as needed.
 // TODO: remove the need for the namer LinkSetters by requiring objects implement either Object or List
 //   interfaces
 func setObjectSelfLink(ctx context.Context, obj runtime.Object, req *http.Request, namer ScopeNamer) error {
 	if utilfeature.DefaultFeatureGate.Enabled(features.RemoveSelfLink) {
+		// Ensure that for empty lists we don't return <nil> items.
+		if meta.IsListType(obj) && meta.LenList(obj) == 0 {
+			if err := meta.SetList(obj, []runtime.Object{}); err != nil {
+				return err
+			}
+		}
 		return nil
 	}
 

vendor/k8s.io/apiserver/pkg/endpoints/handlers/update.go

@@ -153,6 +153,11 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 				}
 				return newObj, nil
 			})
+			transformers = append(transformers, func(ctx context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) {
+				// Dedup owner references again after mutating admission happens
+				dedupOwnerReferencesAndAddWarning(newObj, req.Context(), true)
+				return newObj, nil
+			})
 		}
 
 		createAuthorizerAttributes := authorizer.AttributesRecord{
@@ -188,6 +193,8 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
 			wasCreated = created
 			return obj, err
 		}
+		// Dedup owner references before updating managed fields
+		dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
 		result, err := finishRequest(timeout, func() (runtime.Object, error) {
 			result, err := requestFunc()
 			// If the object wasn't committed to storage because it's serialized size was too large,

vendor/k8s.io/apiserver/pkg/endpoints/installer.go

@@ -43,6 +43,7 @@ import (
 	utilwarning "k8s.io/apiserver/pkg/endpoints/warning"
 	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/rest"
+	"k8s.io/apiserver/pkg/storageversion"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	versioninfo "k8s.io/component-base/version"
 )
@@ -94,8 +95,9 @@ var toDiscoveryKubeVerb = map[string]string{
 }
 
 // Install handlers for API resources.
-func (a *APIInstaller) Install() ([]metav1.APIResource, *restful.WebService, []error) {
+func (a *APIInstaller) Install() ([]metav1.APIResource, []*storageversion.ResourceInfo, *restful.WebService, []error) {
 	var apiResources []metav1.APIResource
+	var resourceInfos []*storageversion.ResourceInfo
 	var errors []error
 	ws := a.newWebService()
 
@@ -108,15 +110,18 @@ func (a *APIInstaller) Install() ([]metav1.APIResource, *restful.WebService, []e
 	}
 	sort.Strings(paths)
 	for _, path := range paths {
-		apiResource, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
+		apiResource, resourceInfo, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
 		if err != nil {
 			errors = append(errors, fmt.Errorf("error in registering resource: %s, %v", path, err))
 		}
 		if apiResource != nil {
 			apiResources = append(apiResources, *apiResource)
 		}
+		if resourceInfo != nil {
+			resourceInfos = append(resourceInfos, resourceInfo)
+		}
 	}
-	return apiResources, ws, errors
+	return apiResources, resourceInfos, ws, errors
 }
 
 // newWebService creates a new restful webservice with the api installer's prefix and version.
@@ -182,7 +187,7 @@ func GetResourceKind(groupVersion schema.GroupVersion, storage rest.Storage, typ
 	return fqKindToRegister, nil
 }
 
-func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {
+func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, *storageversion.ResourceInfo, error) {
 	admit := a.group.Admit
 
 	optionsExternalVersion := a.group.GroupVersion
@@ -192,19 +197,19 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 	resource, subresource, err := splitSubresource(path)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	group, version := a.group.GroupVersion.Group, a.group.GroupVersion.Version
 
 	fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	versionedPtr, err := a.group.Creater.New(fqKindToRegister)
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	defaultVersionedObject := indirectArbitraryPointer(versionedPtr)
 	kind := fqKindToRegister.Kind
@@ -215,18 +220,18 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	if isSubresource {
 		parentStorage, ok := a.group.Storage[resource]
 		if !ok {
-			return nil, fmt.Errorf("missing parent storage: %q", resource)
+			return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
 		}
 		scoper, ok := parentStorage.(rest.Scoper)
 		if !ok {
-			return nil, fmt.Errorf("%q must implement scoper", resource)
+			return nil, nil, fmt.Errorf("%q must implement scoper", resource)
 		}
 		namespaceScoped = scoper.NamespaceScoped()
 
 	} else {
 		scoper, ok := storage.(rest.Scoper)
 		if !ok {
-			return nil, fmt.Errorf("%q must implement scoper", resource)
+			return nil, nil, fmt.Errorf("%q must implement scoper", resource)
 		}
 		namespaceScoped = scoper.NamespaceScoped()
 	}
@@ -255,7 +260,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 	versionedExportOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ExportOptions"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	if isNamedCreater {
@@ -267,30 +272,30 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		list := lister.NewList()
 		listGVKs, _, err := a.group.Typer.ObjectKinds(list)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		versionedListPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listGVKs[0].Kind))
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		versionedList = indirectArbitraryPointer(versionedListPtr)
 	}
 
 	versionedListOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ListOptions"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	versionedCreateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("CreateOptions"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	versionedPatchOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("PatchOptions"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	versionedUpdateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("UpdateOptions"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 
 	var versionedDeleteOptions runtime.Object
@@ -299,7 +304,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	if isGracefulDeleter {
 		versionedDeleteOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind("DeleteOptions"))
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		versionedDeleterObject = indirectArbitraryPointer(versionedDeleteOptions)
 
@@ -310,7 +315,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 	versionedStatusPtr, err := a.group.Creater.New(optionsExternalVersion.WithKind("Status"))
 	if err != nil {
-		return nil, err
+		return nil, nil, err
 	}
 	versionedStatus := indirectArbitraryPointer(versionedStatusPtr)
 	var (
@@ -323,14 +328,14 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		getOptions, getSubpath, _ = getterWithOptions.NewGetOptions()
 		getOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(getOptions)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		getOptionsInternalKind = getOptionsInternalKinds[0]
 		versionedGetOptions, err = a.group.Creater.New(a.group.GroupVersion.WithKind(getOptionsInternalKind.Kind))
 		if err != nil {
 			versionedGetOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(getOptionsInternalKind.Kind))
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 		}
 		isGetter = true
@@ -340,7 +345,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	if isWatcher {
 		versionedWatchEventPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind("WatchEvent"))
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		versionedWatchEvent = indirectArbitraryPointer(versionedWatchEventPtr)
 	}
@@ -356,7 +361,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		if connectOptions != nil {
 			connectOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(connectOptions)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 
 			connectOptionsInternalKind = connectOptionsInternalKinds[0]
@@ -364,7 +369,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			if err != nil {
 				versionedConnectOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(connectOptionsInternalKind.Kind))
 				if err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 		}
@@ -388,7 +393,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	tableProvider, isTableProvider := storage.(rest.TableConvertor)
 	if isLister && !isTableProvider {
 		// All listers must implement TableProvider
-		return nil, fmt.Errorf("%q must implement TableConvertor", resource)
+		return nil, nil, fmt.Errorf("%q must implement TableConvertor", resource)
 	}
 
 	var apiResource metav1.APIResource
@@ -398,7 +403,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		versioner := storageVersionProvider.StorageVersion()
 		gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer)
 		if err != nil {
-			return nil, err
+			return nil, nil, err
 		}
 		apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind)
 	}
@@ -506,6 +511,30 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		}
 	}
 
+	var resourceInfo *storageversion.ResourceInfo
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
+		utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) &&
+		isStorageVersionProvider &&
+		storageVersionProvider.StorageVersion() != nil {
+
+		versioner := storageVersionProvider.StorageVersion()
+		encodingGVK, err := getStorageVersionKind(versioner, storage, a.group.Typer)
+		if err != nil {
+			return nil, nil, err
+		}
+		resourceInfo = &storageversion.ResourceInfo{
+			GroupResource: schema.GroupResource{
+				Group:    a.group.GroupVersion.Group,
+				Resource: apiResource.Name,
+			},
+			EncodingVersion: encodingGVK.GroupVersion().String(),
+			// We record EquivalentResourceMapper first instead of calculate
+			// DecodableVersions immediately because API installation must
+			// be completed first for us to know equivalent APIs
+			EquivalentResourceMapper: a.group.EquivalentResourceRegistry,
+		}
+	}
+
 	// Create Routes for the actions.
 	// TODO: Add status documentation using Returns()
 	// Errors (see api/errors/errors.go as well as go-restful router):
@@ -525,7 +554,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 
 	for _, s := range a.group.Serializer.SupportedMediaTypes() {
 		if len(s.MediaTypeSubType) == 0 || len(s.MediaTypeType) == 0 {
-			return nil, fmt.Errorf("all serializers in the group Serializer must have MediaTypeType and MediaTypeSubType set: %s", s.MediaType)
+			return nil, nil, fmt.Errorf("all serializers in the group Serializer must have MediaTypeType and MediaTypeSubType set: %s", s.MediaType)
 		}
 	}
 	mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
@@ -564,15 +593,16 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	}
 	if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
 		reqScope.FieldManager, err = fieldmanager.NewDefaultFieldManager(
-			a.group.OpenAPIModels,
+			a.group.TypeConverter,
 			a.group.UnsafeConvertor,
 			a.group.Defaulter,
 			a.group.Creater,
 			fqKindToRegister,
 			reqScope.HubGroupVersion,
+			isSubresource,
 		)
 		if err != nil {
-			return nil, fmt.Errorf("failed to create field manager: %v", err)
+			return nil, nil, fmt.Errorf("failed to create field manager: %v", err)
 		}
 	}
 	for _, action := range actions {
@@ -593,6 +623,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 			requestScope = "resource"
 			operationSuffix = operationSuffix + "WithPath"
 		}
+		if strings.Index(action.Path, "/{name}") != -1 || action.Verb == "POST" {
+			requestScope = "resource"
+		}
 		if action.AllNamespaces {
 			requestScope = "cluster"
 			operationSuffix = operationSuffix + "ForAllNamespaces"
@@ -604,7 +637,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				kubeVerbs[kubeVerb] = struct{}{}
 			}
 		} else {
-			return nil, fmt.Errorf("unknown action verb for discovery: %s", action.Verb)
+			return nil, nil, fmt.Errorf("unknown action verb for discovery: %s", action.Verb)
 		}
 
 		routes := []*restful.RouteBuilder{}
@@ -613,12 +646,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 		if isSubresource {
 			parentStorage, ok := a.group.Storage[resource]
 			if !ok {
-				return nil, fmt.Errorf("missing parent storage: %q", resource)
+				return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
 			}
 
 			fqParentKind, err := GetResourceKind(a.group.GroupVersion, parentStorage, a.group.Typer)
 			if err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			kind = fqParentKind.Kind
 		}
@@ -677,12 +710,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Writes(producedObject)
 			if isGetterWithOptions {
 				if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 			if isExporter {
 				if err := AddObjectParams(ws, route, versionedExportOptions); err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 			addParams(route, action.Params)
@@ -704,7 +737,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusOK, "OK", versionedList).
 				Writes(versionedList)
 			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			switch {
 			case isLister && isWatcher:
@@ -743,7 +776,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Reads(defaultVersionedObject).
 				Writes(producedObject)
 			if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -774,7 +807,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Reads(metav1.Patch{}).
 				Writes(producedObject)
 			if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -807,7 +840,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Reads(defaultVersionedObject).
 				Writes(producedObject)
 			if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -837,7 +870,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				route.Reads(versionedDeleterObject)
 				route.ParameterNamed("body").Required(false)
 				if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 			addParams(route, action.Params)
@@ -862,11 +895,11 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				route.Reads(versionedDeleterObject)
 				route.ParameterNamed("body").Required(false)
 				if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
-					return nil, err
+					return nil, nil, err
 				}
 			}
 			if err := AddObjectParams(ws, route, versionedListOptions, "watch", "allowWatchBookmarks"); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -889,7 +922,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusOK, "OK", versionedWatchEvent).
 				Writes(versionedWatchEvent)
 			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -912,7 +945,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				Returns(http.StatusOK, "OK", versionedWatchEvent).
 				Writes(versionedWatchEvent)
 			if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
-				return nil, err
+				return nil, nil, err
 			}
 			addParams(route, action.Params)
 			routes = append(routes, route)
@@ -939,7 +972,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 					Writes(connectProducedObject)
 				if versionedConnectOptions != nil {
 					if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil {
-						return nil, err
+						return nil, nil, err
 					}
 				}
 				addParams(route, action.Params)
@@ -953,7 +986,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 				}
 			}
 		default:
-			return nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)
+			return nil, nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)
 		}
 		for _, route := range routes {
 			route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{
@@ -989,7 +1022,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
 	// Record the existence of the GVR and the corresponding GVK
 	a.group.EquivalentResourceRegistry.RegisterKindFor(reqScope.Resource, reqScope.Subresource, fqKindToRegister)
 
-	return &apiResource, nil
+	return &apiResource, resourceInfo, nil
 }
 
 // indirectArbitraryPointer returns *ptrToObject for an arbitrary pointer

vendor/k8s.io/apiserver/pkg/endpoints/metrics/metrics.go

@@ -32,6 +32,7 @@ import (
 	"k8s.io/apimachinery/pkg/types"
 	utilsets "k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apiserver/pkg/audit"
+	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/features"
 	utilfeature "k8s.io/apiserver/pkg/util/feature"
@@ -185,6 +186,36 @@ var (
 		},
 		[]string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"},
 	)
+
+	apiSelfRequestCounter = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "apiserver_selfrequest_total",
+			Help:           "Counter of apiserver self-requests broken out for each verb, API resource and subresource.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "resource", "subresource"},
+	)
+
+	requestFilterDuration = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Name:           "apiserver_request_filter_duration_seconds",
+			Help:           "Request filter latency distribution in seconds, for each filter type",
+			Buckets:        []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"filter"},
+	)
+
+	// requestAbortsTotal is a number of aborted requests with http.ErrAbortHandler
+	requestAbortsTotal = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Name:           "apiserver_request_aborts_total",
+			Help:           "Number of requests which apiserver aborted possibly due to a timeout, for each group, version, verb, resource, subresource and scope",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"verb", "group", "version", "resource", "subresource", "scope"},
+	)
+
 	kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`)
 
 	metrics = []resettableCollector{
@@ -201,6 +232,9 @@ var (
 		currentInflightRequests,
 		currentInqueueRequests,
 		requestTerminationsTotal,
+		apiSelfRequestCounter,
+		requestFilterDuration,
+		requestAbortsTotal,
 	}
 
 	// these are the known (e.g. whitelisted/known) content types which we will report for
@@ -290,6 +324,26 @@ func UpdateInflightRequestMetrics(phase string, nonmutating, mutating int) {
 	}
 }
 
+func RecordFilterLatency(name string, elapsed time.Duration) {
+	requestFilterDuration.WithLabelValues(name).Observe(elapsed.Seconds())
+}
+
+// RecordRequestAbort records that the request was aborted possibly due to a timeout.
+func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
+	if requestInfo == nil {
+		requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
+	}
+
+	scope := CleanScope(requestInfo)
+	reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
+	resource := requestInfo.Resource
+	subresource := requestInfo.Subresource
+	group := requestInfo.APIGroup
+	version := requestInfo.APIVersion
+
+	requestAbortsTotal.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Inc()
+}
+
 // RecordRequestTermination records that the request was terminated early as part of a resource
 // preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling,
 // proxyHandler errors). RecordRequestTermination should only be called zero or one times
@@ -299,20 +353,17 @@ func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInf
 		requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
 	}
 	scope := CleanScope(requestInfo)
-	// We don't use verb from <requestInfo>, as for the healthy path
-	// MonitorRequest is called from InstrumentRouteFunc which is registered
-	// in installer.go with predefined list of verbs (different than those
-	// translated to RequestInfo).
+
+	// We don't use verb from <requestInfo>, as this may be propagated from
+	// InstrumentRouteFunc which is registered in installer.go with predefined
+	// list of verbs (different than those translated to RequestInfo).
 	// However, we need to tweak it e.g. to differentiate GET from LIST.
-	verb := canonicalVerb(strings.ToUpper(req.Method), scope)
-	// set verbs to a bounded set of known and expected verbs
-	if !validRequestMethods.Has(verb) {
-		verb = OtherRequestMethod
-	}
+	reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
+
 	if requestInfo.IsResourceRequest {
-		requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
+		requestTerminationsTotal.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
 	} else {
-		requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
+		requestTerminationsTotal.WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
 	}
 }
 
@@ -324,12 +375,13 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
 	}
 	var g compbasemetrics.GaugeMetric
 	scope := CleanScope(requestInfo)
-	// We don't use verb from <requestInfo>, as for the healthy path
-	// MonitorRequest is called from InstrumentRouteFunc which is registered
-	// in installer.go with predefined list of verbs (different than those
-	// translated to RequestInfo).
+
+	// We don't use verb from <requestInfo>, as this may be propagated from
+	// InstrumentRouteFunc which is registered in installer.go with predefined
+	// list of verbs (different than those translated to RequestInfo).
 	// However, we need to tweak it e.g. to differentiate GET from LIST.
 	reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
+
 	if requestInfo.IsResourceRequest {
 		g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component)
 	} else {
@@ -343,11 +395,21 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
 // MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
 // a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
 func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, contentType string, httpCode, respSize int, elapsed time.Duration) {
-	reportedVerb := cleanVerb(verb, req)
+	// We don't use verb from <requestInfo>, as this may be propagated from
+	// InstrumentRouteFunc which is registered in installer.go with predefined
+	// list of verbs (different than those translated to RequestInfo).
+	// However, we need to tweak it e.g. to differentiate GET from LIST.
+	reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
+
 	dryRun := cleanDryRun(req.URL)
 	elapsedSeconds := elapsed.Seconds()
 	cleanContentType := cleanContentType(contentType)
 	requestCounter.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, cleanContentType, codeToString(httpCode)).Inc()
+	// MonitorRequest happens after authentication, so we can trust the username given by the request
+	info, ok := request.UserFrom(req.Context())
+	if ok && info.GetName() == user.APIServerUser {
+		apiSelfRequestCounter.WithLabelValues(reportedVerb, resource, subresource).Inc()
+	}
 	if deprecated {
 		deprecatedRequestGauge.WithLabelValues(group, version, resource, subresource, removedRelease).Set(1)
 		audit.AddAuditAnnotation(req.Context(), deprecatedAnnotationKey, "true")
@@ -365,8 +427,11 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
 // InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps
 // the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information.
 func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, routeFunc restful.RouteFunction) restful.RouteFunction {
-	return restful.RouteFunction(func(request *restful.Request, response *restful.Response) {
-		now := time.Now()
+	return restful.RouteFunction(func(req *restful.Request, response *restful.Response) {
+		requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Request.Context())
+		if !ok {
+			requestReceivedTimestamp = time.Now()
+		}
 
 		delegate := &ResponseWriterDelegator{ResponseWriter: response.ResponseWriter}
 
@@ -381,16 +446,19 @@ func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, com
 		}
 		response.ResponseWriter = rw
 
-		routeFunc(request, response)
+		routeFunc(req, response)
 
-		MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
+		MonitorRequest(req.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
 	})
 }
 
 // InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information.
 func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, handler http.HandlerFunc) http.HandlerFunc {
 	return func(w http.ResponseWriter, req *http.Request) {
-		now := time.Now()
+		requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Context())
+		if !ok {
+			requestReceivedTimestamp = time.Now()
+		}
 
 		delegate := &ResponseWriterDelegator{ResponseWriter: w}
 
@@ -405,7 +473,7 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
 
 		handler(w, req)
 
-		MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
+		MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
 	}
 }
 
@@ -440,7 +508,7 @@ func CleanScope(requestInfo *request.RequestInfo) string {
 func canonicalVerb(verb string, scope string) string {
 	switch verb {
 	case "GET", "HEAD":
-		if scope != "resource" {
+		if scope != "resource" && scope != "" {
 			return "LIST"
 		}
 		return "GET"

vendor/k8s.io/apiserver/pkg/endpoints/request/received_time.go

@@ -0,0 +1,45 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package request
+
+import (
+	"context"
+	"time"
+)
+
+type requestReceivedTimestampKeyType int
+
+// requestReceivedTimestampKey is the ReceivedTimestamp (the time the request reached the apiserver)
+// key for the context.
+const requestReceivedTimestampKey requestReceivedTimestampKeyType = iota
+
+// WithReceivedTimestamp returns a copy of parent context in which the ReceivedTimestamp
+// (the time the request reached the apiserver) is set.
+//
+// If the specified ReceivedTimestamp is zero, no value is set and the parent context is returned as is.
+func WithReceivedTimestamp(parent context.Context, receivedTimestamp time.Time) context.Context {
+	if receivedTimestamp.IsZero() {
+		return parent
+	}
+	return WithValue(parent, requestReceivedTimestampKey, receivedTimestamp)
+}
+
+// ReceivedTimestampFrom returns the value of the ReceivedTimestamp key from the specified context.
+func ReceivedTimestampFrom(ctx context.Context) (time.Time, bool) {
+	info, ok := ctx.Value(requestReceivedTimestampKey).(time.Time)
+	return info, ok
+}

vendor/k8s.io/apiserver/pkg/endpoints/request/requestinfo.go

@@ -77,7 +77,7 @@ var specialVerbsNoSubresources = sets.NewString("proxy")
 // this list allows the parser to distinguish between a namespace subresource, and a namespaced resource
 var namespaceSubresources = sets.NewString("status", "finalize")
 
-// NamespaceSubResourcesForTest exports namespaceSubresources for testing in pkg/master/master_test.go, so we never drift
+// NamespaceSubResourcesForTest exports namespaceSubresources for testing in pkg/controlplane/master_test.go, so we never drift
 var NamespaceSubResourcesForTest = sets.NewString(namespaceSubresources.List()...)
 
 type RequestInfoFactory struct {

vendor/k8s.io/apiserver/pkg/features/kube_features.go

@@ -61,6 +61,7 @@ const (
 
 	// owner: @ilackams
 	// alpha: v1.7
+	// beta: v1.16
 	//
 	// Enables compression of REST responses (GET and LIST only)
 	APIResponseCompression featuregate.Feature = "APIResponseCompression"
@@ -85,6 +86,7 @@ const (
 
 	// owner: @caesarxuchao
 	// alpha: v1.15
+	// beta: v1.16
 	//
 	// Allow apiservers to show a count of remaining items in the response
 	// to a chunking list request.
@@ -105,6 +107,12 @@ const (
 	// document.
 	StorageVersionHash featuregate.Feature = "StorageVersionHash"
 
+	// owner: @caesarxuchao @roycaihw
+	// alpha: v1.20
+	//
+	// Enable the storage version API.
+	StorageVersionAPI featuregate.Feature = "StorageVersionAPI"
+
 	// owner: @wojtek-t
 	// alpha: v1.15
 	// beta: v1.16
@@ -122,6 +130,7 @@ const (
 
 	// owner: @wojtek-t
 	// alpha: v1.16
+	// beta: v1.20
 	//
 	// Deprecates and removes SelfLink from ObjectMeta and ListMeta.
 	RemoveSelfLink featuregate.Feature = "RemoveSelfLink"
@@ -129,6 +138,7 @@ const (
 	// owner: @shaloulcy, @wojtek-t
 	// alpha: v1.18
 	// beta: v1.19
+	// GA: v1.20
 	//
 	// Allows label and field based indexes in apiserver watch cache to accelerate list operations.
 	SelectorIndex featuregate.Feature = "SelectorIndex"
@@ -138,6 +148,18 @@ const (
 	//
 	// Allows sending warning headers in API responses.
 	WarningHeaders featuregate.Feature = "WarningHeaders"
+
+	// owner: @wojtek-t
+	// alpha: v1.20
+	//
+	// Allows for updating watchcache resource version with progress notify events.
+	EfficientWatchResumption featuregate.Feature = "EfficientWatchResumption"
+
+	// owner: @roycaihw
+	// alpha: v1.20
+	//
+	// Assigns each kube-apiserver an ID in a cluster.
+	APIServerIdentity featuregate.Feature = "APIServerIdentity"
 )
 
 func init() {
@@ -148,18 +170,21 @@ func init() {
 // To add a new feature, define a key for it above and add it here. The features will be
 // available throughout Kubernetes binaries.
 var defaultKubernetesFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
-	StreamingProxyRedirects: {Default: true, PreRelease: featuregate.Deprecated},
-	ValidateProxyRedirects:  {Default: true, PreRelease: featuregate.Beta},
-	AdvancedAuditing:        {Default: true, PreRelease: featuregate.GA},
-	APIResponseCompression:  {Default: true, PreRelease: featuregate.Beta},
-	APIListChunking:         {Default: true, PreRelease: featuregate.Beta},
-	DryRun:                  {Default: true, PreRelease: featuregate.GA},
-	RemainingItemCount:      {Default: true, PreRelease: featuregate.Beta},
-	ServerSideApply:         {Default: true, PreRelease: featuregate.Beta},
-	StorageVersionHash:      {Default: true, PreRelease: featuregate.Beta},
-	WatchBookmark:           {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
-	APIPriorityAndFairness:  {Default: false, PreRelease: featuregate.Alpha},
-	RemoveSelfLink:          {Default: false, PreRelease: featuregate.Alpha},
-	SelectorIndex:           {Default: true, PreRelease: featuregate.Beta},
-	WarningHeaders:          {Default: true, PreRelease: featuregate.Beta},
+	StreamingProxyRedirects:  {Default: true, PreRelease: featuregate.Deprecated},
+	ValidateProxyRedirects:   {Default: true, PreRelease: featuregate.Beta},
+	AdvancedAuditing:         {Default: true, PreRelease: featuregate.GA},
+	APIResponseCompression:   {Default: true, PreRelease: featuregate.Beta},
+	APIListChunking:          {Default: true, PreRelease: featuregate.Beta},
+	DryRun:                   {Default: true, PreRelease: featuregate.GA},
+	RemainingItemCount:       {Default: true, PreRelease: featuregate.Beta},
+	ServerSideApply:          {Default: true, PreRelease: featuregate.Beta},
+	StorageVersionHash:       {Default: true, PreRelease: featuregate.Beta},
+	StorageVersionAPI:        {Default: false, PreRelease: featuregate.Alpha},
+	WatchBookmark:            {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
+	APIPriorityAndFairness:   {Default: true, PreRelease: featuregate.Beta},
+	RemoveSelfLink:           {Default: true, PreRelease: featuregate.Beta},
+	SelectorIndex:            {Default: true, PreRelease: featuregate.GA, LockToDefault: true},
+	WarningHeaders:           {Default: true, PreRelease: featuregate.Beta},
+	EfficientWatchResumption: {Default: false, PreRelease: featuregate.Alpha},
+	APIServerIdentity:        {Default: false, PreRelease: featuregate.Alpha},
 }

vendor/k8s.io/apiserver/pkg/quota/v1/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- deads2k
+- derekwaynecarr
+- vishh
+reviewers:
+- deads2k
+- derekwaynecarr
+- smarterclayton
+- vishh
+labels:
+- sig/api-machinery

vendor/k8s.io/apiserver/pkg/quota/v1/interfaces.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apiserver/pkg/admission"
+	"k8s.io/client-go/tools/cache"
+)
+
+// UsageStatsOptions is an options structs that describes how stats should be calculated
+type UsageStatsOptions struct {
+	// Namespace where stats should be calculate
+	Namespace string
+	// Scopes that must match counted objects
+	Scopes []corev1.ResourceQuotaScope
+	// Resources are the set of resources to include in the measurement
+	Resources     []corev1.ResourceName
+	ScopeSelector *corev1.ScopeSelector
+}
+
+// UsageStats is result of measuring observed resource use in the system
+type UsageStats struct {
+	// Used maps resource to quantity used
+	Used corev1.ResourceList
+}
+
+// Evaluator knows how to evaluate quota usage for a particular group resource
+type Evaluator interface {
+	// Constraints ensures that each required resource is present on item
+	Constraints(required []corev1.ResourceName, item runtime.Object) error
+	// GroupResource returns the groupResource that this object knows how to evaluate
+	GroupResource() schema.GroupResource
+	// Handles determines if quota could be impacted by the specified attribute.
+	// If true, admission control must perform quota processing for the operation, otherwise it is safe to ignore quota.
+	Handles(operation admission.Attributes) bool
+	// Matches returns true if the specified quota matches the input item
+	Matches(resourceQuota *corev1.ResourceQuota, item runtime.Object) (bool, error)
+	// MatchingScopes takes the input specified list of scopes and input object and returns the set of scopes that matches input object.
+	MatchingScopes(item runtime.Object, scopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error)
+	// UncoveredQuotaScopes takes the input matched scopes which are limited by configuration and the matched quota scopes. It returns the scopes which are in limited scopes but dont have a corresponding covering quota scope
+	UncoveredQuotaScopes(limitedScopes []corev1.ScopedResourceSelectorRequirement, matchedQuotaScopes []corev1.ScopedResourceSelectorRequirement) ([]corev1.ScopedResourceSelectorRequirement, error)
+	// MatchingResources takes the input specified list of resources and returns the set of resources evaluator matches.
+	MatchingResources(input []corev1.ResourceName) []corev1.ResourceName
+	// Usage returns the resource usage for the specified object
+	Usage(item runtime.Object) (corev1.ResourceList, error)
+	// UsageStats calculates latest observed usage stats for all objects
+	UsageStats(options UsageStatsOptions) (UsageStats, error)
+}
+
+// Configuration defines how the quota system is configured.
+type Configuration interface {
+	// IgnoredResources are ignored by quota.
+	IgnoredResources() map[schema.GroupResource]struct{}
+	// Evaluators for quota evaluation.
+	Evaluators() []Evaluator
+}
+
+// Registry maintains a list of evaluators
+type Registry interface {
+	// Add to registry
+	Add(e Evaluator)
+	// Remove from registry
+	Remove(e Evaluator)
+	// Get by group resource
+	Get(gr schema.GroupResource) Evaluator
+	// List from registry
+	List() []Evaluator
+}
+
+// ListerForResourceFunc knows how to get a lister for a specific resource
+type ListerForResourceFunc func(schema.GroupVersionResource) (cache.GenericLister, error)

vendor/k8s.io/apiserver/pkg/quota/v1/resources.go

@@ -0,0 +1,293 @@
+/*
+Copyright 2016 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1
+
+import (
+	"sort"
+	"strings"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/resource"
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/sets"
+)
+
+// Equals returns true if the two lists are equivalent
+func Equals(a corev1.ResourceList, b corev1.ResourceList) bool {
+	if len(a) != len(b) {
+		return false
+	}
+
+	for key, value1 := range a {
+		value2, found := b[key]
+		if !found {
+			return false
+		}
+		if value1.Cmp(value2) != 0 {
+			return false
+		}
+	}
+
+	return true
+}
+
+// LessThanOrEqual returns true if a < b for each key in b
+// If false, it returns the keys in a that exceeded b
+func LessThanOrEqual(a corev1.ResourceList, b corev1.ResourceList) (bool, []corev1.ResourceName) {
+	result := true
+	resourceNames := []corev1.ResourceName{}
+	for key, value := range b {
+		if other, found := a[key]; found {
+			if other.Cmp(value) > 0 {
+				result = false
+				resourceNames = append(resourceNames, key)
+			}
+		}
+	}
+	return result, resourceNames
+}
+
+// Max returns the result of Max(a, b) for each named resource
+func Max(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList {
+	result := corev1.ResourceList{}
+	for key, value := range a {
+		if other, found := b[key]; found {
+			if value.Cmp(other) <= 0 {
+				result[key] = other.DeepCopy()
+				continue
+			}
+		}
+		result[key] = value.DeepCopy()
+	}
+	for key, value := range b {
+		if _, found := result[key]; !found {
+			result[key] = value.DeepCopy()
+		}
+	}
+	return result
+}
+
+// Add returns the result of a + b for each named resource
+func Add(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList {
+	result := corev1.ResourceList{}
+	for key, value := range a {
+		quantity := value.DeepCopy()
+		if other, found := b[key]; found {
+			quantity.Add(other)
+		}
+		result[key] = quantity
+	}
+	for key, value := range b {
+		if _, found := result[key]; !found {
+			result[key] = value.DeepCopy()
+		}
+	}
+	return result
+}
+
+// SubtractWithNonNegativeResult - subtracts and returns result of a - b but
+// makes sure we don't return negative values to prevent negative resource usage.
+func SubtractWithNonNegativeResult(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList {
+	zero := resource.MustParse("0")
+
+	result := corev1.ResourceList{}
+	for key, value := range a {
+		quantity := value.DeepCopy()
+		if other, found := b[key]; found {
+			quantity.Sub(other)
+		}
+		if quantity.Cmp(zero) > 0 {
+			result[key] = quantity
+		} else {
+			result[key] = zero
+		}
+	}
+
+	for key := range b {
+		if _, found := result[key]; !found {
+			result[key] = zero
+		}
+	}
+	return result
+}
+
+// Subtract returns the result of a - b for each named resource
+func Subtract(a corev1.ResourceList, b corev1.ResourceList) corev1.ResourceList {
+	result := corev1.ResourceList{}
+	for key, value := range a {
+		quantity := value.DeepCopy()
+		if other, found := b[key]; found {
+			quantity.Sub(other)
+		}
+		result[key] = quantity
+	}
+	for key, value := range b {
+		if _, found := result[key]; !found {
+			quantity := value.DeepCopy()
+			quantity.Neg()
+			result[key] = quantity
+		}
+	}
+	return result
+}
+
+// Mask returns a new resource list that only has the values with the specified names
+func Mask(resources corev1.ResourceList, names []corev1.ResourceName) corev1.ResourceList {
+	nameSet := ToSet(names)
+	result := corev1.ResourceList{}
+	for key, value := range resources {
+		if nameSet.Has(string(key)) {
+			result[key] = value.DeepCopy()
+		}
+	}
+	return result
+}
+
+// ResourceNames returns a list of all resource names in the ResourceList
+func ResourceNames(resources corev1.ResourceList) []corev1.ResourceName {
+	result := []corev1.ResourceName{}
+	for resourceName := range resources {
+		result = append(result, resourceName)
+	}
+	return result
+}
+
+// Contains returns true if the specified item is in the list of items
+func Contains(items []corev1.ResourceName, item corev1.ResourceName) bool {
+	for _, i := range items {
+		if i == item {
+			return true
+		}
+	}
+	return false
+}
+
+// ContainsPrefix returns true if the specified item has a prefix that contained in given prefix Set
+func ContainsPrefix(prefixSet []string, item corev1.ResourceName) bool {
+	for _, prefix := range prefixSet {
+		if strings.HasPrefix(string(item), prefix) {
+			return true
+		}
+	}
+	return false
+}
+
+// Intersection returns the intersection of both list of resources, deduped and sorted
+func Intersection(a []corev1.ResourceName, b []corev1.ResourceName) []corev1.ResourceName {
+	result := make([]corev1.ResourceName, 0, len(a))
+	for _, item := range a {
+		if Contains(result, item) {
+			continue
+		}
+		if !Contains(b, item) {
+			continue
+		}
+		result = append(result, item)
+	}
+	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })
+	return result
+}
+
+// Difference returns the list of resources resulting from a-b, deduped and sorted
+func Difference(a []corev1.ResourceName, b []corev1.ResourceName) []corev1.ResourceName {
+	result := make([]corev1.ResourceName, 0, len(a))
+	for _, item := range a {
+		if Contains(b, item) || Contains(result, item) {
+			continue
+		}
+		result = append(result, item)
+	}
+	sort.Slice(result, func(i, j int) bool { return result[i] < result[j] })
+	return result
+}
+
+// IsZero returns true if each key maps to the quantity value 0
+func IsZero(a corev1.ResourceList) bool {
+	zero := resource.MustParse("0")
+	for _, v := range a {
+		if v.Cmp(zero) != 0 {
+			return false
+		}
+	}
+	return true
+}
+
+// IsNegative returns the set of resource names that have a negative value.
+func IsNegative(a corev1.ResourceList) []corev1.ResourceName {
+	results := []corev1.ResourceName{}
+	zero := resource.MustParse("0")
+	for k, v := range a {
+		if v.Cmp(zero) < 0 {
+			results = append(results, k)
+		}
+	}
+	return results
+}
+
+// ToSet takes a list of resource names and converts to a string set
+func ToSet(resourceNames []corev1.ResourceName) sets.String {
+	result := sets.NewString()
+	for _, resourceName := range resourceNames {
+		result.Insert(string(resourceName))
+	}
+	return result
+}
+
+// CalculateUsage calculates and returns the requested ResourceList usage.
+// If an error is returned, usage only contains the resources which encountered no calculation errors.
+func CalculateUsage(namespaceName string, scopes []corev1.ResourceQuotaScope, hardLimits corev1.ResourceList, registry Registry, scopeSelector *corev1.ScopeSelector) (corev1.ResourceList, error) {
+	// find the intersection between the hard resources on the quota
+	// and the resources this controller can track to know what we can
+	// look to measure updated usage stats for
+	hardResources := ResourceNames(hardLimits)
+	potentialResources := []corev1.ResourceName{}
+	evaluators := registry.List()
+	for _, evaluator := range evaluators {
+		potentialResources = append(potentialResources, evaluator.MatchingResources(hardResources)...)
+	}
+	// NOTE: the intersection just removes duplicates since the evaluator match intersects with hard
+	matchedResources := Intersection(hardResources, potentialResources)
+
+	errors := []error{}
+
+	// sum the observed usage from each evaluator
+	newUsage := corev1.ResourceList{}
+	for _, evaluator := range evaluators {
+		// only trigger the evaluator if it matches a resource in the quota, otherwise, skip calculating anything
+		intersection := evaluator.MatchingResources(matchedResources)
+		if len(intersection) == 0 {
+			continue
+		}
+
+		usageStatsOptions := UsageStatsOptions{Namespace: namespaceName, Scopes: scopes, Resources: intersection, ScopeSelector: scopeSelector}
+		stats, err := evaluator.UsageStats(usageStatsOptions)
+		if err != nil {
+			// remember the error
+			errors = append(errors, err)
+			// exclude resources which encountered calculation errors
+			matchedResources = Difference(matchedResources, intersection)
+			continue
+		}
+		newUsage = Add(newUsage, stats.Used)
+	}
+
+	// mask the observed usage to only the set of resources tracked by this quota
+	// merge our observed usage with the quota usage status
+	// if the new usage is different than the last usage, we will need to do an update
+	newUsage = Mask(newUsage, matchedResources)
+	return newUsage, utilerrors.NewAggregate(errors)
+}

vendor/k8s.io/apiserver/pkg/registry/generic/OWNERS

@@ -24,6 +24,5 @@ reviewers:
 - krousey
 - xiang90
 - resouer
-- mqliang
 - sdminonne
 - enj

vendor/k8s.io/apiserver/pkg/registry/generic/registry/dryrun.go

@@ -78,7 +78,7 @@ func (s *DryRunnableStorage) List(ctx context.Context, key string, opts storage.
 
 func (s *DryRunnableStorage) GuaranteedUpdate(
 	ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
-	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, dryRun bool, suggestion ...runtime.Object) error {
+	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, dryRun bool, suggestion runtime.Object) error {
 	if dryRun {
 		err := s.Storage.Get(ctx, key, storage.GetOptions{IgnoreNotFound: ignoreNotFound}, ptrToType)
 		if err != nil {
@@ -98,7 +98,7 @@ func (s *DryRunnableStorage) GuaranteedUpdate(
 		}
 		return s.copyInto(out, ptrToType)
 	}
-	return s.Storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, suggestion...)
+	return s.Storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, suggestion)
 }
 
 func (s *DryRunnableStorage) Count(key string) (int64, error) {

vendor/k8s.io/apiserver/pkg/registry/generic/registry/storage_factory.go

@@ -45,7 +45,7 @@ func StorageWithCacher() generic.StorageDecorator {
 		triggerFuncs storage.IndexerFuncs,
 		indexers *cache.Indexers) (storage.Interface, factory.DestroyFunc, error) {
 
-		s, d, err := generic.NewRawStorage(storageConfig)
+		s, d, err := generic.NewRawStorage(storageConfig, newFunc)
 		if err != nil {
 			return s, d, err
 		}

vendor/k8s.io/apiserver/pkg/registry/generic/registry/store.go

@@ -568,7 +568,7 @@ func (e *Store) Update(ctx context.Context, name string, objInfo rest.UpdatedObj
 			return obj, &ttl, nil
 		}
 		return obj, nil, nil
-	}, dryrun.IsDryRun(options.DryRun))
+	}, dryrun.IsDryRun(options.DryRun), nil)
 
 	if err != nil {
 		// delete the object
@@ -683,10 +683,7 @@ func shouldOrphanDependents(ctx context.Context, e *Store, accessor metav1.Objec
 	}
 
 	// Get default orphan policy from this REST object type if it exists
-	if defaultGCPolicy == rest.OrphanDependents {
-		return true
-	}
-	return false
+	return defaultGCPolicy == rest.OrphanDependents
 }
 
 // shouldDeleteDependents returns true if the finalizer for foreground deletion should be set
@@ -858,6 +855,7 @@ func (e *Store) updateForGracefulDeletionAndFinalizers(ctx context.Context, name
 			return existing, nil
 		}),
 		dryrun.IsDryRun(options.DryRun),
+		nil,
 	)
 	switch err {
 	case nil:

vendor/k8s.io/apiserver/pkg/registry/generic/storage_decorator.go

@@ -47,12 +47,12 @@ func UndecoratedStorage(
 	getAttrsFunc storage.AttrFunc,
 	trigger storage.IndexerFuncs,
 	indexers *cache.Indexers) (storage.Interface, factory.DestroyFunc, error) {
-	return NewRawStorage(config)
+	return NewRawStorage(config, newFunc)
 }
 
 // NewRawStorage creates the low level kv storage. This is a work-around for current
 // two layer of same storage interface.
 // TODO: Once cacher is enabled on all registries (event registry is special), we will remove this method.
-func NewRawStorage(config *storagebackend.Config) (storage.Interface, factory.DestroyFunc, error) {
-	return factory.Create(*config)
+func NewRawStorage(config *storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, factory.DestroyFunc, error) {
+	return factory.Create(*config, newFunc)
 }

vendor/k8s.io/apiserver/pkg/registry/rest/OWNERS

@@ -17,7 +17,6 @@ reviewers:
 - dims
 - hongchaodeng
 - krousey
-- euank
 - ingvagabund
 - jianhuiz
 - sdminonne

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -50,6 +50,7 @@ import (
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	authorizerunion "k8s.io/apiserver/pkg/authorization/union"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/apiserver/pkg/endpoints/filterlatency"
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -61,6 +62,7 @@ import (
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
+	"k8s.io/apiserver/pkg/storageversion"
 	"k8s.io/apiserver/pkg/util/feature"
 	utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
 	"k8s.io/client-go/informers"
@@ -223,6 +225,12 @@ type Config struct {
 	// EquivalentResourceRegistry provides information about resources equivalent to a given resource,
 	// and the kind associated with a given resource. As resources are installed, they are registered here.
 	EquivalentResourceRegistry runtime.EquivalentResourceRegistry
+
+	// APIServerID is the ID of this API server
+	APIServerID string
+
+	// StorageVersionManager holds the storage versions of the API resources installed by this server.
+	StorageVersionManager storageversion.Manager
 }
 
 type RecommendedConfig struct {
@@ -286,6 +294,10 @@ type AuthorizationInfo struct {
 // NewConfig returns a Config struct with the default values
 func NewConfig(codecs serializer.CodecFactory) *Config {
 	defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
+	var id string
+	if feature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
+		id = "kube-apiserver-" + uuid.New().String()
+	}
 	return &Config{
 		Serializer:                  codecs,
 		BuildHandlerChainFunc:       DefaultBuildHandlerChain,
@@ -323,7 +335,9 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 
 		// Default to treating watch as a long-running operation
 		// Generic API servers have no inherent long-running subresources
-		LongRunningFunc: genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
+		LongRunningFunc:       genericfilters.BasicLongRunningRequestCheck(sets.NewString("watch"), sets.NewString()),
+		APIServerID:           id,
+		StorageVersionManager: storageversion.NewDefaultManager(),
 	}
 }
 
@@ -573,6 +587,9 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 
 		maxRequestBodyBytes: c.MaxRequestBodyBytes,
 		livezClock:          clock.RealClock{},
+
+		APIServerID:           c.APIServerID,
+		StorageVersionManager: c.StorageVersionManager,
 	}
 
 	for {
@@ -638,6 +655,31 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		klog.V(3).Infof("Not requested to run hook %s", priorityAndFairnessConfigConsumerHookName)
 	}
 
+	// Add PostStartHooks for maintaining the watermarks for the Priority-and-Fairness and the Max-in-Flight filters.
+	if c.FlowControl != nil {
+		const priorityAndFairnessFilterHookName = "priority-and-fairness-filter"
+		if !s.isPostStartHookRegistered(priorityAndFairnessFilterHookName) {
+			err := s.AddPostStartHook(priorityAndFairnessFilterHookName, func(context PostStartHookContext) error {
+				genericfilters.StartPriorityAndFairnessWatermarkMaintenance(context.StopCh)
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+	} else {
+		const maxInFlightFilterHookName = "max-in-flight-filter"
+		if !s.isPostStartHookRegistered(maxInFlightFilterHookName) {
+			err := s.AddPostStartHook(maxInFlightFilterHookName, func(context PostStartHookContext) error {
+				genericfilters.StartMaxInFlightWatermarkMaintenance(context.StopCh)
+				return nil
+			})
+			if err != nil {
+				return nil, err
+			}
+		}
+	}
+
 	for _, delegateCheck := range delegationTarget.HealthzChecks() {
 		skip := false
 		for _, existingCheck := range c.HealthzChecks {
@@ -668,18 +710,41 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 	return s, nil
 }
 
+func BuildHandlerChainWithStorageVersionPrecondition(apiHandler http.Handler, c *Config) http.Handler {
+	// WithStorageVersionPrecondition needs the WithRequestInfo to run first
+	handler := genericapifilters.WithStorageVersionPrecondition(apiHandler, c.StorageVersionManager, c.Serializer)
+	return DefaultBuildHandlerChain(handler, c)
+}
+
 func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
-	handler := genericapifilters.WithAuthorization(apiHandler, c.Authorization.Authorizer, c.Serializer)
+	handler := filterlatency.TrackCompleted(apiHandler)
+	handler = genericapifilters.WithAuthorization(handler, c.Authorization.Authorizer, c.Serializer)
+	handler = filterlatency.TrackStarted(handler, "authorization")
+
 	if c.FlowControl != nil {
+		handler = filterlatency.TrackCompleted(handler)
 		handler = genericfilters.WithPriorityAndFairness(handler, c.LongRunningFunc, c.FlowControl)
+		handler = filterlatency.TrackStarted(handler, "priorityandfairness")
 	} else {
 		handler = genericfilters.WithMaxInFlightLimit(handler, c.MaxRequestsInFlight, c.MaxMutatingRequestsInFlight, c.LongRunningFunc)
 	}
+
+	handler = filterlatency.TrackCompleted(handler)
 	handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
+	handler = filterlatency.TrackStarted(handler, "impersonation")
+
+	handler = filterlatency.TrackCompleted(handler)
 	handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
+	handler = filterlatency.TrackStarted(handler, "audit")
+
 	failedHandler := genericapifilters.Unauthorized(c.Serializer)
 	failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyChecker)
+
+	failedHandler = filterlatency.TrackCompleted(failedHandler)
+	handler = filterlatency.TrackCompleted(handler)
 	handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
+	handler = filterlatency.TrackStarted(handler, "authentication")
+
 	handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
 	handler = genericfilters.WithTimeoutForNonLongRunningRequests(handler, c.LongRunningFunc, c.RequestTimeout)
 	handler = genericfilters.WithWaitGroup(handler, c.LongRunningFunc, c.HandlerChainWaitGroup)
@@ -690,7 +755,8 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
 	handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyChecker)
 	handler = genericapifilters.WithWarningRecorder(handler)
 	handler = genericapifilters.WithCacheControl(handler)
-	handler = genericfilters.WithPanicRecovery(handler)
+	handler = genericapifilters.WithRequestReceivedTimestamp(handler)
+	handler = genericfilters.WithPanicRecovery(handler, c.RequestInfoResolver)
 	return handler
 }
 
@@ -719,7 +785,7 @@ func installAPI(s *GenericAPIServer, c *Config) {
 	if c.EnableDiscovery {
 		s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
 	}
-	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
+	if c.FlowControl != nil && feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
 		c.FlowControl.Install(s.Handler.NonGoRestfulMux)
 	}
 }

vendor/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -22,6 +22,7 @@ import (
 	"strings"
 
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/validation/field"
 	"k8s.io/apiserver/pkg/apis/apiserver"
 	"k8s.io/apiserver/pkg/apis/apiserver/install"
@@ -32,6 +33,10 @@ import (
 
 var cfgScheme = runtime.NewScheme()
 
+// validEgressSelectorNames contains the set of valid egress selctor names.
+// 'master' is deprecated in favor of 'controlplane' and will be removed in v1.22.
+var validEgressSelectorNames = sets.NewString("master", "controlplane", "cluster", "etcd")
+
 func init() {
 	install.Install(cfgScheme)
 }
@@ -97,6 +102,30 @@ func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfigu
 				}))
 		}
 	}
+
+	var foundControlPlane, foundMaster bool
+	for _, service := range config.EgressSelections {
+		canonicalName := strings.ToLower(service.Name)
+
+		if !validEgressSelectorNames.Has(canonicalName) {
+			allErrs = append(allErrs, field.NotSupported(field.NewPath("egressSelection", "name"), canonicalName, validEgressSelectorNames.List()))
+			continue
+		}
+
+		if canonicalName == "master" {
+			foundMaster = true
+		}
+
+		if canonicalName == "controlplane" {
+			foundControlPlane = true
+		}
+	}
+
+	// error if both master and controlplane egress selectors are set
+	if foundMaster && foundControlPlane {
+		allErrs = append(allErrs, field.Forbidden(field.NewPath("egressSelection", "name"), "both egressSelection names 'master' and 'controlplane' are specified, only one is allowed"))
+	}
+
 	return allErrs
 }
 
@@ -199,7 +228,7 @@ func validateTLSConfig(tlsConfig *apiserver.TLSConfig, fldPath *field.Path) fiel
 		return allErrs
 	}
 	if tlsConfig.CABundle != "" {
-		if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.CABundle); exists == false || err != nil {
+		if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.CABundle); !exists || err != nil {
 			allErrs = append(allErrs, field.Invalid(
 				fldPath.Child("tlsConfig", "caBundle"),
 				tlsConfig.CABundle,
@@ -211,7 +240,7 @@ func validateTLSConfig(tlsConfig *apiserver.TLSConfig, fldPath *field.Path) fiel
 			fldPath.Child("tlsConfig", "clientCert"),
 			"nil",
 			"Using TLS requires clientCert"))
-	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientCert); exists == false || err != nil {
+	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientCert); !exists || err != nil {
 		allErrs = append(allErrs, field.Invalid(
 			fldPath.Child("tlsConfig", "clientCert"),
 			tlsConfig.ClientCert,
@@ -222,7 +251,7 @@ func validateTLSConfig(tlsConfig *apiserver.TLSConfig, fldPath *field.Path) fiel
 			fldPath.Child("tlsConfig", "clientKey"),
 			"nil",
 			"Using TLS requires requires clientKey"))
-	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientKey); exists == false || err != nil {
+	} else if exists, err := path.Exists(path.CheckFollowSymlink, tlsConfig.ClientKey); !exists || err != nil {
 		allErrs = append(allErrs, field.Invalid(
 			fldPath.Child("tlsConfig", "clientKey"),
 			tlsConfig.ClientKey,

vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -51,8 +51,8 @@ type EgressSelector struct {
 type EgressType int
 
 const (
-	// Master is the EgressType for traffic intended to go to the control plane.
-	Master EgressType = iota
+	// ControlPlane is the EgressType for traffic intended to go to the control plane.
+	ControlPlane EgressType = iota
 	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
 	Etcd
 	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
@@ -73,8 +73,8 @@ type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
 // String returns the canonical string representation of the egress type
 func (s EgressType) String() string {
 	switch s {
-	case Master:
-		return "master"
+	case ControlPlane:
+		return "controlplane"
 	case Etcd:
 		return "etcd"
 	case Cluster:
@@ -91,8 +91,12 @@ func (s EgressType) AsNetworkContext() NetworkContext {
 
 func lookupServiceName(name string) (EgressType, error) {
 	switch strings.ToLower(name) {
+	// 'master' is deprecated, interpret "master" as controlplane internally until removed in v1.22.
 	case "master":
-		return Master, nil
+		klog.Warning("EgressSelection name 'master' is deprecated, use 'controlplane' instead")
+		return ControlPlane, nil
+	case "controlplane":
+		return ControlPlane, nil
 	case "etcd":
 		return Etcd, nil
 	case "cluster":
@@ -364,5 +368,6 @@ func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFun
 		// The round trip wrapper will over-ride the dialContext method appropriately
 		return nil, nil
 	}
+
 	return cs.egressToDialer[networkContext.EgressSelectionName], nil
 }

vendor/k8s.io/apiserver/pkg/server/filters/goaway.go

@@ -80,9 +80,5 @@ type probabilisticGoawayDecider struct {
 
 // Goaway implement GoawayDecider
 func (p *probabilisticGoawayDecider) Goaway(r *http.Request) bool {
-	if p.next() < p.chance {
-		return true
-	}
-
-	return false
+	return p.next() < p.chance
 }

vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -41,6 +41,10 @@ const (
 	// the metrics tracks maximal value over period making this
 	// longer will increase the metric value.
 	inflightUsageMetricUpdatePeriod = time.Second
+
+	// How often to run maintenance on observations to ensure
+	// that they do not fall too far behind.
+	observationMaintenancePeriod = 10 * time.Second
 )
 
 var nonMutatingRequestVerbs = sets.NewString("get", "list", "watch")
@@ -88,23 +92,29 @@ var watermark = &requestWatermark{
 	mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{metrics.MutatingKind}).RequestsExecuting,
 }
 
-func startRecordingUsage(watermark *requestWatermark) {
-	go func() {
-		wait.Forever(func() {
-			watermark.lock.Lock()
-			readOnlyWatermark := watermark.readOnlyWatermark
-			mutatingWatermark := watermark.mutatingWatermark
-			watermark.readOnlyWatermark = 0
-			watermark.mutatingWatermark = 0
-			watermark.lock.Unlock()
+// startWatermarkMaintenance starts the goroutines to observe and maintain the specified watermark.
+func startWatermarkMaintenance(watermark *requestWatermark, stopCh <-chan struct{}) {
+	// Periodically update the inflight usage metric.
+	go wait.Until(func() {
+		watermark.lock.Lock()
+		readOnlyWatermark := watermark.readOnlyWatermark
+		mutatingWatermark := watermark.mutatingWatermark
+		watermark.readOnlyWatermark = 0
+		watermark.mutatingWatermark = 0
+		watermark.lock.Unlock()
 
-			metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
-		}, inflightUsageMetricUpdatePeriod)
-	}()
+		metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
+	}, inflightUsageMetricUpdatePeriod, stopCh)
+
+	// Periodically observe the watermarks. This is done to ensure that they do not fall too far behind. When they do
+	// fall too far behind, then there is a long delay in responding to the next request received while the observer
+	// catches back up.
+	go wait.Until(func() {
+		watermark.readOnlyObserver.Add(0)
+		watermark.mutatingObserver.Add(0)
+	}, observationMaintenancePeriod, stopCh)
 }
 
-var startOnce sync.Once
-
 // WithMaxInFlightLimit limits the number of in-flight requests to buffer size of the passed in channel.
 func WithMaxInFlightLimit(
 	handler http.Handler,
@@ -112,7 +122,6 @@ func WithMaxInFlightLimit(
 	mutatingLimit int,
 	longRunningRequestCheck apirequest.LongRunningRequestCheck,
 ) http.Handler {
-	startOnce.Do(func() { startRecordingUsage(watermark) })
 	if nonMutatingLimit == 0 && mutatingLimit == 0 {
 		return handler
 	}
@@ -198,6 +207,12 @@ func WithMaxInFlightLimit(
 	})
 }
 
+// StartMaxInFlightWatermarkMaintenance starts the goroutines to observe and maintain watermarks for max-in-flight
+// requests.
+func StartMaxInFlightWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+}
+
 func tooManyRequests(req *http.Request, w http.ResponseWriter) {
 	// Return a 429 status indicating "Too Many Requests"
 	w.Header().Set("Retry-After", retryAfter)

vendor/k8s.io/apiserver/pkg/server/filters/priority-and-fairness.go

@@ -22,7 +22,7 @@ import (
 	"net/http"
 	"sync/atomic"
 
-	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
 	apitypes "k8s.io/apimachinery/pkg/types"
 	epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
@@ -35,11 +35,6 @@ type priorityAndFairnessKeyType int
 
 const priorityAndFairnessKey priorityAndFairnessKeyType = iota
 
-const (
-	responseHeaderMatchedPriorityLevelConfigurationUID = "X-Kubernetes-PF-PriorityLevel-UID"
-	responseHeaderMatchedFlowSchemaUID                 = "X-Kubernetes-PF-FlowSchema-UID"
-)
-
 // PriorityAndFairnessClassification identifies the results of
 // classification for API Priority and Fairness
 type PriorityAndFairnessClassification struct {
@@ -76,10 +71,6 @@ func WithPriorityAndFairness(
 		klog.Warningf("priority and fairness support not found, skipping")
 		return handler
 	}
-	startOnce.Do(func() {
-		startRecordingUsage(watermark)
-		startRecordingUsage(waitingMark)
-	})
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		ctx := r.Context()
 		requestInfo, ok := apirequest.RequestInfoFrom(ctx)
@@ -101,7 +92,7 @@ func WithPriorityAndFairness(
 		}
 
 		var classification *PriorityAndFairnessClassification
-		note := func(fs *fcv1a1.FlowSchema, pl *fcv1a1.PriorityLevelConfiguration) {
+		note := func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration) {
 			classification = &PriorityAndFairnessClassification{
 				FlowSchemaName:    fs.Name,
 				FlowSchemaUID:     fs.UID,
@@ -131,11 +122,11 @@ func WithPriorityAndFairness(
 			served = true
 			innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
 			innerReq := r.Clone(innerCtx)
-			w.Header().Set(responseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
-			w.Header().Set(responseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
+			w.Header().Set(flowcontrol.ResponseHeaderMatchedPriorityLevelConfigurationUID, string(classification.PriorityLevelUID))
+			w.Header().Set(flowcontrol.ResponseHeaderMatchedFlowSchemaUID, string(classification.FlowSchemaUID))
 			handler.ServeHTTP(w, innerReq)
 		}
-		digest := utilflowcontrol.RequestDigest{requestInfo, user}
+		digest := utilflowcontrol.RequestDigest{RequestInfo: requestInfo, User: user}
 		fcIfc.Handle(ctx, digest, note, func(inQueue bool) {
 			if inQueue {
 				noteWaitingDelta(1)
@@ -144,9 +135,21 @@ func WithPriorityAndFairness(
 			}
 		}, execute)
 		if !served {
+			if isMutatingRequest {
+				epmetrics.DroppedRequests.WithLabelValues(epmetrics.MutatingKind).Inc()
+			} else {
+				epmetrics.DroppedRequests.WithLabelValues(epmetrics.ReadOnlyKind).Inc()
+			}
 			epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
 			tooManyRequests(r, w)
 		}
 
 	})
 }
+
+// StartPriorityAndFairnessWatermarkMaintenance starts the goroutines to observe and maintain watermarks for
+// priority-and-fairness requests.
+func StartPriorityAndFairnessWatermarkMaintenance(stopCh <-chan struct{}) {
+	startWatermarkMaintenance(watermark, stopCh)
+	startWatermarkMaintenance(waitingMark, stopCh)
+}

vendor/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -33,8 +33,6 @@ import (
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
 )
 
-var errConnKilled = fmt.Errorf("killing connection/stream because serving request timed out and response had been started")
-
 // WithTimeoutForNonLongRunningRequests times out non-long-running requests after the time given by timeout.
 func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apirequest.LongRunningRequestCheck, timeout time.Duration) http.Handler {
 	if longRunning == nil {
@@ -246,15 +244,17 @@ func (tw *baseTimeoutWriter) timeout(err *apierrors.StatusError) {
 		// no way to timeout the HTTP request at the point. We have to shutdown
 		// the connection for HTTP1 or reset stream for HTTP2.
 		//
-		// Note from: Brad Fitzpatrick
-		// if the ServeHTTP goroutine panics, that will do the best possible thing for both
-		// HTTP/1 and HTTP/2. In HTTP/1, assuming you're replying with at least HTTP/1.1 and
-		// you've already flushed the headers so it's using HTTP chunking, it'll kill the TCP
-		// connection immediately without a proper 0-byte EOF chunk, so the peer will recognize
-		// the response as bogus. In HTTP/2 the server will just RST_STREAM the stream, leaving
-		// the TCP connection open, but resetting the stream to the peer so it'll have an error,
-		// like the HTTP/1 case.
-		panic(errConnKilled)
+		// Note from the golang's docs:
+		// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+		// that the effect of the panic was isolated to the active request.
+		// It recovers the panic, logs a stack trace to the server error log,
+		// and either closes the network connection or sends an HTTP/2
+		// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+		// the client sees an interrupted response but the server doesn't log
+		// an error, panic with the value ErrAbortHandler.
+		//
+		// We are throwing http.ErrAbortHandler deliberately so that a client is notified and to suppress a not helpful stacktrace in the logs
+		panic(http.ErrAbortHandler)
 	}
 }
 

vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -17,22 +17,41 @@ limitations under the License.
 package filters
 
 import (
+	"fmt"
 	"net/http"
 
-	"k8s.io/klog/v2"
-
 	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/endpoints/metrics"
+	"k8s.io/apiserver/pkg/endpoints/request"
 	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/klog/v2"
 )
 
 // WithPanicRecovery wraps an http Handler to recover and log panics (except in the special case of http.ErrAbortHandler panics, which suppress logging).
-func WithPanicRecovery(handler http.Handler) http.Handler {
+func WithPanicRecovery(handler http.Handler, resolver request.RequestInfoResolver) http.Handler {
 	return withPanicRecovery(handler, func(w http.ResponseWriter, req *http.Request, err interface{}) {
 		if err == http.ErrAbortHandler {
-			// honor the http.ErrAbortHandler sentinel panic value:
-			//   ErrAbortHandler is a sentinel panic value to abort a handler.
-			//   While any panic from ServeHTTP aborts the response to the client,
-			//   panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
+			// Honor the http.ErrAbortHandler sentinel panic value
+			//
+			// If ServeHTTP panics, the server (the caller of ServeHTTP) assumes
+			// that the effect of the panic was isolated to the active request.
+			// It recovers the panic, logs a stack trace to the server error log,
+			// and either closes the network connection or sends an HTTP/2
+			// RST_STREAM, depending on the HTTP protocol. To abort a handler so
+			// the client sees an interrupted response but the server doesn't log
+			// an error, panic with the value ErrAbortHandler.
+			//
+			// Note that the ReallyCrash variable controls the behaviour of the HandleCrash function
+			// So it might actually crash, after calling the handlers
+			if info, err := resolver.NewRequestInfo(req); err != nil {
+				metrics.RecordRequestAbort(req, nil)
+			} else {
+				metrics.RecordRequestAbort(req, info)
+			}
+			// This call can have different handlers, but the default chain rate limits. Call it after the metrics are updated
+			// in case the rate limit delays it.  If you outrun the rate for this one timed out requests, something has gone
+			// seriously wrong with your server, but generally having a logging signal for timeouts is useful.
+			runtime.HandleError(fmt.Errorf("timeout or abort while handling: %v %q", req.Method, req.URL.Path))
 			return
 		}
 		http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -40,9 +40,13 @@ import (
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	genericapi "k8s.io/apiserver/pkg/endpoints"
 	"k8s.io/apiserver/pkg/endpoints/discovery"
+	"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
+	"k8s.io/apiserver/pkg/features"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
+	"k8s.io/apiserver/pkg/storageversion"
+	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
 	restclient "k8s.io/client-go/rest"
 	"k8s.io/klog/v2"
@@ -179,10 +183,6 @@ type GenericAPIServer struct {
 	// and the kind associated with a given resource. As resources are installed, they are registered here.
 	EquivalentResourceRegistry runtime.EquivalentResourceRegistry
 
-	// enableAPIResponseCompression indicates whether API Responses should support compression
-	// if the client requests it via Accept-Encoding
-	enableAPIResponseCompression bool
-
 	// delegationTarget is the next delegate in the chain. This is never nil.
 	delegationTarget DelegationTarget
 
@@ -197,6 +197,12 @@ type GenericAPIServer struct {
 	// The limit on the request body size that would be accepted and decoded in a write request.
 	// 0 means no limit.
 	maxRequestBodyBytes int64
+
+	// APIServerID is the ID of this API server
+	APIServerID string
+
+	// StorageVersionManager holds the storage versions of the API resources installed by this server.
+	StorageVersionManager storageversion.Manager
 }
 
 // DelegationTarget is an interface which allows for composition of API servers with top level handling that works
@@ -407,6 +413,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) (<-chan
 
 // installAPIResources is a private method for installing the REST storage backing each api groupversionresource
 func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *APIGroupInfo, openAPIModels openapiproto.Models) error {
+	var resourceInfos []*storageversion.ResourceInfo
 	for _, groupVersion := range apiGroupInfo.PrioritizedVersions {
 		if len(apiGroupInfo.VersionedResourcesStorageMap[groupVersion.Version]) == 0 {
 			klog.Warningf("Skipping API %v because it has no resources.", groupVersion)
@@ -418,11 +425,30 @@ func (s *GenericAPIServer) installAPIResources(apiPrefix string, apiGroupInfo *A
 			apiGroupVersion.OptionsExternalVersion = apiGroupInfo.OptionsExternalVersion
 		}
 		apiGroupVersion.OpenAPIModels = openAPIModels
+
+		if openAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
+			typeConverter, err := fieldmanager.NewTypeConverter(openAPIModels, false)
+			if err != nil {
+				return err
+			}
+			apiGroupVersion.TypeConverter = typeConverter
+		}
+
 		apiGroupVersion.MaxRequestBodyBytes = s.maxRequestBodyBytes
 
-		if err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer); err != nil {
+		r, err := apiGroupVersion.InstallREST(s.Handler.GoRestfulContainer)
+		if err != nil {
 			return fmt.Errorf("unable to setup API %v: %v", apiGroupInfo, err)
 		}
+		resourceInfos = append(resourceInfos, r...)
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
+		utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) {
+		// API installation happens before we start listening on the handlers,
+		// therefore it is safe to register ResourceInfos here. The handler will block
+		// write requests until the storage versions of the targeting resources are updated.
+		s.StorageVersionManager.AddResourceInfo(resourceInfos...)
 	}
 
 	return nil

vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -161,6 +161,7 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
 
 	klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
 
+	name := strings.Split(strings.TrimPrefix(path, "/"), "/")[0]
 	mux.Handle(path,
 		metrics.InstrumentHandlerFunc("GET",
 			/* group = */ "",
@@ -171,7 +172,7 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
 			/* component = */ "",
 			/* deprecated */ false,
 			/* removedRelease */ "",
-			handleRootHealthz(checks...)))
+			handleRootHealth(name, checks...)))
 	for _, check := range checks {
 		mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
 	}
@@ -207,8 +208,8 @@ func getExcludedChecks(r *http.Request) sets.String {
 	return sets.NewString()
 }
 
-// handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
-func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
+// handleRootHealth returns an http.HandlerFunc that serves the provided checks.
+func handleRootHealth(name string, checks ...HealthChecker) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		excluded := getExcludedChecks(r)
 		// failedVerboseLogOutput is for output to the log.  It indicates detailed failed output information for the log.
@@ -240,8 +241,8 @@ func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
 		}
 		// always be verbose on failure
 		if len(failedChecks) > 0 {
-			klog.V(2).Infof("healthz check failed: %s\n%v", strings.Join(failedChecks, ","), failedVerboseLogOutput.String())
-			http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%shealthz check failed", individualCheckOutput.String()), http.StatusInternalServerError)
+			klog.V(2).Infof("%s check failed: %s\n%v", strings.Join(failedChecks, ","), name, failedVerboseLogOutput.String())
+			http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%s%s check failed", individualCheckOutput.String(), name), http.StatusInternalServerError)
 			return
 		}
 
@@ -253,7 +254,7 @@ func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
 		}
 
 		individualCheckOutput.WriteTo(w)
-		fmt.Fprint(w, "healthz check passed\n")
+		fmt.Fprintf(w, "%s check passed\n", name)
 	})
 }
 

vendor/k8s.io/apiserver/pkg/server/options/audit.go

@@ -37,6 +37,7 @@ import (
 	"k8s.io/apiserver/pkg/audit/policy"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/egressselector"
+	"k8s.io/apiserver/pkg/util/webhook"
 	pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
 	pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
 	plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
@@ -120,6 +121,7 @@ type AuditLogOptions struct {
 	MaxBackups int
 	MaxSize    int
 	Format     string
+	Compress   bool
 
 	BatchOptions    AuditBatchOptions
 	TruncateOptions AuditTruncateOptions
@@ -153,7 +155,7 @@ type AuditDynamicOptions struct {
 func NewAuditOptions() *AuditOptions {
 	return &AuditOptions{
 		WebhookOptions: AuditWebhookOptions{
-			InitialBackoff: pluginwebhook.DefaultInitialBackoff,
+			InitialBackoff: pluginwebhook.DefaultInitialBackoffDelay,
 			BatchOptions: AuditBatchOptions{
 				Mode:        ModeBatch,
 				BatchConfig: defaultWebhookBatchConfig(),
@@ -306,7 +308,7 @@ func (o *AuditOptions) ApplyTo(
 			klog.V(2).Info("No audit policy file provided, no events will be recorded for webhook backend")
 		} else {
 			if c.EgressSelector != nil {
-				egressDialer, err := c.EgressSelector.Lookup(egressselector.Master.AsNetworkContext())
+				egressDialer, err := c.EgressSelector.Lookup(egressselector.ControlPlane.AsNetworkContext())
 				if err != nil {
 					return err
 				}
@@ -449,6 +451,7 @@ func (o *AuditLogOptions) AddFlags(fs *pflag.FlagSet) {
 			strings.Join(pluginlog.AllowedFormats, ",")+".")
 	fs.StringVar(&o.GroupVersionString, "audit-log-version", o.GroupVersionString,
 		"API group and version used for serializing audit events written to log.")
+	fs.BoolVar(&o.Compress, "audit-log-compress", o.Compress, "If set, the rotated log files will be compressed using gzip.")
 }
 
 func (o *AuditLogOptions) Validate() []error {
@@ -513,6 +516,7 @@ func (o *AuditLogOptions) getWriter() io.Writer {
 			MaxAge:     o.MaxAge,
 			MaxBackups: o.MaxBackups,
 			MaxSize:    o.MaxSize,
+			Compress:   o.Compress,
 		}
 	}
 	return w
@@ -566,7 +570,7 @@ func (o *AuditWebhookOptions) enabled() bool {
 // this is done so that the same trucate backend can wrap both the webhook and dynamic backends
 func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc) (audit.Backend, error) {
 	groupVersion, _ := schema.ParseGroupVersion(o.GroupVersionString)
-	webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, o.InitialBackoff, customDial)
+	webhook, err := pluginwebhook.NewBackend(o.ConfigFile, groupVersion, webhook.DefaultRetryBackoffWithInitialDelay(o.InitialBackoff), customDial)
 	if err != nil {
 		return nil, fmt.Errorf("initializing audit webhook: %v", err)
 	}

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -26,6 +26,7 @@ import (
 	"github.com/spf13/pflag"
 
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
 	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server"
@@ -36,6 +37,17 @@ import (
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
 
+// DefaultAuthWebhookRetryBackoff is the default backoff parameters for
+// both authentication and authorization webhook used by the apiserver.
+func DefaultAuthWebhookRetryBackoff() *wait.Backoff {
+	return &wait.Backoff{
+		Duration: 500 * time.Millisecond,
+		Factor:   1.5,
+		Jitter:   0.2,
+		Steps:    5,
+	}
+}
+
 type RequestHeaderAuthenticationOptions struct {
 	// ClientCAFile is the root certificate bundle to verify client certificates on incoming requests
 	// before trusting usernames in headers.
@@ -177,6 +189,15 @@ type DelegatingAuthenticationOptions struct {
 	// TolerateInClusterLookupFailure indicates failures to look up authentication configuration from the cluster configmap should not be fatal.
 	// Setting this can result in an authenticator that will reject all requests.
 	TolerateInClusterLookupFailure bool
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authentication webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
+
+	// ClientTimeout specifies a time limit for requests made by the authorization webhook client.
+	// The default value is set to 10 seconds.
+	ClientTimeout time.Duration
 }
 
 func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
@@ -189,13 +210,29 @@ func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
 			GroupHeaders:        []string{"x-remote-group"},
 			ExtraHeaderPrefixes: []string{"x-remote-extra-"},
 		},
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
+		ClientTimeout:       10 * time.Second,
 	}
 }
 
+// WithCustomRetryBackoff sets the custom backoff parameters for the authentication webhook retry logic.
+func (s *DelegatingAuthenticationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
+// WithClientTimeout sets the given timeout for the authentication webhook client.
+func (s *DelegatingAuthenticationOptions) WithClientTimeout(timeout time.Duration) {
+	s.ClientTimeout = timeout
+}
+
 func (s *DelegatingAuthenticationOptions) Validate() []error {
 	allErrors := []error{}
 	allErrors = append(allErrors, s.RequestHeader.Validate()...)
 
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
 	return allErrors
 }
 
@@ -233,8 +270,9 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
 	}
 
 	cfg := authenticatorfactory.DelegatingAuthenticatorConfig{
-		Anonymous: true,
-		CacheTTL:  s.CacheTTL,
+		Anonymous:           true,
+		CacheTTL:            s.CacheTTL,
+		WebhookRetryBackoff: s.WebhookRetryBackoff,
 	}
 
 	client, err := s.getClient()
@@ -377,6 +415,7 @@ func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, err
 	// set high qps/burst limits since this will effectively limit API server responsiveness
 	clientConfig.QPS = 200
 	clientConfig.Burst = 400
+	clientConfig.Timeout = s.ClientTimeout
 
 	return kubernetes.NewForConfig(clientConfig)
 }

vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -23,6 +23,7 @@ import (
 	"github.com/spf13/pflag"
 	"k8s.io/klog/v2"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/authorization/authorizer"
 	"k8s.io/apiserver/pkg/authorization/authorizerfactory"
 	"k8s.io/apiserver/pkg/authorization/path"
@@ -59,13 +60,24 @@ type DelegatingAuthorizationOptions struct {
 
 	// AlwaysAllowGroups are groups which are allowed to take any actions.  In kube, this is system:masters.
 	AlwaysAllowGroups []string
+
+	// ClientTimeout specifies a time limit for requests made by SubjectAccessReviews client.
+	// The default value is set to 10 seconds.
+	ClientTimeout time.Duration
+
+	// WebhookRetryBackoff specifies the backoff parameters for the authorization webhook retry logic.
+	// This allows us to configure the sleep time at each iteration and the maximum number of retries allowed
+	// before we fail the webhook call in order to limit the fan out that ensues when the system is degraded.
+	WebhookRetryBackoff *wait.Backoff
 }
 
 func NewDelegatingAuthorizationOptions() *DelegatingAuthorizationOptions {
 	return &DelegatingAuthorizationOptions{
 		// very low for responsiveness, but high enough to handle storms
-		AllowCacheTTL: 10 * time.Second,
-		DenyCacheTTL:  10 * time.Second,
+		AllowCacheTTL:       10 * time.Second,
+		DenyCacheTTL:        10 * time.Second,
+		ClientTimeout:       10 * time.Second,
+		WebhookRetryBackoff: DefaultAuthWebhookRetryBackoff(),
 	}
 }
 
@@ -81,8 +93,23 @@ func (s *DelegatingAuthorizationOptions) WithAlwaysAllowPaths(paths ...string) *
 	return s
 }
 
+// WithClientTimeout sets the given timeout for SAR client used by this authorizer
+func (s *DelegatingAuthorizationOptions) WithClientTimeout(timeout time.Duration) {
+	s.ClientTimeout = timeout
+}
+
+// WithCustomRetryBackoff sets the custom backoff parameters for the authorization webhook retry logic.
+func (s *DelegatingAuthorizationOptions) WithCustomRetryBackoff(backoff wait.Backoff) {
+	s.WebhookRetryBackoff = &backoff
+}
+
 func (s *DelegatingAuthorizationOptions) Validate() []error {
 	allErrors := []error{}
+
+	if s.WebhookRetryBackoff != nil && s.WebhookRetryBackoff.Steps <= 0 {
+		allErrors = append(allErrors, fmt.Errorf("number of webhook retry attempts must be greater than 1, but is: %d", s.WebhookRetryBackoff.Steps))
+	}
+
 	return allErrors
 }
 
@@ -149,6 +176,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac
 			SubjectAccessReviewClient: client.AuthorizationV1().SubjectAccessReviews(),
 			AllowCacheTTL:             s.AllowCacheTTL,
 			DenyCacheTTL:              s.DenyCacheTTL,
+			WebhookRetryBackoff:       s.WebhookRetryBackoff,
 		}
 		delegatedAuthorizer, err := cfg.New()
 		if err != nil {
@@ -186,6 +214,7 @@ func (s *DelegatingAuthorizationOptions) getClient() (kubernetes.Interface, erro
 	// set high qps/burst limits since this will effectively limit API server responsiveness
 	clientConfig.QPS = 200
 	clientConfig.Burst = 400
+	clientConfig.Timeout = s.ClientTimeout
 
 	return kubernetes.NewForConfig(clientConfig)
 }

vendor/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -18,6 +18,7 @@ package options
 
 import (
 	"fmt"
+
 	"github.com/spf13/pflag"
 	"k8s.io/utils/path"
 
@@ -84,7 +85,7 @@ func (o *EgressSelectorOptions) Validate() []error {
 
 	errs := []error{}
 
-	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil {
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); !exists || err != nil {
 		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
 	}
 

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -180,6 +180,9 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 
 	fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
 		"The interval of requests to poll etcd and update metric. 0 disables the metric collection")
+
+	fs.DurationVar(&s.StorageConfig.HealthcheckTimeout, "etcd-healthcheck-timeout", s.StorageConfig.HealthcheckTimeout,
+		"The timeout to use when checking etcd health.")
 }
 
 func (s *EtcdOptions) ApplyTo(c *server.Config) error {

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -126,7 +126,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
 		config.FlowControl = utilflowcontrol.New(
 			config.SharedInformerFactory,
-			kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1alpha1(),
+			kubernetes.NewForConfigOrDie(config.ClientConfig).FlowcontrolV1beta1(),
 			config.MaxRequestsInFlight+config.MaxMutatingRequestsInFlight,
 			config.RequestTimeout/4,
 		)

vendor/k8s.io/apiserver/pkg/server/routes/openapi.go

@@ -38,7 +38,7 @@ func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) (*hand
 	if err != nil {
 		klog.Fatalf("Failed to build open api spec for root: %v", err)
 	}
-
+	spec.Definitions = handler.PruneDefaults(spec.Definitions)
 	openAPIVersionedService, err := handler.NewOpenAPIService(spec)
 	if err != nil {
 		klog.Fatalf("Failed to create OpenAPIService: %v", err)

vendor/k8s.io/apiserver/pkg/server/storage/resource_config.go

@@ -85,11 +85,7 @@ func (o *ResourceConfig) EnableVersions(versions ...schema.GroupVersion) {
 
 func (o *ResourceConfig) VersionEnabled(version schema.GroupVersion) bool {
 	enabled, _ := o.GroupVersionConfigs[version]
-	if enabled {
-		return true
-	}
-
-	return false
+	return enabled
 }
 
 func (o *ResourceConfig) DisableResources(resources ...schema.GroupVersionResource) {

vendor/k8s.io/apiserver/pkg/storage/OWNERS

@@ -23,6 +23,4 @@ reviewers:
 - ingvagabund
 - resouer
 - mbohlool
-- mqliang
-- rrati
 - enj

vendor/k8s.io/apiserver/pkg/storage/cacher/cacher.go

@@ -158,8 +158,10 @@ func (i *indexedWatchers) terminateAll(objectType reflect.Type, done func(*cache
 // second in a bucket, and pop up them once at the timeout. To be more specific,
 // if you set fire time at X, you can get the bookmark within (X-1,X+1) period.
 type watcherBookmarkTimeBuckets struct {
-	lock              sync.Mutex
+	lock sync.Mutex
+	// the key of watcherBuckets is the number of seconds since createTime
 	watchersBuckets   map[int64][]*cacheWatcher
+	createTime        time.Time
 	startBucketID     int64
 	clock             clock.Clock
 	bookmarkFrequency time.Duration
@@ -168,7 +170,8 @@ type watcherBookmarkTimeBuckets struct {
 func newTimeBucketWatchers(clock clock.Clock, bookmarkFrequency time.Duration) *watcherBookmarkTimeBuckets {
 	return &watcherBookmarkTimeBuckets{
 		watchersBuckets:   make(map[int64][]*cacheWatcher),
-		startBucketID:     clock.Now().Unix(),
+		createTime:        clock.Now(),
+		startBucketID:     0,
 		clock:             clock,
 		bookmarkFrequency: bookmarkFrequency,
 	}
@@ -181,7 +184,7 @@ func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
 	if !ok {
 		return false
 	}
-	bucketID := nextTime.Unix()
+	bucketID := int64(nextTime.Sub(t.createTime) / time.Second)
 	t.lock.Lock()
 	defer t.lock.Unlock()
 	if bucketID < t.startBucketID {
@@ -193,7 +196,7 @@ func (t *watcherBookmarkTimeBuckets) addWatcher(w *cacheWatcher) bool {
 }
 
 func (t *watcherBookmarkTimeBuckets) popExpiredWatchers() [][]*cacheWatcher {
-	currentBucketID := t.clock.Now().Unix()
+	currentBucketID := int64(t.clock.Since(t.createTime) / time.Second)
 	// There should be one or two elements in almost all cases
 	expiredWatchers := make([][]*cacheWatcher, 0, 2)
 	t.lock.Lock()
@@ -263,7 +266,7 @@ type Cacher struct {
 
 	// Defines a time budget that can be spend on waiting for not-ready watchers
 	// while dispatching event before shutting them down.
-	dispatchTimeoutBudget *timeBudget
+	dispatchTimeoutBudget timeBudget
 
 	// Handling graceful termination.
 	stopLock sync.RWMutex
@@ -730,7 +733,7 @@ func (c *Cacher) List(ctx context.Context, key string, opts storage.ListOptions,
 // GuaranteedUpdate implements storage.Interface.
 func (c *Cacher) GuaranteedUpdate(
 	ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
-	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, _ ...runtime.Object) error {
+	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, _ runtime.Object) error {
 	// Ignore the suggestion and try to pass down the current version of the object
 	// read from cache.
 	if elem, exists, err := c.watchCache.GetByKey(key); err != nil {
@@ -740,7 +743,7 @@ func (c *Cacher) GuaranteedUpdate(
 		return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, currObj)
 	}
 	// If we couldn't get the object, fallback to no-suggestion.
-	return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate)
+	return c.storage.GuaranteedUpdate(ctx, key, ptrToType, ignoreNotFound, preconditions, tryUpdate, nil)
 }
 
 // Count implements storage.Interface.
@@ -793,7 +796,19 @@ func (c *Cacher) dispatchEvents() {
 			if !ok {
 				return
 			}
-			c.dispatchEvent(&event)
+			// Don't dispatch bookmarks coming from the storage layer.
+			// They can be very frequent (even to the level of subseconds)
+			// to allow efficient watch resumption on kube-apiserver restarts,
+			// and propagating them down may overload the whole system.
+			//
+			// TODO: If at some point we decide the performance and scalability
+			// footprint is acceptable, this is the place to hook them in.
+			// However, we then need to check if this was called as a result
+			// of a bookmark event or regular Add/Update/Delete operation by
+			// checking if resourceVersion here has changed.
+			if event.Type != watch.Bookmark {
+				c.dispatchEvent(&event)
+			}
 			lastProcessedResourceVersion = event.ResourceVersion
 		case <-bookmarkTimer.C():
 			bookmarkTimer.Reset(wait.Jitter(time.Second, 0.25))
@@ -1098,7 +1113,14 @@ func (lw *cacherListerWatcher) List(options metav1.ListOptions) (runtime.Object,
 
 // Implements cache.ListerWatcher interface.
 func (lw *cacherListerWatcher) Watch(options metav1.ListOptions) (watch.Interface, error) {
-	return lw.storage.WatchList(context.TODO(), lw.resourcePrefix, storage.ListOptions{ResourceVersion: options.ResourceVersion, Predicate: storage.Everything})
+	opts := storage.ListOptions{
+		ResourceVersion: options.ResourceVersion,
+		Predicate:       storage.Everything,
+	}
+	if utilfeature.DefaultFeatureGate.Enabled(features.EfficientWatchResumption) {
+		opts.ProgressNotify = true
+	}
+	return lw.storage.WatchList(context.TODO(), lw.resourcePrefix, opts)
 }
 
 // errWatcher implements watch.Interface to return a single error

vendor/k8s.io/apiserver/pkg/storage/cacher/time_budget.go

@@ -39,7 +39,12 @@ const (
 // NOTE: It's not recommended to be used concurrently from multiple threads -
 // if first user takes the whole timeout, the second one will get 0 timeout
 // even though the first one may return something later.
-type timeBudget struct {
+type timeBudget interface {
+	takeAvailable() time.Duration
+	returnUnused(unused time.Duration)
+}
+
+type timeBudgetImpl struct {
 	sync.Mutex
 	budget time.Duration
 
@@ -47,8 +52,8 @@ type timeBudget struct {
 	maxBudget time.Duration
 }
 
-func newTimeBudget(stopCh <-chan struct{}) *timeBudget {
-	result := &timeBudget{
+func newTimeBudget(stopCh <-chan struct{}) timeBudget {
+	result := &timeBudgetImpl{
 		budget:    time.Duration(0),
 		refresh:   refreshPerSecond,
 		maxBudget: maxBudget,
@@ -57,7 +62,7 @@ func newTimeBudget(stopCh <-chan struct{}) *timeBudget {
 	return result
 }
 
-func (t *timeBudget) periodicallyRefresh(stopCh <-chan struct{}) {
+func (t *timeBudgetImpl) periodicallyRefresh(stopCh <-chan struct{}) {
 	ticker := time.NewTicker(time.Second)
 	defer ticker.Stop()
 	for {
@@ -74,7 +79,7 @@ func (t *timeBudget) periodicallyRefresh(stopCh <-chan struct{}) {
 	}
 }
 
-func (t *timeBudget) takeAvailable() time.Duration {
+func (t *timeBudgetImpl) takeAvailable() time.Duration {
 	t.Lock()
 	defer t.Unlock()
 	result := t.budget
@@ -82,7 +87,7 @@ func (t *timeBudget) takeAvailable() time.Duration {
 	return result
 }
 
-func (t *timeBudget) returnUnused(unused time.Duration) {
+func (t *timeBudgetImpl) returnUnused(unused time.Duration) {
 	t.Lock()
 	defer t.Unlock()
 	if unused < 0 {

vendor/k8s.io/apiserver/pkg/storage/cacher/watch_cache.go

@@ -320,8 +320,9 @@ func (w *watchCache) processEvent(event watch.Event, resourceVersion uint64, upd
 	}
 
 	// Avoid calling event handler under lock.
-	// This is safe as long as there is at most one call to processEvent in flight
-	// at any point in time.
+	// This is safe as long as there is at most one call to Add/Update/Delete and
+	// UpdateResourceVersion in flight at any point in time, which is true now,
+	// because reflector calls them synchronously from its main thread.
 	if w.eventHandler != nil {
 		w.eventHandler(wcEvent)
 	}
@@ -381,6 +382,32 @@ func (w *watchCache) doCacheResizeLocked(capacity int) {
 	w.capacity = capacity
 }
 
+func (w *watchCache) UpdateResourceVersion(resourceVersion string) {
+	rv, err := w.versioner.ParseResourceVersion(resourceVersion)
+	if err != nil {
+		klog.Errorf("Couldn't parse resourceVersion: %v", err)
+		return
+	}
+
+	func() {
+		w.Lock()
+		defer w.Unlock()
+		w.resourceVersion = rv
+	}()
+
+	// Avoid calling event handler under lock.
+	// This is safe as long as there is at most one call to Add/Update/Delete and
+	// UpdateResourceVersion in flight at any point in time, which is true now,
+	// because reflector calls them synchronously from its main thread.
+	if w.eventHandler != nil {
+		wcEvent := &watchCacheEvent{
+			Type:            watch.Bookmark,
+			ResourceVersion: rv,
+		}
+		w.eventHandler(wcEvent)
+	}
+}
+
 // List returns list of pointers to <storeElement> objects.
 func (w *watchCache) List() []interface{} {
 	return w.store.List()

vendor/k8s.io/apiserver/pkg/storage/etcd3/event.go

@@ -23,12 +23,13 @@ import (
 )
 
 type event struct {
-	key       string
-	value     []byte
-	prevValue []byte
-	rev       int64
-	isDeleted bool
-	isCreated bool
+	key              string
+	value            []byte
+	prevValue        []byte
+	rev              int64
+	isDeleted        bool
+	isCreated        bool
+	isProgressNotify bool
 }
 
 // parseKV converts a KeyValue retrieved from an initial sync() listing to a synthetic isCreated event.
@@ -61,3 +62,10 @@ func parseEvent(e *clientv3.Event) (*event, error) {
 	}
 	return ret, nil
 }
+
+func progressNotifyEvent(rev int64) *event {
+	return &event{
+		rev:              rev,
+		isProgressNotify: true,
+	}
+}

vendor/k8s.io/apiserver/pkg/storage/etcd3/metrics/metrics.go

@@ -35,8 +35,12 @@ import (
 var (
 	etcdRequestLatency = compbasemetrics.NewHistogramVec(
 		&compbasemetrics.HistogramOpts{
-			Name:           "etcd_request_duration_seconds",
-			Help:           "Etcd request latency in seconds for each operation and object type.",
+			Name: "etcd_request_duration_seconds",
+			Help: "Etcd request latency in seconds for each operation and object type.",
+			// Keeping it similar to the buckets used by the apiserver_request_duration_seconds metric so that
+			// api latency and etcd latency can be more comparable side by side.
+			Buckets: []float64{.005, .01, .025, 0.05, 0.1, 0.15, 0.2, 0.25, 0.3, 0.35, 0.4, 0.45, 0.5, 0.6, 0.7,
+				0.8, 0.9, 1.0, 1.25, 1.5, 1.75, 2.0, 2.5, 3.0, 3.5, 4.0, 4.5, 5, 6, 7, 8, 9, 10, 15, 20, 25, 30, 40, 50, 60},
 			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{"operation", "type"},
@@ -57,6 +61,14 @@ var (
 		},
 		[]string{"endpoint"},
 	)
+	etcdBookmarkCounts = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Name:           "etcd_bookmark_counts",
+			Help:           "Number of etcd bookmarks (progress notify events) split by kind.",
+			StabilityLevel: compbasemetrics.ALPHA,
+		},
+		[]string{"resource"},
+	)
 )
 
 var registerMetrics sync.Once
@@ -68,6 +80,7 @@ func Register() {
 		legacyregistry.MustRegister(etcdRequestLatency)
 		legacyregistry.MustRegister(objectCounts)
 		legacyregistry.MustRegister(dbTotalSize)
+		legacyregistry.MustRegister(etcdBookmarkCounts)
 	})
 }
 
@@ -81,6 +94,11 @@ func RecordEtcdRequestLatency(verb, resource string, startTime time.Time) {
 	etcdRequestLatency.WithLabelValues(verb, resource).Observe(sinceInSeconds(startTime))
 }
 
+// RecordEtcdBookmark updates the etcd_bookmark_counts metric.
+func RecordEtcdBookmark(resource string) {
+	etcdBookmarkCounts.WithLabelValues(resource).Inc()
+}
+
 // Reset resets the etcd_request_duration_seconds metric.
 func Reset() {
 	etcdRequestLatency.Reset()

vendor/k8s.io/apiserver/pkg/storage/etcd3/store.go

@@ -83,11 +83,11 @@ type objState struct {
 }
 
 // New returns an etcd3 implementation of storage.Interface.
-func New(c *clientv3.Client, codec runtime.Codec, prefix string, transformer value.Transformer, pagingEnabled bool) storage.Interface {
-	return newStore(c, pagingEnabled, codec, prefix, transformer)
+func New(c *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, prefix string, transformer value.Transformer, pagingEnabled bool) storage.Interface {
+	return newStore(c, newFunc, pagingEnabled, codec, prefix, transformer)
 }
 
-func newStore(c *clientv3.Client, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
+func newStore(c *clientv3.Client, newFunc func() runtime.Object, pagingEnabled bool, codec runtime.Codec, prefix string, transformer value.Transformer) *store {
 	versioner := APIObjectVersioner{}
 	result := &store{
 		client:        c,
@@ -99,7 +99,7 @@ func newStore(c *clientv3.Client, pagingEnabled bool, codec runtime.Codec, prefi
 		// no-op for default prefix of '/registry'.
 		// keeps compatibility with etcd2 impl for custom prefixes that don't start with '/'
 		pathPrefix:   path.Join("/", prefix),
-		watcher:      newWatcher(c, codec, versioner, transformer),
+		watcher:      newWatcher(c, codec, newFunc, versioner, transformer),
 		leaseManager: newDefaultLeaseManager(c),
 	}
 	return result
@@ -238,7 +238,7 @@ func (s *store) conditionalDelete(ctx context.Context, key string, out runtime.O
 // GuaranteedUpdate implements storage.Interface.GuaranteedUpdate.
 func (s *store) GuaranteedUpdate(
 	ctx context.Context, key string, out runtime.Object, ignoreNotFound bool,
-	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion ...runtime.Object) error {
+	preconditions *storage.Preconditions, tryUpdate storage.UpdateFunc, suggestion runtime.Object) error {
 	trace := utiltrace.New("GuaranteedUpdate etcd3", utiltrace.Field{"type", getTypeName(out)})
 	defer trace.LogIfLong(500 * time.Millisecond)
 
@@ -260,8 +260,8 @@ func (s *store) GuaranteedUpdate(
 
 	var origState *objState
 	var mustCheckData bool
-	if len(suggestion) == 1 && suggestion[0] != nil {
-		origState, err = s.getStateFromObject(suggestion[0])
+	if suggestion != nil {
+		origState, err = s.getStateFromObject(suggestion)
 		if err != nil {
 			return err
 		}
@@ -784,7 +784,7 @@ func (s *store) watch(ctx context.Context, key string, opts storage.ListOptions,
 		return nil, err
 	}
 	key = path.Join(s.pathPrefix, key)
-	return s.watcher.Watch(ctx, key, int64(rev), recursive, opts.Predicate)
+	return s.watcher.Watch(ctx, key, int64(rev), recursive, opts.ProgressNotify, opts.Predicate)
 }
 
 func (s *store) getState(getResp *clientv3.GetResponse, key string, v reflect.Value, ignoreNotFound bool) (*objState, error) {

vendor/k8s.io/apiserver/pkg/storage/etcd3/watcher.go

@@ -21,6 +21,7 @@ import (
 	"errors"
 	"fmt"
 	"os"
+	"reflect"
 	"strconv"
 	"strings"
 	"sync"
@@ -29,6 +30,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
 	"k8s.io/apiserver/pkg/storage"
+	"k8s.io/apiserver/pkg/storage/etcd3/metrics"
 	"k8s.io/apiserver/pkg/storage/value"
 
 	"go.etcd.io/etcd/clientv3"
@@ -68,6 +70,8 @@ func TestOnlySetFatalOnDecodeError(b bool) {
 type watcher struct {
 	client      *clientv3.Client
 	codec       runtime.Codec
+	newFunc     func() runtime.Object
+	objectType  string
 	versioner   storage.Versioner
 	transformer value.Transformer
 }
@@ -78,6 +82,7 @@ type watchChan struct {
 	key               string
 	initialRev        int64
 	recursive         bool
+	progressNotify    bool
 	internalPred      storage.SelectionPredicate
 	ctx               context.Context
 	cancel            context.CancelFunc
@@ -86,13 +91,20 @@ type watchChan struct {
 	errChan           chan error
 }
 
-func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.Versioner, transformer value.Transformer) *watcher {
-	return &watcher{
+func newWatcher(client *clientv3.Client, codec runtime.Codec, newFunc func() runtime.Object, versioner storage.Versioner, transformer value.Transformer) *watcher {
+	res := &watcher{
 		client:      client,
 		codec:       codec,
+		newFunc:     newFunc,
 		versioner:   versioner,
 		transformer: transformer,
 	}
+	if newFunc == nil {
+		res.objectType = "<unknown>"
+	} else {
+		res.objectType = reflect.TypeOf(newFunc()).String()
+	}
+	return res
 }
 
 // Watch watches on a key and returns a watch.Interface that transfers relevant notifications.
@@ -102,21 +114,22 @@ func newWatcher(client *clientv3.Client, codec runtime.Codec, versioner storage.
 // If recursive is false, it watches on given key.
 // If recursive is true, it watches any children and directories under the key, excluding the root key itself.
 // pred must be non-nil. Only if pred matches the change, it will be returned.
-func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) (watch.Interface, error) {
+func (w *watcher) Watch(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) (watch.Interface, error) {
 	if recursive && !strings.HasSuffix(key, "/") {
 		key += "/"
 	}
-	wc := w.createWatchChan(ctx, key, rev, recursive, pred)
+	wc := w.createWatchChan(ctx, key, rev, recursive, progressNotify, pred)
 	go wc.run()
 	return wc, nil
 }
 
-func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive bool, pred storage.SelectionPredicate) *watchChan {
+func (w *watcher) createWatchChan(ctx context.Context, key string, rev int64, recursive, progressNotify bool, pred storage.SelectionPredicate) *watchChan {
 	wc := &watchChan{
 		watcher:           w,
 		key:               key,
 		initialRev:        rev,
 		recursive:         recursive,
+		progressNotify:    progressNotify,
 		internalPred:      pred,
 		incomingEventChan: make(chan *event, incomingBufSize),
 		resultChan:        make(chan watch.Event, outgoingBufSize),
@@ -223,6 +236,9 @@ func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
 	if wc.recursive {
 		opts = append(opts, clientv3.WithPrefix())
 	}
+	if wc.progressNotify {
+		opts = append(opts, clientv3.WithProgressNotify())
+	}
 	wch := wc.watcher.client.Watch(wc.ctx, wc.key, opts...)
 	for wres := range wch {
 		if wres.Err() != nil {
@@ -232,6 +248,12 @@ func (wc *watchChan) startWatching(watchClosedCh chan struct{}) {
 			wc.sendError(err)
 			return
 		}
+		if wres.IsProgressNotify() {
+			wc.sendEvent(progressNotifyEvent(wres.Header.GetRevision()))
+			metrics.RecordEtcdBookmark(wc.watcher.objectType)
+			continue
+		}
+
 		for _, e := range wres.Events {
 			parsedEvent, err := parseEvent(e)
 			if err != nil {
@@ -299,6 +321,19 @@ func (wc *watchChan) transform(e *event) (res *watch.Event) {
 	}
 
 	switch {
+	case e.isProgressNotify:
+		if wc.watcher.newFunc == nil {
+			return nil
+		}
+		object := wc.watcher.newFunc()
+		if err := wc.watcher.versioner.UpdateObject(object, uint64(e.rev)); err != nil {
+			klog.Errorf("failed to propagate object version: %v", err)
+			return nil
+		}
+		res = &watch.Event{
+			Type:   watch.Bookmark,
+			Object: object,
+		}
 	case e.isDeleted:
 		if !wc.filter(oldObj) {
 			return nil
@@ -376,6 +411,11 @@ func (wc *watchChan) sendEvent(e *event) {
 }
 
 func (wc *watchChan) prepareObjs(e *event) (curObj runtime.Object, oldObj runtime.Object, err error) {
+	if e.isProgressNotify {
+		// progressNotify events doesn't contain neither current nor previous object version,
+		return nil, nil, nil
+	}
+
 	if !e.isDeleted {
 		data, _, err := wc.watcher.transformer.TransformFromStorage(e.value, authenticatedDataString(e.key))
 		if err != nil {

vendor/k8s.io/apiserver/pkg/storage/interfaces.go

@@ -215,9 +215,9 @@ type Interface interface {
 	// or zero value in 'ptrToType' parameter otherwise.
 	// If the object to update has the same value as previous, it won't do any update
 	// but will return the object in 'ptrToType' parameter.
-	// If 'suggestion' can contain zero or one element - in such case this can be used as
-	// a suggestion about the current version of the object to avoid read operation from
-	// storage to get it.
+	// If 'suggestion' is non-nil, it can be used as a suggestion about the current version
+	// of the object to avoid read operation from storage to get it. However, the
+	// implementations have to retry in case suggestion is stale.
 	//
 	// Example:
 	//
@@ -239,7 +239,7 @@ type Interface interface {
 	// )
 	GuaranteedUpdate(
 		ctx context.Context, key string, ptrToType runtime.Object, ignoreNotFound bool,
-		precondtions *Preconditions, tryUpdate UpdateFunc, suggestion ...runtime.Object) error
+		precondtions *Preconditions, tryUpdate UpdateFunc, suggestion runtime.Object) error
 
 	// Count returns number of different entries under the key (generally being path prefix).
 	Count(key string) (int64, error)
@@ -269,4 +269,7 @@ type ListOptions struct {
 	ResourceVersionMatch metav1.ResourceVersionMatch
 	// Predicate provides the selection rules for the list operation.
 	Predicate SelectionPredicate
+	// ProgressNotify determines whether storage-originated bookmark (progress notify) events should
+	// be delivered to the users. The option is ignored for non-watch requests.
+	ProgressNotify bool
 }

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -30,6 +30,7 @@ const (
 
 	DefaultCompactInterval      = 5 * time.Minute
 	DefaultDBMetricPollInterval = 30 * time.Second
+	DefaultHealthcheckTimeout   = 2 * time.Second
 )
 
 // TransportConfig holds all connection related info,  i.e. equal TransportConfig means equal servers we talk to.
@@ -74,6 +75,8 @@ type Config struct {
 	CountMetricPollPeriod time.Duration
 	// DBMetricPollInterval specifies how often should storage backend metric be updated.
 	DBMetricPollInterval time.Duration
+	// HealthcheckTimeout specifies the timeout used when checking health
+	HealthcheckTimeout time.Duration
 }
 
 func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
@@ -83,5 +86,6 @@ func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
 		Codec:                codec,
 		CompactionInterval:   DefaultCompactInterval,
 		DBMetricPollInterval: DefaultDBMetricPollInterval,
+		HealthcheckTimeout:   DefaultHealthcheckTimeout,
 	}
 }

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -31,6 +31,7 @@ import (
 	"go.etcd.io/etcd/pkg/transport"
 	"google.golang.org/grpc"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/server/egressselector"
@@ -91,7 +92,11 @@ func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
 			return fmt.Errorf(errMsg)
 		}
 		client := clientValue.Load().(*clientv3.Client)
-		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		healthcheckTimeout := storagebackend.DefaultHealthcheckTimeout
+		if c.HealthcheckTimeout != time.Duration(0) {
+			healthcheckTimeout = c.HealthcheckTimeout
+		}
+		ctx, cancel := context.WithTimeout(context.Background(), healthcheckTimeout)
 		defer cancel()
 		// See https://github.com/etcd-io/etcd/blob/c57f8b3af865d1b531b979889c602ba14377420e/etcdctl/ctlv3/command/ep_command.go#L118
 		_, err := client.Get(ctx, path.Join("/", c.Prefix, "health"))
@@ -217,7 +222,7 @@ func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration
 	}, nil
 }
 
-func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+func newETCD3Storage(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
 	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
 	if err != nil {
 		return nil, nil, err
@@ -249,7 +254,7 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 	if transformer == nil {
 		transformer = value.IdentityTransformer
 	}
-	return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
+	return etcd3.New(client, c.Codec, newFunc, c.Prefix, transformer, c.Paging), destroyFunc, nil
 }
 
 // startDBSizeMonitorPerEndpoint starts a loop to monitor etcd database size and update the

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go

@@ -19,6 +19,7 @@ package factory
 import (
 	"fmt"
 
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 )
@@ -27,12 +28,12 @@ import (
 type DestroyFunc func()
 
 // Create creates a storage backend based on given config.
-func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+func Create(c storagebackend.Config, newFunc func() runtime.Object) (storage.Interface, DestroyFunc, error) {
 	switch c.Type {
 	case "etcd2":
 		return nil, nil, fmt.Errorf("%v is no longer a supported storage backend", c.Type)
 	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
-		return newETCD3Storage(c)
+		return newETCD3Storage(c, newFunc)
 	default:
 		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
 	}

vendor/k8s.io/apiserver/pkg/storageversion/OWNERS

@@ -0,0 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- caesarxuchao
+- roycaihw

vendor/k8s.io/apiserver/pkg/storageversion/manager.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storageversion
+
+import (
+	"fmt"
+	"sort"
+	"sync"
+	"sync/atomic"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/client-go/kubernetes"
+	"k8s.io/client-go/rest"
+	_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
+	"k8s.io/klog/v2"
+)
+
+// ResourceInfo contains the information to register the resource to the
+// storage version API.
+type ResourceInfo struct {
+	GroupResource schema.GroupResource
+
+	EncodingVersion string
+	// Used to calculate decodable versions. Can only be used after all
+	// equivalent versions are registered by InstallREST.
+	EquivalentResourceMapper runtime.EquivalentResourceRegistry
+}
+
+// Manager records the resources whose StorageVersions need updates, and provides a method to update those StorageVersions.
+type Manager interface {
+	// AddResourceInfo records resources whose StorageVersions need updates
+	AddResourceInfo(resources ...*ResourceInfo)
+	// UpdateStorageVersions tries to update the StorageVersions of the recorded resources
+	UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, apiserverID string)
+	// PendingUpdate returns true if the StorageVersion of the given resource is still pending update.
+	PendingUpdate(gr schema.GroupResource) bool
+	// LastUpdateError returns the last error hit when updating the storage version of the given resource.
+	LastUpdateError(gr schema.GroupResource) error
+	// Completed returns true if updating StorageVersions of all recorded resources has completed.
+	Completed() bool
+}
+
+var _ Manager = &defaultManager{}
+
+// defaultManager indicates if an apiserver has completed reporting its storage versions.
+type defaultManager struct {
+	completed atomic.Value
+
+	mu sync.RWMutex
+	// managedResourceInfos records the ResourceInfos whose StorageVersions will get updated in the next
+	// UpdateStorageVersions call
+	managedResourceInfos map[*ResourceInfo]struct{}
+	// managedStatus records the update status of StorageVersion for each GroupResource. Since one
+	// ResourceInfo may expand into multiple GroupResource (e.g. ingresses.networking.k8s.io and ingresses.extensions),
+	// this map allows quick status lookup for a GroupResource, during API request handling.
+	managedStatus map[schema.GroupResource]*updateStatus
+}
+
+type updateStatus struct {
+	done    bool
+	lastErr error
+}
+
+// NewDefaultManager creates a new defaultManager.
+func NewDefaultManager() Manager {
+	s := &defaultManager{}
+	s.completed.Store(false)
+	s.managedResourceInfos = make(map[*ResourceInfo]struct{})
+	s.managedStatus = make(map[schema.GroupResource]*updateStatus)
+	return s
+}
+
+// AddResourceInfo adds ResourceInfo to the manager.
+func (s *defaultManager) AddResourceInfo(resources ...*ResourceInfo) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	for _, r := range resources {
+		s.managedResourceInfos[r] = struct{}{}
+		s.addPendingManagedStatusLocked(r)
+	}
+}
+
+func (s *defaultManager) addPendingManagedStatusLocked(r *ResourceInfo) {
+	gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
+	for _, gvr := range gvrs {
+		gr := gvr.GroupResource()
+		if _, ok := s.managedStatus[gr]; !ok {
+			s.managedStatus[gr] = &updateStatus{}
+		}
+	}
+}
+
+// UpdateStorageVersions tries to update the StorageVersions of the recorded resources
+func (s *defaultManager) UpdateStorageVersions(kubeAPIServerClientConfig *rest.Config, serverID string) {
+	clientset, err := kubernetes.NewForConfig(kubeAPIServerClientConfig)
+	if err != nil {
+		utilruntime.HandleError(fmt.Errorf("failed to get clientset: %v", err))
+		return
+	}
+	sc := clientset.InternalV1alpha1().StorageVersions()
+
+	s.mu.RLock()
+	resources := []ResourceInfo{}
+	for resource := range s.managedResourceInfos {
+		resources = append(resources, *resource)
+	}
+	s.mu.RUnlock()
+	hasFailure := false
+	// Sorting the list to make sure we have a consistent dedup result, and
+	// therefore avoid creating unnecessarily duplicated StorageVersion objects.
+	// For example, extensions.ingresses and networking.k8s.io.ingresses share
+	// the same underlying storage. Without sorting, in an HA cluster, one
+	// apiserver may dedup and update StorageVersion for extensions.ingresses,
+	// while another apiserver may dedup and update StorageVersion for
+	// networking.k8s.io.ingresses. The storage migrator (which migrates objects
+	// per GroupResource) will migrate these resources twice, since both
+	// StorageVersion objects have CommonEncodingVersion (each with one server registered).
+	sortResourceInfosByGroupResource(resources)
+	for _, r := range dedupResourceInfos(resources) {
+		dv := decodableVersions(r.EquivalentResourceMapper, r.GroupResource)
+		gr := r.GroupResource
+		// Group must be a valid subdomain in DNS (RFC 1123)
+		if len(gr.Group) == 0 {
+			gr.Group = "core"
+		}
+		if err := updateStorageVersionFor(sc, serverID, gr, r.EncodingVersion, dv); err != nil {
+			utilruntime.HandleError(fmt.Errorf("failed to update storage version for %v: %v", r.GroupResource, err))
+			s.recordStatusFailure(&r, err)
+			hasFailure = true
+			continue
+		}
+		klog.V(2).Infof("successfully updated storage version for %v", r.GroupResource)
+		s.recordStatusSuccess(&r)
+	}
+	if hasFailure {
+		return
+	}
+	klog.V(2).Infof("storage version updates complete")
+	s.setComplete()
+}
+
+// dedupResourceInfos dedups ResourceInfos with the same underlying storage.
+// ResourceInfos from the same Group with different Versions share the same underlying storage.
+// ResourceInfos from different Groups may share the same underlying storage, e.g.
+// networking.k8s.io ingresses and extensions ingresses. The StorageVersion manager
+// only needs to update one StorageVersion for the equivalent Groups.
+func dedupResourceInfos(infos []ResourceInfo) []ResourceInfo {
+	var ret []ResourceInfo
+	seen := make(map[schema.GroupResource]struct{})
+	for _, info := range infos {
+		gr := info.GroupResource
+		if _, ok := seen[gr]; ok {
+			continue
+		}
+		gvrs := info.EquivalentResourceMapper.EquivalentResourcesFor(gr.WithVersion(""), "")
+		for _, gvr := range gvrs {
+			seen[gvr.GroupResource()] = struct{}{}
+		}
+		ret = append(ret, info)
+	}
+	return ret
+}
+
+func sortResourceInfosByGroupResource(infos []ResourceInfo) {
+	sort.Sort(byGroupResource(infos))
+}
+
+type byGroupResource []ResourceInfo
+
+func (s byGroupResource) Len() int { return len(s) }
+
+func (s byGroupResource) Less(i, j int) bool {
+	if s[i].GroupResource.Group == s[j].GroupResource.Group {
+		return s[i].GroupResource.Resource < s[j].GroupResource.Resource
+	}
+	return s[i].GroupResource.Group < s[j].GroupResource.Group
+}
+
+func (s byGroupResource) Swap(i, j int) { s[i], s[j] = s[j], s[i] }
+
+// recordStatusSuccess marks updated ResourceInfo as completed.
+func (s *defaultManager) recordStatusSuccess(r *ResourceInfo) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.recordStatusSuccessLocked(r)
+}
+
+func (s *defaultManager) recordStatusSuccessLocked(r *ResourceInfo) {
+	gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
+	for _, gvr := range gvrs {
+		s.recordSuccessGroupResourceLocked(gvr.GroupResource())
+	}
+}
+
+func (s *defaultManager) recordSuccessGroupResourceLocked(gr schema.GroupResource) {
+	if _, ok := s.managedStatus[gr]; !ok {
+		return
+	}
+	s.managedStatus[gr].done = true
+	s.managedStatus[gr].lastErr = nil
+}
+
+// recordStatusFailure records latest error updating ResourceInfo.
+func (s *defaultManager) recordStatusFailure(r *ResourceInfo, err error) {
+	s.mu.Lock()
+	defer s.mu.Unlock()
+	s.recordStatusFailureLocked(r, err)
+}
+
+func (s *defaultManager) recordStatusFailureLocked(r *ResourceInfo, err error) {
+	gvrs := r.EquivalentResourceMapper.EquivalentResourcesFor(r.GroupResource.WithVersion(""), "")
+	for _, gvr := range gvrs {
+		s.recordErrorGroupResourceLocked(gvr.GroupResource(), err)
+	}
+}
+
+func (s *defaultManager) recordErrorGroupResourceLocked(gr schema.GroupResource, err error) {
+	if _, ok := s.managedStatus[gr]; !ok {
+		return
+	}
+	s.managedStatus[gr].lastErr = err
+}
+
+// PendingUpdate returns if the StorageVersion of a resource is still wait to be updated.
+func (s *defaultManager) PendingUpdate(gr schema.GroupResource) bool {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if _, ok := s.managedStatus[gr]; !ok {
+		return false
+	}
+	return !s.managedStatus[gr].done
+}
+
+// LastUpdateError returns the last error hit when updating the storage version of the given resource.
+func (s *defaultManager) LastUpdateError(gr schema.GroupResource) error {
+	s.mu.RLock()
+	defer s.mu.RUnlock()
+	if _, ok := s.managedStatus[gr]; !ok {
+		return fmt.Errorf("couldn't find managed status for %v", gr)
+	}
+	return s.managedStatus[gr].lastErr
+}
+
+// setComplete marks the completion of updating StorageVersions. No write requests need to be blocked anymore.
+func (s *defaultManager) setComplete() {
+	s.completed.Store(true)
+}
+
+// Completed returns if updating StorageVersions has completed.
+func (s *defaultManager) Completed() bool {
+	return s.completed.Load().(bool)
+}
+
+func decodableVersions(e runtime.EquivalentResourceRegistry, gr schema.GroupResource) []string {
+	var versions []string
+	decodingGVRs := e.EquivalentResourcesFor(gr.WithVersion(""), "")
+	for _, v := range decodingGVRs {
+		versions = append(versions, v.GroupVersion().String())
+	}
+	return versions
+}

vendor/k8s.io/apiserver/pkg/storageversion/updater.go

@@ -0,0 +1,128 @@
+/*
+Copyright 2020 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package storageversion
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"k8s.io/api/apiserverinternal/v1alpha1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/klog/v2"
+)
+
+// Client has the methods required to update the storage version.
+type Client interface {
+	Create(context.Context, *v1alpha1.StorageVersion, metav1.CreateOptions) (*v1alpha1.StorageVersion, error)
+	UpdateStatus(context.Context, *v1alpha1.StorageVersion, metav1.UpdateOptions) (*v1alpha1.StorageVersion, error)
+	Get(context.Context, string, metav1.GetOptions) (*v1alpha1.StorageVersion, error)
+}
+
+func setCommonEncodingVersion(sv *v1alpha1.StorageVersion) {
+	if len(sv.Status.StorageVersions) == 0 {
+		return
+	}
+	firstVersion := sv.Status.StorageVersions[0].EncodingVersion
+	agreed := true
+	for _, ssv := range sv.Status.StorageVersions {
+		if ssv.EncodingVersion != firstVersion {
+			agreed = false
+			break
+		}
+	}
+	if agreed {
+		sv.Status.CommonEncodingVersion = &firstVersion
+	} else {
+		sv.Status.CommonEncodingVersion = nil
+	}
+}
+
+// updateStorageVersionFor updates the storage version object for the resource.
+func updateStorageVersionFor(c Client, apiserverID string, gr schema.GroupResource, encodingVersion string, decodableVersions []string) error {
+	retries := 3
+	var retry int
+	var err error
+	for retry < retries {
+		err = singleUpdate(c, apiserverID, gr, encodingVersion, decodableVersions)
+		if err == nil {
+			return nil
+		}
+		if apierrors.IsAlreadyExists(err) || apierrors.IsConflict(err) {
+			time.Sleep(1 * time.Second)
+			continue
+		}
+		if err != nil {
+			klog.Errorf("retry %d, failed to update storage version for %v: %v", retry, gr, err)
+			retry++
+			time.Sleep(1 * time.Second)
+		}
+	}
+	return err
+}
+
+func singleUpdate(c Client, apiserverID string, gr schema.GroupResource, encodingVersion string, decodableVersions []string) error {
+	shouldCreate := false
+	name := fmt.Sprintf("%s.%s", gr.Group, gr.Resource)
+	sv, err := c.Get(context.TODO(), name, metav1.GetOptions{})
+	if err != nil && !apierrors.IsNotFound(err) {
+		return err
+	}
+	if apierrors.IsNotFound(err) {
+		shouldCreate = true
+		sv = &v1alpha1.StorageVersion{}
+		sv.ObjectMeta.Name = name
+	}
+	updatedSV := localUpdateStorageVersion(sv, apiserverID, encodingVersion, decodableVersions)
+	if shouldCreate {
+		createdSV, err := c.Create(context.TODO(), updatedSV, metav1.CreateOptions{})
+		if err != nil {
+			return err
+		}
+		// assign the calculated status to the object just created, then update status
+		createdSV.Status = updatedSV.Status
+		_, err = c.UpdateStatus(context.TODO(), createdSV, metav1.UpdateOptions{})
+		return err
+	}
+	_, err = c.UpdateStatus(context.TODO(), updatedSV, metav1.UpdateOptions{})
+	return err
+}
+
+// localUpdateStorageVersion updates the input storageversion with given server storageversion info.
+// The function updates the input storageversion in place.
+func localUpdateStorageVersion(sv *v1alpha1.StorageVersion, apiserverID, encodingVersion string, decodableVersions []string) *v1alpha1.StorageVersion {
+	newSSV := v1alpha1.ServerStorageVersion{
+		APIServerID:       apiserverID,
+		EncodingVersion:   encodingVersion,
+		DecodableVersions: decodableVersions,
+	}
+	foundSSV := false
+	for i, ssv := range sv.Status.StorageVersions {
+		if ssv.APIServerID == apiserverID {
+			sv.Status.StorageVersions[i] = newSSV
+			foundSSV = true
+			break
+		}
+	}
+	if !foundSSV {
+		sv.Status.StorageVersions = append(sv.Status.StorageVersions, newSSV)
+	}
+	setCommonEncodingVersion(sv)
+	return sv
+}

vendor/k8s.io/apiserver/pkg/util/apihelpers/helpers.go

@@ -19,7 +19,7 @@ package apihelpers
 import (
 	"sort"
 
-	flowcontrol "k8s.io/api/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
 )
 
 // SetFlowSchemaCondition sets conditions.

vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go

@@ -47,9 +47,9 @@ import (
 	"k8s.io/client-go/util/workqueue"
 	"k8s.io/klog/v2"
 
-	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
-	fcclientv1a1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1"
-	fclistersv1a1 "k8s.io/client-go/listers/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
+	flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1"
+	flowcontrollister "k8s.io/client-go/listers/flowcontrol/v1beta1"
 )
 
 // This file contains a simple local (to the apiserver) controller
@@ -91,13 +91,13 @@ type configController struct {
 	// objects need to be reprocessed.
 	configQueue workqueue.RateLimitingInterface
 
-	plLister         fclistersv1a1.PriorityLevelConfigurationLister
+	plLister         flowcontrollister.PriorityLevelConfigurationLister
 	plInformerSynced cache.InformerSynced
 
-	fsLister         fclistersv1a1.FlowSchemaLister
+	fsLister         flowcontrollister.FlowSchemaLister
 	fsInformerSynced cache.InformerSynced
 
-	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface
+	flowcontrolClient flowcontrolclient.FlowcontrolV1beta1Interface
 
 	// serverConcurrencyLimit is the limit on the server's total
 	// number of non-exempt requests being served at once.  This comes
@@ -127,7 +127,7 @@ type configController struct {
 type priorityLevelState struct {
 	// the API object or prototype prescribing this level.  Nothing
 	// reached through this pointer is mutable.
-	pl *fctypesv1a1.PriorityLevelConfiguration
+	pl *flowcontrol.PriorityLevelConfiguration
 
 	// qsCompleter holds the QueueSetCompleter derived from `config`
 	// and `queues` if config is not exempt, nil otherwise.
@@ -153,7 +153,7 @@ type priorityLevelState struct {
 // NewTestableController is extra flexible to facilitate testing
 func newTestableController(
 	informerFactory kubeinformers.SharedInformerFactory,
-	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	flowcontrolClient flowcontrolclient.FlowcontrolV1beta1Interface,
 	serverConcurrencyLimit int,
 	requestWaitLimit time.Duration,
 	obsPairGenerator metrics.TimedObserverPairGenerator,
@@ -178,7 +178,7 @@ func newTestableController(
 // config API objects.
 func (cfgCtlr *configController) initializeConfigController(informerFactory kubeinformers.SharedInformerFactory) {
 	cfgCtlr.configQueue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 8*time.Hour), "priority_and_fairness_config_queue")
-	fci := informerFactory.Flowcontrol().V1alpha1()
+	fci := informerFactory.Flowcontrol().V1beta1()
 	pli := fci.PriorityLevelConfigurations()
 	fsi := fci.FlowSchemas()
 	cfgCtlr.plLister = pli.Lister()
@@ -187,13 +187,13 @@ func (cfgCtlr *configController) initializeConfigController(informerFactory kube
 	cfgCtlr.fsInformerSynced = fsi.Informer().HasSynced
 	pli.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
-			pl := obj.(*fctypesv1a1.PriorityLevelConfiguration)
+			pl := obj.(*flowcontrol.PriorityLevelConfiguration)
 			klog.V(7).Infof("Triggered API priority and fairness config reloading due to creation of PLC %s", pl.Name)
 			cfgCtlr.configQueue.Add(0)
 		},
 		UpdateFunc: func(oldObj, newObj interface{}) {
-			newPL := newObj.(*fctypesv1a1.PriorityLevelConfiguration)
-			oldPL := oldObj.(*fctypesv1a1.PriorityLevelConfiguration)
+			newPL := newObj.(*flowcontrol.PriorityLevelConfiguration)
+			oldPL := oldObj.(*flowcontrol.PriorityLevelConfiguration)
 			if !apiequality.Semantic.DeepEqual(oldPL.Spec, newPL.Spec) {
 				klog.V(7).Infof("Triggered API priority and fairness config reloading due to spec update of PLC %s", newPL.Name)
 				cfgCtlr.configQueue.Add(0)
@@ -207,13 +207,13 @@ func (cfgCtlr *configController) initializeConfigController(informerFactory kube
 		}})
 	fsi.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: func(obj interface{}) {
-			fs := obj.(*fctypesv1a1.FlowSchema)
+			fs := obj.(*flowcontrol.FlowSchema)
 			klog.V(7).Infof("Triggered API priority and fairness config reloading due to creation of FS %s", fs.Name)
 			cfgCtlr.configQueue.Add(0)
 		},
 		UpdateFunc: func(oldObj, newObj interface{}) {
-			newFS := newObj.(*fctypesv1a1.FlowSchema)
-			oldFS := oldObj.(*fctypesv1a1.FlowSchema)
+			newFS := newObj.(*flowcontrol.FlowSchema)
+			oldFS := oldObj.(*flowcontrol.FlowSchema)
 			if !apiequality.Semantic.DeepEqual(oldFS.Spec, newFS.Spec) {
 				klog.V(7).Infof("Triggered API priority and fairness config reloading due to spec update of FS %s", newFS.Name)
 				cfgCtlr.configQueue.Add(0)
@@ -332,14 +332,14 @@ type cfgMeal struct {
 
 // A buffered set of status updates for a FlowSchema
 type fsStatusUpdate struct {
-	flowSchema *fctypesv1a1.FlowSchema
-	condition  fctypesv1a1.FlowSchemaCondition
-	oldValue   fctypesv1a1.FlowSchemaCondition
+	flowSchema *flowcontrol.FlowSchema
+	condition  flowcontrol.FlowSchemaCondition
+	oldValue   flowcontrol.FlowSchemaCondition
 }
 
 // digestConfigObjects is given all the API objects that configure
 // cfgCtlr and writes its consequent new configState.
-func (cfgCtlr *configController) digestConfigObjects(newPLs []*fctypesv1a1.PriorityLevelConfiguration, newFSs []*fctypesv1a1.FlowSchema) error {
+func (cfgCtlr *configController) digestConfigObjects(newPLs []*flowcontrol.PriorityLevelConfiguration, newFSs []*flowcontrol.FlowSchema) error {
 	fsStatusUpdates := cfgCtlr.lockAndDigestConfigObjects(newPLs, newFSs)
 	var errs []error
 	for _, fsu := range fsStatusUpdates {
@@ -360,7 +360,7 @@ func (cfgCtlr *configController) digestConfigObjects(newPLs []*fctypesv1a1.Prior
 	return apierrors.NewAggregate(errs)
 }
 
-func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*fctypesv1a1.PriorityLevelConfiguration, newFSs []*fctypesv1a1.FlowSchema) []fsStatusUpdate {
+func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*flowcontrol.PriorityLevelConfiguration, newFSs []*flowcontrol.FlowSchema) []fsStatusUpdate {
 	cfgCtlr.lock.Lock()
 	defer cfgCtlr.lock.Unlock()
 	meal := cfgMeal{
@@ -390,7 +390,7 @@ func (cfgCtlr *configController) lockAndDigestConfigObjects(newPLs []*fctypesv1a
 
 // Digest the new set of PriorityLevelConfiguration objects.
 // Pretend broken ones do not exist.
-func (meal *cfgMeal) digestNewPLsLocked(newPLs []*fctypesv1a1.PriorityLevelConfiguration) {
+func (meal *cfgMeal) digestNewPLsLocked(newPLs []*flowcontrol.PriorityLevelConfiguration) {
 	for _, pl := range newPLs {
 		state := meal.cfgCtlr.priorityLevelStates[pl.Name]
 		if state == nil {
@@ -411,8 +411,8 @@ func (meal *cfgMeal) digestNewPLsLocked(newPLs []*fctypesv1a1.PriorityLevelConfi
 		if state.pl.Spec.Limited != nil {
 			meal.shareSum += float64(state.pl.Spec.Limited.AssuredConcurrencyShares)
 		}
-		meal.haveExemptPL = meal.haveExemptPL || pl.Name == fctypesv1a1.PriorityLevelConfigurationNameExempt
-		meal.haveCatchAllPL = meal.haveCatchAllPL || pl.Name == fctypesv1a1.PriorityLevelConfigurationNameCatchAll
+		meal.haveExemptPL = meal.haveExemptPL || pl.Name == flowcontrol.PriorityLevelConfigurationNameExempt
+		meal.haveCatchAllPL = meal.haveCatchAllPL || pl.Name == flowcontrol.PriorityLevelConfigurationNameCatchAll
 	}
 }
 
@@ -423,9 +423,9 @@ func (meal *cfgMeal) digestNewPLsLocked(newPLs []*fctypesv1a1.PriorityLevelConfi
 // reflect this.  This function also adds any missing mandatory
 // FlowSchema objects.  The given objects must all have distinct
 // names.
-func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*fctypesv1a1.FlowSchema) {
+func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*flowcontrol.FlowSchema) {
 	fsSeq := make(apihelpers.FlowSchemaSequence, 0, len(newFSs))
-	fsMap := make(map[string]*fctypesv1a1.FlowSchema, len(newFSs))
+	fsMap := make(map[string]*flowcontrol.FlowSchema, len(newFSs))
 	var haveExemptFS, haveCatchAllFS bool
 	for i, fs := range newFSs {
 		otherFS := fsMap[fs.Name]
@@ -448,8 +448,8 @@ func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*fctypesv1a1.FlowSchema) {
 			continue
 		}
 		fsSeq = append(fsSeq, newFSs[i])
-		haveExemptFS = haveExemptFS || fs.Name == fctypesv1a1.FlowSchemaNameExempt
-		haveCatchAllFS = haveCatchAllFS || fs.Name == fctypesv1a1.FlowSchemaNameCatchAll
+		haveExemptFS = haveExemptFS || fs.Name == flowcontrol.FlowSchemaNameExempt
+		haveCatchAllFS = haveCatchAllFS || fs.Name == flowcontrol.FlowSchemaNameCatchAll
 	}
 	// sort into the order to be used for matching
 	sort.Sort(fsSeq)
@@ -481,7 +481,7 @@ func (meal *cfgMeal) processOldPLsLocked() {
 			// Still desired and already updated
 			continue
 		}
-		if plName == fctypesv1a1.PriorityLevelConfigurationNameExempt && !meal.haveExemptPL || plName == fctypesv1a1.PriorityLevelConfigurationNameCatchAll && !meal.haveCatchAllPL {
+		if plName == flowcontrol.PriorityLevelConfigurationNameExempt && !meal.haveExemptPL || plName == flowcontrol.PriorityLevelConfigurationNameCatchAll && !meal.haveCatchAllPL {
 			// BTW, we know the Spec has not changed because the
 			// mandatory objects have immutable Specs
 			klog.V(3).Infof("Retaining mandatory priority level %q despite lack of API object", plName)
@@ -513,8 +513,8 @@ func (meal *cfgMeal) processOldPLsLocked() {
 			// regular way.
 			meal.shareSum += float64(plState.pl.Spec.Limited.AssuredConcurrencyShares)
 		}
-		meal.haveExemptPL = meal.haveExemptPL || plName == fctypesv1a1.PriorityLevelConfigurationNameExempt
-		meal.haveCatchAllPL = meal.haveCatchAllPL || plName == fctypesv1a1.PriorityLevelConfigurationNameCatchAll
+		meal.haveExemptPL = meal.haveExemptPL || plName == flowcontrol.PriorityLevelConfigurationNameExempt
+		meal.haveCatchAllPL = meal.haveCatchAllPL || plName == flowcontrol.PriorityLevelConfigurationNameCatchAll
 		meal.newPLStates[plName] = plState
 	}
 }
@@ -548,18 +548,18 @@ func (meal *cfgMeal) finishQueueSetReconfigsLocked() {
 // given priority level configuration.  Returns nil if that config
 // does not call for limiting.  Returns nil and an error if the given
 // object is malformed in a way that is a problem for this package.
-func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *fctypesv1a1.PriorityLevelConfiguration, requestWaitLimit time.Duration, intPair metrics.TimedObserverPair) (fq.QueueSetCompleter, error) {
-	if (pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt) != (pl.Spec.Limited == nil) {
+func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *flowcontrol.PriorityLevelConfiguration, requestWaitLimit time.Duration, intPair metrics.TimedObserverPair) (fq.QueueSetCompleter, error) {
+	if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt) != (pl.Spec.Limited == nil) {
 		return nil, errors.New("broken union structure at the top")
 	}
-	if (pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt) != (pl.Name == fctypesv1a1.PriorityLevelConfigurationNameExempt) {
+	if (pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt) != (pl.Name == flowcontrol.PriorityLevelConfigurationNameExempt) {
 		// This package does not attempt to cope with a priority level dynamically switching between exempt and not.
 		return nil, errors.New("non-alignment between name and type")
 	}
 	if pl.Spec.Limited == nil {
 		return nil, nil
 	}
-	if (pl.Spec.Limited.LimitResponse.Type == fctypesv1a1.LimitResponseTypeReject) != (pl.Spec.Limited.LimitResponse.Queuing == nil) {
+	if (pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeReject) != (pl.Spec.Limited.LimitResponse.Queuing == nil) {
 		return nil, errors.New("broken union structure for limit response")
 	}
 	qcAPI := pl.Spec.Limited.LimitResponse.Queuing
@@ -585,17 +585,17 @@ func queueSetCompleterForPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *fcty
 	return qsc, err
 }
 
-func (meal *cfgMeal) presyncFlowSchemaStatus(fs *fctypesv1a1.FlowSchema, isDangling bool, plName string) {
-	danglingCondition := apihelpers.GetFlowSchemaConditionByType(fs, fctypesv1a1.FlowSchemaConditionDangling)
+func (meal *cfgMeal) presyncFlowSchemaStatus(fs *flowcontrol.FlowSchema, isDangling bool, plName string) {
+	danglingCondition := apihelpers.GetFlowSchemaConditionByType(fs, flowcontrol.FlowSchemaConditionDangling)
 	if danglingCondition == nil {
-		danglingCondition = &fctypesv1a1.FlowSchemaCondition{
-			Type: fctypesv1a1.FlowSchemaConditionDangling,
+		danglingCondition = &flowcontrol.FlowSchemaCondition{
+			Type: flowcontrol.FlowSchemaConditionDangling,
 		}
 	}
-	desiredStatus := fctypesv1a1.ConditionFalse
+	desiredStatus := flowcontrol.ConditionFalse
 	var desiredReason, desiredMessage string
 	if isDangling {
-		desiredStatus = fctypesv1a1.ConditionTrue
+		desiredStatus = flowcontrol.ConditionTrue
 		desiredReason = "NotFound"
 		desiredMessage = fmt.Sprintf("This FlowSchema references the PriorityLevelConfiguration object named %q but there is no such object", plName)
 	} else {
@@ -607,8 +607,8 @@ func (meal *cfgMeal) presyncFlowSchemaStatus(fs *fctypesv1a1.FlowSchema, isDangl
 	}
 	meal.fsStatusUpdates = append(meal.fsStatusUpdates, fsStatusUpdate{
 		flowSchema: fs,
-		condition: fctypesv1a1.FlowSchemaCondition{
-			Type:               fctypesv1a1.FlowSchemaConditionDangling,
+		condition: flowcontrol.FlowSchemaCondition{
+			Type:               flowcontrol.FlowSchemaConditionDangling,
 			Status:             desiredStatus,
 			LastTransitionTime: metav1.Now(),
 			Reason:             desiredReason,
@@ -619,7 +619,7 @@ func (meal *cfgMeal) presyncFlowSchemaStatus(fs *fctypesv1a1.FlowSchema, isDangl
 
 // imaginePL adds a priority level based on one of the mandatory ones
 // that does not actually exist (right now) as a real API object.
-func (meal *cfgMeal) imaginePL(proto *fctypesv1a1.PriorityLevelConfiguration, requestWaitLimit time.Duration) {
+func (meal *cfgMeal) imaginePL(proto *flowcontrol.PriorityLevelConfiguration, requestWaitLimit time.Duration) {
 	klog.V(3).Infof("No %s PriorityLevelConfiguration found, imagining one", proto.Name)
 	obsPair := meal.cfgCtlr.obsPairGenerator.Generate(1, 1, []string{proto.Name})
 	qsCompleter, err := queueSetCompleterForPL(meal.cfgCtlr.queueSetFactory, nil, proto, requestWaitLimit, obsPair)
@@ -651,48 +651,59 @@ func (immediateRequest) Finish(execute func()) bool {
 // The returned bool indicates whether the request is exempt from
 // limitation.  The startWaitingTime is when the request started
 // waiting in its queue, or `Time{}` if this did not happen.
-func (cfgCtlr *configController) startRequest(ctx context.Context, rd RequestDigest, queueNoteFn fq.QueueNoteFn) (fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration, isExempt bool, req fq.Request, startWaitingTime time.Time) {
+func (cfgCtlr *configController) startRequest(ctx context.Context, rd RequestDigest, queueNoteFn fq.QueueNoteFn) (fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration, isExempt bool, req fq.Request, startWaitingTime time.Time) {
 	klog.V(7).Infof("startRequest(%#+v)", rd)
 	cfgCtlr.lock.Lock()
 	defer cfgCtlr.lock.Unlock()
+	var selectedFlowSchema *flowcontrol.FlowSchema
 	for _, fs := range cfgCtlr.flowSchemas {
 		if matchesFlowSchema(rd, fs) {
-			plName := fs.Spec.PriorityLevelConfiguration.Name
-			plState := cfgCtlr.priorityLevelStates[plName]
-			if plState.pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt {
-				klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, immediate", rd, fs.Name, fs.Spec.DistinguisherMethod, plName)
-				return fs, plState.pl, true, immediateRequest{}, time.Time{}
-			}
-			var numQueues int32
-			if plState.pl.Spec.Limited.LimitResponse.Type == fctypesv1a1.LimitResponseTypeQueue {
-				numQueues = plState.pl.Spec.Limited.LimitResponse.Queuing.Queues
-
-			}
-			var flowDistinguisher string
-			var hashValue uint64
-			if numQueues > 1 {
-				flowDistinguisher = computeFlowDistinguisher(rd, fs.Spec.DistinguisherMethod)
-				hashValue = hashFlowID(fs.Name, flowDistinguisher)
-			}
-			startWaitingTime = time.Now()
-			klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, numQueues=%d", rd, fs.Name, fs.Spec.DistinguisherMethod, plName, numQueues)
-			req, idle := plState.queues.StartRequest(ctx, hashValue, flowDistinguisher, fs.Name, rd.RequestInfo, rd.User, queueNoteFn)
-			if idle {
-				cfgCtlr.maybeReapLocked(plName, plState)
-			}
-			return fs, plState.pl, false, req, startWaitingTime
+			selectedFlowSchema = fs
+			break
 		}
 	}
-	// This can never happen because every configState has a
-	// FlowSchema that matches everything.  If somehow control reaches
-	// here, panic with some relevant information.
-	var catchAll *fctypesv1a1.FlowSchema
-	for _, fs := range cfgCtlr.flowSchemas {
-		if fs.Name == fctypesv1a1.FlowSchemaNameCatchAll {
-			catchAll = fs
+	if selectedFlowSchema == nil {
+		// This should never happen. If the requestDigest's User is a part of
+		// system:authenticated or system:unauthenticated, the catch-all flow
+		// schema should match it. However, if that invariant somehow fails,
+		// fallback to the catch-all flow schema anyway.
+		for _, fs := range cfgCtlr.flowSchemas {
+			if fs.Name == flowcontrol.FlowSchemaNameCatchAll {
+				selectedFlowSchema = fs
+				break
+			}
 		}
+		if selectedFlowSchema == nil {
+			// This should absolutely never, ever happen! APF guarantees two
+			// undeletable flow schemas at all times: an exempt flow schema and a
+			// catch-all flow schema.
+			panic(fmt.Sprintf("no fallback catch-all flow schema found for request %#+v and user %#+v", rd.RequestInfo, rd.User))
+		}
+		klog.Warningf("no match found for request %#+v and user %#+v; selecting catchAll=%s as fallback flow schema", rd.RequestInfo, rd.User, fcfmt.Fmt(selectedFlowSchema))
 	}
-	panic(fmt.Sprintf("No match; rd=%#+v, catchAll=%s", rd, fcfmt.Fmt(catchAll)))
+	plName := selectedFlowSchema.Spec.PriorityLevelConfiguration.Name
+	plState := cfgCtlr.priorityLevelStates[plName]
+	if plState.pl.Spec.Type == flowcontrol.PriorityLevelEnablementExempt {
+		klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, immediate", rd, selectedFlowSchema.Name, selectedFlowSchema.Spec.DistinguisherMethod, plName)
+		return selectedFlowSchema, plState.pl, true, immediateRequest{}, time.Time{}
+	}
+	var numQueues int32
+	if plState.pl.Spec.Limited.LimitResponse.Type == flowcontrol.LimitResponseTypeQueue {
+		numQueues = plState.pl.Spec.Limited.LimitResponse.Queuing.Queues
+	}
+	var flowDistinguisher string
+	var hashValue uint64
+	if numQueues > 1 {
+		flowDistinguisher = computeFlowDistinguisher(rd, selectedFlowSchema.Spec.DistinguisherMethod)
+		hashValue = hashFlowID(selectedFlowSchema.Name, flowDistinguisher)
+	}
+	startWaitingTime = time.Now()
+	klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, numQueues=%d", rd, selectedFlowSchema.Name, selectedFlowSchema.Spec.DistinguisherMethod, plName, numQueues)
+	req, idle := plState.queues.StartRequest(ctx, hashValue, flowDistinguisher, selectedFlowSchema.Name, rd.RequestInfo, rd.User, queueNoteFn)
+	if idle {
+		cfgCtlr.maybeReapLocked(plName, plState)
+	}
+	return selectedFlowSchema, plState.pl, false, req, startWaitingTime
 }
 
 // Call this after getting a clue that the given priority level is undesired and idle
@@ -726,14 +737,14 @@ func (cfgCtlr *configController) maybeReapLocked(plName string, plState *priorit
 }
 
 // computeFlowDistinguisher extracts the flow distinguisher according to the given method
-func computeFlowDistinguisher(rd RequestDigest, method *fctypesv1a1.FlowDistinguisherMethod) string {
+func computeFlowDistinguisher(rd RequestDigest, method *flowcontrol.FlowDistinguisherMethod) string {
 	if method == nil {
 		return ""
 	}
 	switch method.Type {
-	case fctypesv1a1.FlowDistinguisherMethodByUserType:
+	case flowcontrol.FlowDistinguisherMethodByUserType:
 		return rd.User.GetName()
-	case fctypesv1a1.FlowDistinguisherMethodByNamespaceType:
+	case flowcontrol.FlowDistinguisherMethodByNamespaceType:
 		return rd.RequestInfo.Namespace
 	default:
 		// this line shall never reach

vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller_debug.go

@@ -164,28 +164,6 @@ func (cfgCtlr *configController) dumpRequests(w http.ResponseWriter, r *http.Req
 	endLine(tabWriter)
 	for _, plState := range cfgCtlr.priorityLevelStates {
 		if plState.queues == nil {
-			tabPrint(tabWriter, row(
-				plState.pl.Name, // 1
-				"<none>",        // 2
-				"<none>",        // 3
-				"<none>",        // 4
-				"<none>",        // 5
-				"<none>",        // 6
-			))
-			if includeRequestDetails {
-				continueLine(tabWriter)
-				tabPrint(tabWriter, row(
-					"<none>", // 7
-					"<none>", // 8
-					"<none>", // 9
-					"<none>", // 10
-					"<none>", // 11
-					"<none>", // 12
-					"<none>", // 13
-					"<none>", // 14
-				))
-			}
-			endLine(tabWriter)
 			continue
 		}
 		queueSetDigest := plState.queues.Dump(includeRequestDetails)

vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go

@@ -30,8 +30,8 @@ import (
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/klog/v2"
 
-	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
-	fcclientv1a1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
+	flowcontrolclient "k8s.io/client-go/kubernetes/typed/flowcontrol/v1beta1"
 )
 
 // Interface defines how the API Priority and Fairness filter interacts with the underlying system.
@@ -47,7 +47,7 @@ type Interface interface {
 	// not be invoked.
 	Handle(ctx context.Context,
 		requestDigest RequestDigest,
-		noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
+		noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration),
 		queueNoteFn fq.QueueNoteFn,
 		execFn func(),
 	)
@@ -69,7 +69,7 @@ type Interface interface {
 // New creates a new instance to implement API priority and fairness
 func New(
 	informerFactory kubeinformers.SharedInformerFactory,
-	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	flowcontrolClient flowcontrolclient.FlowcontrolV1beta1Interface,
 	serverConcurrencyLimit int,
 	requestWaitLimit time.Duration,
 ) Interface {
@@ -87,7 +87,7 @@ func New(
 // NewTestable is extra flexible to facilitate testing
 func NewTestable(
 	informerFactory kubeinformers.SharedInformerFactory,
-	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	flowcontrolClient flowcontrolclient.FlowcontrolV1beta1Interface,
 	serverConcurrencyLimit int,
 	requestWaitLimit time.Duration,
 	obsPairGenerator metrics.TimedObserverPairGenerator,
@@ -97,7 +97,7 @@ func NewTestable(
 }
 
 func (cfgCtlr *configController) Handle(ctx context.Context, requestDigest RequestDigest,
-	noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
+	noteFn func(fs *flowcontrol.FlowSchema, pl *flowcontrol.PriorityLevelConfiguration),
 	queueNoteFn fq.QueueNoteFn,
 	execFn func()) {
 	fs, pl, isExempt, req, startWaitingTime := cfgCtlr.startRequest(ctx, requestDigest, queueNoteFn)

vendor/k8s.io/apiserver/pkg/util/flowcontrol/debug/dump.go

@@ -17,8 +17,9 @@ limitations under the License.
 package debug
 
 import (
-	"k8s.io/apiserver/pkg/endpoints/request"
 	"time"
+
+	"k8s.io/apiserver/pkg/endpoints/request"
 )
 
 // QueueSetDump is an instant dump of queue-set.

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go

@@ -645,6 +645,7 @@ func (qs *queueSet) selectQueueLocked() *queue {
 		qs.robinIndex = (qs.robinIndex + 1) % nq
 		queue := qs.queues[qs.robinIndex]
 		if len(queue.requests) != 0 {
+
 			currentVirtualFinish := queue.GetVirtualFinish(0, qs.estimatedServiceTime)
 			if currentVirtualFinish < minVirtualFinish {
 				minVirtualFinish = currentVirtualFinish
@@ -657,6 +658,23 @@ func (qs *queueSet) selectQueueLocked() *queue {
 	// for the next round.  This way the non-selected queues
 	// win in the case that the virtual finish times are the same
 	qs.robinIndex = minIndex
+	// according to the original FQ formula:
+	//
+	//   Si = MAX(R(t), Fi-1)
+	//
+	// the virtual start (excluding the estimated cost) of the chose
+	// queue should always be greater or equal to the global virtual
+	// time.
+	//
+	// hence we're refreshing the per-queue virtual time for the chosen
+	// queue here. if the last virtual start time (excluded estimated cost)
+	// falls behind the global virtual time, we update the latest virtual
+	// start by: <latest global virtual time> + <previously estimated cost>
+	previouslyEstimatedServiceTime := float64(minQueue.requestsExecuting) * qs.estimatedServiceTime
+	if qs.virtualTime > minQueue.virtualStart-previouslyEstimatedServiceTime {
+		// per-queue virtual time should not fall behind the global
+		minQueue.virtualStart = qs.virtualTime + previouslyEstimatedServiceTime
+	}
 	return minQueue
 }
 

vendor/k8s.io/apiserver/pkg/util/flowcontrol/format/formatting.go

@@ -21,7 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 
-	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
 )
@@ -46,25 +46,25 @@ func (sr Stringer) String() string {
 		return "nil"
 	}
 	switch typed := sr.val.(type) {
-	case *fcv1a1.FlowSchema,
-		fcv1a1.FlowSchema,
-		fcv1a1.FlowSchemaSpec,
-		fcv1a1.FlowDistinguisherMethod,
-		*fcv1a1.FlowDistinguisherMethod,
-		*fcv1a1.PolicyRulesWithSubjects,
-		fcv1a1.PolicyRulesWithSubjects,
-		fcv1a1.Subject,
-		fcv1a1.ResourcePolicyRule,
-		fcv1a1.NonResourcePolicyRule,
-		fcv1a1.FlowSchemaCondition,
-		*fcv1a1.PriorityLevelConfiguration,
-		fcv1a1.PriorityLevelConfiguration,
-		fcv1a1.PriorityLevelConfigurationSpec,
-		*fcv1a1.LimitedPriorityLevelConfiguration,
-		fcv1a1.LimitedPriorityLevelConfiguration,
-		fcv1a1.LimitResponse,
-		*fcv1a1.QueuingConfiguration,
-		fcv1a1.QueuingConfiguration:
+	case *flowcontrol.FlowSchema,
+		flowcontrol.FlowSchema,
+		flowcontrol.FlowSchemaSpec,
+		flowcontrol.FlowDistinguisherMethod,
+		*flowcontrol.FlowDistinguisherMethod,
+		*flowcontrol.PolicyRulesWithSubjects,
+		flowcontrol.PolicyRulesWithSubjects,
+		flowcontrol.Subject,
+		flowcontrol.ResourcePolicyRule,
+		flowcontrol.NonResourcePolicyRule,
+		flowcontrol.FlowSchemaCondition,
+		*flowcontrol.PriorityLevelConfiguration,
+		flowcontrol.PriorityLevelConfiguration,
+		flowcontrol.PriorityLevelConfigurationSpec,
+		*flowcontrol.LimitedPriorityLevelConfiguration,
+		flowcontrol.LimitedPriorityLevelConfiguration,
+		flowcontrol.LimitResponse,
+		*flowcontrol.QueuingConfiguration,
+		flowcontrol.QueuingConfiguration:
 		return ToJSON(sr.val)
 	case []user.Info:
 		return FmtUsers(typed)
@@ -88,12 +88,12 @@ func ToJSON(val interface{}) string {
 
 // FmtPriorityLevelConfiguration returns a golang source expression
 // equivalent to the given value
-func FmtPriorityLevelConfiguration(pl *fcv1a1.PriorityLevelConfiguration) string {
+func FmtPriorityLevelConfiguration(pl *flowcontrol.PriorityLevelConfiguration) string {
 	if pl == nil {
 		return "nil"
 	}
 	var buf bytes.Buffer
-	buf.WriteString(fmt.Sprintf("&v1alpha1.PriorityLevelConfiguration{ObjectMeta: %#+v, Spec: ",
+	buf.WriteString(fmt.Sprintf("&flowcontrolv1beta1.PriorityLevelConfiguration{ObjectMeta: %#+v, Spec: ",
 		pl.ObjectMeta))
 	BufferPriorityLevelConfigurationSpec(&buf, &pl.Spec)
 	buf.WriteString(fmt.Sprintf(", Status: %#+v}", pl.Status))
@@ -102,7 +102,7 @@ func FmtPriorityLevelConfiguration(pl *fcv1a1.PriorityLevelConfiguration) string
 
 // FmtPriorityLevelConfigurationSpec returns a golang source
 // expression equivalent to the given value
-func FmtPriorityLevelConfigurationSpec(plSpec *fcv1a1.PriorityLevelConfigurationSpec) string {
+func FmtPriorityLevelConfigurationSpec(plSpec *flowcontrol.PriorityLevelConfigurationSpec) string {
 	var buf bytes.Buffer
 	BufferPriorityLevelConfigurationSpec(&buf, plSpec)
 	return buf.String()
@@ -110,10 +110,10 @@ func FmtPriorityLevelConfigurationSpec(plSpec *fcv1a1.PriorityLevelConfiguration
 
 // BufferPriorityLevelConfigurationSpec writes a golang source
 // expression for the given value to the given buffer
-func BufferPriorityLevelConfigurationSpec(buf *bytes.Buffer, plSpec *fcv1a1.PriorityLevelConfigurationSpec) {
-	buf.WriteString(fmt.Sprintf("v1alpha1.PriorityLevelConfigurationSpec{Type: %#v", plSpec.Type))
+func BufferPriorityLevelConfigurationSpec(buf *bytes.Buffer, plSpec *flowcontrol.PriorityLevelConfigurationSpec) {
+	buf.WriteString(fmt.Sprintf("flowcontrolv1beta1.PriorityLevelConfigurationSpec{Type: %#v", plSpec.Type))
 	if plSpec.Limited != nil {
-		buf.WriteString(fmt.Sprintf(", Limited: &v1alpha1.LimitedPriorityLevelConfiguration{AssuredConcurrencyShares:%d, LimitResponse:v1alpha1.LimitResponse{Type:%#v", plSpec.Limited.AssuredConcurrencyShares, plSpec.Limited.LimitResponse.Type))
+		buf.WriteString(fmt.Sprintf(", Limited: &flowcontrol.LimitedPriorityLevelConfiguration{AssuredConcurrencyShares:%d, LimitResponse:flowcontrol.LimitResponse{Type:%#v", plSpec.Limited.AssuredConcurrencyShares, plSpec.Limited.LimitResponse.Type))
 		if plSpec.Limited.LimitResponse.Queuing != nil {
 			buf.WriteString(fmt.Sprintf(", Queuing:&%#+v", *plSpec.Limited.LimitResponse.Queuing))
 		}
@@ -123,12 +123,12 @@ func BufferPriorityLevelConfigurationSpec(buf *bytes.Buffer, plSpec *fcv1a1.Prio
 }
 
 // FmtFlowSchema produces a golang source expression of the value.
-func FmtFlowSchema(fs *fcv1a1.FlowSchema) string {
+func FmtFlowSchema(fs *flowcontrol.FlowSchema) string {
 	if fs == nil {
 		return "nil"
 	}
 	var buf bytes.Buffer
-	buf.WriteString(fmt.Sprintf("&v1alpha1.FlowSchema{ObjectMeta: %#+v, Spec: ",
+	buf.WriteString(fmt.Sprintf("&flowcontrolv1beta1.FlowSchema{ObjectMeta: %#+v, Spec: ",
 		fs.ObjectMeta))
 	BufferFlowSchemaSpec(&buf, &fs.Spec)
 	buf.WriteString(fmt.Sprintf(", Status: %#+v}", fs.Status))
@@ -137,7 +137,7 @@ func FmtFlowSchema(fs *fcv1a1.FlowSchema) string {
 
 // FmtFlowSchemaSpec produces a golang source expression equivalent to
 // the given spec
-func FmtFlowSchemaSpec(fsSpec *fcv1a1.FlowSchemaSpec) string {
+func FmtFlowSchemaSpec(fsSpec *flowcontrol.FlowSchemaSpec) string {
 	var buf bytes.Buffer
 	BufferFlowSchemaSpec(&buf, fsSpec)
 	return buf.String()
@@ -145,8 +145,8 @@ func FmtFlowSchemaSpec(fsSpec *fcv1a1.FlowSchemaSpec) string {
 
 // BufferFlowSchemaSpec writes a golang source expression for the
 // given value to the given buffer
-func BufferFlowSchemaSpec(buf *bytes.Buffer, fsSpec *fcv1a1.FlowSchemaSpec) {
-	buf.WriteString(fmt.Sprintf("v1alpha1.FlowSchemaSpec{PriorityLevelConfiguration: %#+v, MatchingPrecedence: %d, DistinguisherMethod: ",
+func BufferFlowSchemaSpec(buf *bytes.Buffer, fsSpec *flowcontrol.FlowSchemaSpec) {
+	buf.WriteString(fmt.Sprintf("flowcontrolv1beta1.FlowSchemaSpec{PriorityLevelConfiguration: %#+v, MatchingPrecedence: %d, DistinguisherMethod: ",
 		fsSpec.PriorityLevelConfiguration,
 		fsSpec.MatchingPrecedence))
 	if fsSpec.DistinguisherMethod == nil {
@@ -154,7 +154,7 @@ func BufferFlowSchemaSpec(buf *bytes.Buffer, fsSpec *fcv1a1.FlowSchemaSpec) {
 	} else {
 		buf.WriteString(fmt.Sprintf("&%#+v", *fsSpec.DistinguisherMethod))
 	}
-	buf.WriteString(", Rules: []v1alpha1.PolicyRulesWithSubjects{")
+	buf.WriteString(", Rules: []flowcontrol.PolicyRulesWithSubjects{")
 	for idx, rule := range fsSpec.Rules {
 		if idx > 0 {
 			buf.WriteString(", ")
@@ -165,14 +165,14 @@ func BufferFlowSchemaSpec(buf *bytes.Buffer, fsSpec *fcv1a1.FlowSchemaSpec) {
 }
 
 // FmtPolicyRulesWithSubjects produces a golang source expression of the value.
-func FmtPolicyRulesWithSubjects(rule fcv1a1.PolicyRulesWithSubjects) string {
-	return "v1alpha1.PolicyRulesWithSubjects" + FmtPolicyRulesWithSubjectsSlim(rule)
+func FmtPolicyRulesWithSubjects(rule flowcontrol.PolicyRulesWithSubjects) string {
+	return "flowcontrolv1beta1.PolicyRulesWithSubjects" + FmtPolicyRulesWithSubjectsSlim(rule)
 }
 
 // FmtPolicyRulesWithSubjectsSlim produces a golang source expression
 // of the value but without the leading type name.  See above for an
 // example context where this is useful.
-func FmtPolicyRulesWithSubjectsSlim(rule fcv1a1.PolicyRulesWithSubjects) string {
+func FmtPolicyRulesWithSubjectsSlim(rule flowcontrol.PolicyRulesWithSubjects) string {
 	var buf bytes.Buffer
 	BufferFmtPolicyRulesWithSubjectsSlim(&buf, rule)
 	return buf.String()
@@ -181,8 +181,8 @@ func FmtPolicyRulesWithSubjectsSlim(rule fcv1a1.PolicyRulesWithSubjects) string
 // BufferFmtPolicyRulesWithSubjectsSlim writes a golang source
 // expression for the given value to the given buffer but excludes the
 // leading type name
-func BufferFmtPolicyRulesWithSubjectsSlim(buf *bytes.Buffer, rule fcv1a1.PolicyRulesWithSubjects) {
-	buf.WriteString("{Subjects: []v1alpha1.Subject{")
+func BufferFmtPolicyRulesWithSubjectsSlim(buf *bytes.Buffer, rule flowcontrol.PolicyRulesWithSubjects) {
+	buf.WriteString("{Subjects: []flowcontrolv1beta1.Subject{")
 	for jdx, subj := range rule.Subjects {
 		if jdx > 0 {
 			buf.WriteString(", ")

vendor/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go

@@ -34,8 +34,8 @@ const (
 
 const (
 	requestKind   = "request_kind"
-	priorityLevel = "priorityLevel"
-	flowSchema    = "flowSchema"
+	priorityLevel = "priority_level"
+	flowSchema    = "flow_schema"
 	phase         = "phase"
 	mark          = "mark"
 )
@@ -84,19 +84,21 @@ func (rs Registerables) Append(more ...compbasemetrics.Registerable) Registerabl
 var (
 	apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "rejected_requests_total",
-			Help:      "Number of requests rejected by API Priority and Fairness system",
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "rejected_requests_total",
+			Help:           "Number of requests rejected by API Priority and Fairness system",
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema, "reason"},
 	)
 	apiserverDispatchedRequestsTotal = compbasemetrics.NewCounterVec(
 		&compbasemetrics.CounterOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "dispatched_requests_total",
-			Help:      "Number of requests released by API Priority and Fairness system for service",
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "dispatched_requests_total",
+			Help:           "Number of requests released by API Priority and Fairness system for service",
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema},
 	)
@@ -143,58 +145,64 @@ var (
 
 	apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "current_inqueue_requests",
-			Help:      "Number of requests currently pending in queues of the API Priority and Fairness system",
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "current_inqueue_requests",
+			Help:           "Number of requests currently pending in queues of the API Priority and Fairness system",
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema},
 	)
 	apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
 		&compbasemetrics.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "request_queue_length_after_enqueue",
-			Help:      "Length of queue in the API Priority and Fairness system, as seen by each request after it is enqueued",
-			Buckets:   queueLengthBuckets,
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "request_queue_length_after_enqueue",
+			Help:           "Length of queue in the API Priority and Fairness system, as seen by each request after it is enqueued",
+			Buckets:        queueLengthBuckets,
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema},
 	)
 	apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "request_concurrency_limit",
-			Help:      "Shared concurrency limit in the API Priority and Fairness system",
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "request_concurrency_limit",
+			Help:           "Shared concurrency limit in the API Priority and Fairness system",
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel},
 	)
 	apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
 		&compbasemetrics.GaugeOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "current_executing_requests",
-			Help:      "Number of requests currently executing in the API Priority and Fairness system",
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "current_executing_requests",
+			Help:           "Number of requests currently executing in the API Priority and Fairness system",
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema},
 	)
 	apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
 		&compbasemetrics.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "request_wait_duration_seconds",
-			Help:      "Length of time a request spent waiting in its queue",
-			Buckets:   requestDurationSecondsBuckets,
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "request_wait_duration_seconds",
+			Help:           "Length of time a request spent waiting in its queue",
+			Buckets:        requestDurationSecondsBuckets,
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema, "execute"},
 	)
 	apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
 		&compbasemetrics.HistogramOpts{
-			Namespace: namespace,
-			Subsystem: subsystem,
-			Name:      "request_execution_seconds",
-			Help:      "Duration of request execution in the API Priority and Fairness system",
-			Buckets:   requestDurationSecondsBuckets,
+			Namespace:      namespace,
+			Subsystem:      subsystem,
+			Name:           "request_execution_seconds",
+			Help:           "Duration of request execution in the API Priority and Fairness system",
+			Buckets:        requestDurationSecondsBuckets,
+			StabilityLevel: compbasemetrics.ALPHA,
 		},
 		[]string{priorityLevel, flowSchema},
 	)

vendor/k8s.io/apiserver/pkg/util/flowcontrol/metrics/sample_and_watermark.go

@@ -64,13 +64,16 @@ func (spg SampleAndWaterMarkPairGenerator) metrics() Registerables {
 // SampleAndWaterMarkObserverGenerator creates TimedObservers that
 // populate histograms of samples and low- and high-water-marks.  The
 // generator has a samplePeriod, and the histograms get an observation
-// every samplePeriod.
+// every samplePeriod.  The sampling windows are quantized based on
+// the monotonic rather than wall-clock times.  The `t0` field is
+// there so to provide a baseline for monotonic clock differences.
 type SampleAndWaterMarkObserverGenerator struct {
 	*sampleAndWaterMarkObserverGenerator
 }
 
 type sampleAndWaterMarkObserverGenerator struct {
 	clock        clock.PassiveClock
+	t0           time.Time
 	samplePeriod time.Duration
 	samples      *compbasemetrics.HistogramVec
 	waterMarks   *compbasemetrics.HistogramVec
@@ -83,6 +86,7 @@ func NewSampleAndWaterMarkHistogramsGenerator(clock clock.PassiveClock, samplePe
 	return SampleAndWaterMarkObserverGenerator{
 		&sampleAndWaterMarkObserverGenerator{
 			clock:        clock,
+			t0:           clock.Now(),
 			samplePeriod: samplePeriod,
 			samples:      compbasemetrics.NewHistogramVec(sampleOpts, labelNames),
 			waterMarks:   compbasemetrics.NewHistogramVec(waterMarkOpts, append([]string{labelNameMark}, labelNames...)),
@@ -90,7 +94,7 @@ func NewSampleAndWaterMarkHistogramsGenerator(clock clock.PassiveClock, samplePe
 }
 
 func (swg *sampleAndWaterMarkObserverGenerator) quantize(when time.Time) int64 {
-	return when.UnixNano() / int64(swg.samplePeriod)
+	return int64(when.Sub(swg.t0) / swg.samplePeriod)
 }
 
 // Generate makes a new TimedObserver
@@ -156,31 +160,46 @@ func (saw *sampleAndWaterMarkHistograms) SetX1(x1 float64) {
 }
 
 func (saw *sampleAndWaterMarkHistograms) innerSet(updateXOrX1 func()) {
-	saw.Lock()
-	when := saw.clock.Now()
-	whenInt := saw.quantize(when)
-	acc := saw.sampleAndWaterMarkAccumulator
-	wellOrdered := !when.Before(acc.lastSet)
-	if wellOrdered {
+	var when time.Time
+	var whenInt int64
+	var acc sampleAndWaterMarkAccumulator
+	var wellOrdered bool
+	func() {
+		saw.Lock()
+		defer saw.Unlock()
+		when = saw.clock.Now()
+		whenInt = saw.quantize(when)
+		acc = saw.sampleAndWaterMarkAccumulator
+		wellOrdered = !when.Before(acc.lastSet)
 		updateXOrX1()
 		saw.relX = saw.x / saw.x1
-		if acc.lastSetInt < whenInt {
-			saw.loRelX, saw.hiRelX = acc.relX, acc.relX
-			saw.lastSetInt = whenInt
+		if wellOrdered {
+			if acc.lastSetInt < whenInt {
+				saw.loRelX, saw.hiRelX = acc.relX, acc.relX
+				saw.lastSetInt = whenInt
+			}
+			saw.lastSet = when
 		}
+		// `wellOrdered` should always be true because we are using
+		// monotonic clock readings and they never go backwards.  Yet
+		// very small backwards steps (under 1 microsecond) have been
+		// observed
+		// (https://github.com/kubernetes/kubernetes/issues/96459).
+		// In the backwards case, treat the current reading as if it
+		// had occurred at time `saw.lastSet` and log an error.  It
+		// would be wrong to update `saw.lastSet` in this case because
+		// that plants a time bomb for future updates to
+		// `saw.lastSetInt`.
 		if saw.relX < saw.loRelX {
 			saw.loRelX = saw.relX
 		} else if saw.relX > saw.hiRelX {
 			saw.hiRelX = saw.relX
 		}
-		saw.lastSet = when
-	}
-	saw.Unlock()
+	}()
 	if !wellOrdered {
-		lastSetS := acc.lastSet.Format(time.RFC3339Nano)
-		whenS := when.Format(time.RFC3339Nano)
-		klog.Fatalf("Time went backwards from %s to %s for labelValues=%#+v", lastSetS, whenS, saw.labelValues)
-		panic(append([]string{lastSetS, whenS}, saw.labelValues...))
+		lastSetS := acc.lastSet.String()
+		whenS := when.String()
+		klog.Errorf("Time went backwards from %s to %s for labelValues=%#+v", lastSetS, whenS, saw.labelValues)
 	}
 	for acc.lastSetInt < whenInt {
 		saw.samples.WithLabelValues(saw.labelValues...).Observe(acc.relX)

vendor/k8s.io/apiserver/pkg/util/flowcontrol/rule.go

@@ -19,7 +19,7 @@ package flowcontrol
 import (
 	"strings"
 
-	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	flowcontrol "k8s.io/api/flowcontrol/v1beta1"
 	"k8s.io/apiserver/pkg/authentication/serviceaccount"
 	"k8s.io/apiserver/pkg/authentication/user"
 	"k8s.io/apiserver/pkg/endpoints/request"
@@ -27,7 +27,7 @@ import (
 
 // Tests whether a given request and FlowSchema match.  Nobody mutates
 // either input.
-func matchesFlowSchema(digest RequestDigest, flowSchema *fctypesv1a1.FlowSchema) bool {
+func matchesFlowSchema(digest RequestDigest, flowSchema *flowcontrol.FlowSchema) bool {
 	for _, policyRule := range flowSchema.Spec.Rules {
 		if matchesPolicyRule(digest, &policyRule) {
 			return true
@@ -36,7 +36,7 @@ func matchesFlowSchema(digest RequestDigest, flowSchema *fctypesv1a1.FlowSchema)
 	return false
 }
 
-func matchesPolicyRule(digest RequestDigest, policyRule *fctypesv1a1.PolicyRulesWithSubjects) bool {
+func matchesPolicyRule(digest RequestDigest, policyRule *flowcontrol.PolicyRulesWithSubjects) bool {
 	if !matchesASubject(digest.User, policyRule.Subjects) {
 		return false
 	}
@@ -46,7 +46,7 @@ func matchesPolicyRule(digest RequestDigest, policyRule *fctypesv1a1.PolicyRules
 	return matchesANonResourceRule(digest.RequestInfo, policyRule.NonResourceRules)
 }
 
-func matchesASubject(user user.Info, subjects []fctypesv1a1.Subject) bool {
+func matchesASubject(user user.Info, subjects []flowcontrol.Subject) bool {
 	for _, subject := range subjects {
 		if matchesSubject(user, subject) {
 			return true
@@ -55,11 +55,11 @@ func matchesASubject(user user.Info, subjects []fctypesv1a1.Subject) bool {
 	return false
 }
 
-func matchesSubject(user user.Info, subject fctypesv1a1.Subject) bool {
+func matchesSubject(user user.Info, subject flowcontrol.Subject) bool {
 	switch subject.Kind {
-	case fctypesv1a1.SubjectKindUser:
-		return subject.User != nil && (subject.User.Name == fctypesv1a1.NameAll || subject.User.Name == user.GetName())
-	case fctypesv1a1.SubjectKindGroup:
+	case flowcontrol.SubjectKindUser:
+		return subject.User != nil && (subject.User.Name == flowcontrol.NameAll || subject.User.Name == user.GetName())
+	case flowcontrol.SubjectKindGroup:
 		if subject.Group == nil {
 			return false
 		}
@@ -73,11 +73,11 @@ func matchesSubject(user user.Info, subject fctypesv1a1.Subject) bool {
 			}
 		}
 		return false
-	case fctypesv1a1.SubjectKindServiceAccount:
+	case flowcontrol.SubjectKindServiceAccount:
 		if subject.ServiceAccount == nil {
 			return false
 		}
-		if subject.ServiceAccount.Name == fctypesv1a1.NameAll {
+		if subject.ServiceAccount.Name == flowcontrol.NameAll {
 			return serviceAccountMatchesNamespace(subject.ServiceAccount.Namespace, user.GetName())
 		}
 		return serviceaccount.MatchesUsername(subject.ServiceAccount.Namespace, subject.ServiceAccount.Name, user.GetName())
@@ -107,7 +107,7 @@ func serviceAccountMatchesNamespace(namespace string, username string) bool {
 	return strings.HasPrefix(username, ServiceAccountUsernameSeparator)
 }
 
-func matchesAResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.ResourcePolicyRule) bool {
+func matchesAResourceRule(ri *request.RequestInfo, rules []flowcontrol.ResourcePolicyRule) bool {
 	for _, rr := range rules {
 		if matchesResourcePolicyRule(ri, rr) {
 			return true
@@ -116,7 +116,7 @@ func matchesAResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.ResourceP
 	return false
 }
 
-func matchesResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a1.ResourcePolicyRule) bool {
+func matchesResourcePolicyRule(ri *request.RequestInfo, policyRule flowcontrol.ResourcePolicyRule) bool {
 	if !matchPolicyRuleVerb(policyRule.Verbs, ri.Verb) {
 		return false
 	}
@@ -129,10 +129,10 @@ func matchesResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a1.R
 	if len(ri.Namespace) == 0 {
 		return policyRule.ClusterScope
 	}
-	return containsString(ri.Namespace, policyRule.Namespaces, fctypesv1a1.NamespaceEvery)
+	return containsString(ri.Namespace, policyRule.Namespaces, flowcontrol.NamespaceEvery)
 }
 
-func matchesANonResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.NonResourcePolicyRule) bool {
+func matchesANonResourceRule(ri *request.RequestInfo, rules []flowcontrol.NonResourcePolicyRule) bool {
 	for _, rr := range rules {
 		if matchesNonResourcePolicyRule(ri, rr) {
 			return true
@@ -141,7 +141,7 @@ func matchesANonResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.NonRes
 	return false
 }
 
-func matchesNonResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a1.NonResourcePolicyRule) bool {
+func matchesNonResourcePolicyRule(ri *request.RequestInfo, policyRule flowcontrol.NonResourcePolicyRule) bool {
 	if !matchPolicyRuleVerb(policyRule.Verbs, ri.Verb) {
 		return false
 	}
@@ -149,12 +149,12 @@ func matchesNonResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a
 }
 
 func matchPolicyRuleVerb(policyRuleVerbs []string, requestVerb string) bool {
-	return containsString(requestVerb, policyRuleVerbs, fctypesv1a1.VerbAll)
+	return containsString(requestVerb, policyRuleVerbs, flowcontrol.VerbAll)
 }
 
 func matchPolicyRuleNonResourceURL(policyRuleRequestURLs []string, requestPath string) bool {
 	for _, rulePath := range policyRuleRequestURLs {
-		if rulePath == fctypesv1a1.NonResourceAll || rulePath == requestPath {
+		if rulePath == flowcontrol.NonResourceAll || rulePath == requestPath {
 			return true
 		}
 		rulePrefix := strings.TrimSuffix(rulePath, "*")
@@ -169,7 +169,7 @@ func matchPolicyRuleNonResourceURL(policyRuleRequestURLs []string, requestPath s
 }
 
 func matchPolicyRuleAPIGroup(policyRuleAPIGroups []string, requestAPIGroup string) bool {
-	return containsString(requestAPIGroup, policyRuleAPIGroups, fctypesv1a1.APIGroupAll)
+	return containsString(requestAPIGroup, policyRuleAPIGroups, flowcontrol.APIGroupAll)
 }
 
 func rsJoin(requestResource, requestSubresource string) string {
@@ -181,7 +181,7 @@ func rsJoin(requestResource, requestSubresource string) string {
 }
 
 func matchPolicyRuleResource(policyRuleRequestResources []string, requestResource, requestSubresource string) bool {
-	return containsString(rsJoin(requestResource, requestSubresource), policyRuleRequestResources, fctypesv1a1.ResourceAll)
+	return containsString(rsJoin(requestResource, requestSubresource), policyRuleRequestResources, flowcontrol.ResourceAll)
 }
 
 // containsString returns true if either `x` or `wildcard` is in

vendor/k8s.io/apiserver/pkg/util/webhook/authentication.go

@@ -55,7 +55,7 @@ func NewDefaultAuthenticationInfoResolverWrapper(
 				}
 
 				if egressSelector != nil {
-					networkContext := egressselector.Master.AsNetworkContext()
+					networkContext := egressselector.ControlPlane.AsNetworkContext()
 					var egressDialer utilnet.DialFunc
 					egressDialer, err = egressSelector.Lookup(networkContext)
 

vendor/k8s.io/apiserver/pkg/util/webhook/webhook.go

@@ -36,12 +36,23 @@ import (
 // timeout of the HTTP request, including reading the response body.
 const defaultRequestTimeout = 30 * time.Second
 
+// DefaultRetryBackoffWithInitialDelay returns the default backoff parameters for webhook retry from a given initial delay.
+// Handy for the client that provides a custom initial delay only.
+func DefaultRetryBackoffWithInitialDelay(initialBackoffDelay time.Duration) wait.Backoff {
+	return wait.Backoff{
+		Duration: initialBackoffDelay,
+		Factor:   1.5,
+		Jitter:   0.2,
+		Steps:    5,
+	}
+}
+
 // GenericWebhook defines a generic client for webhooks with commonly used capabilities,
 // such as retry requests.
 type GenericWebhook struct {
-	RestClient     *rest.RESTClient
-	InitialBackoff time.Duration
-	ShouldRetry    func(error) bool
+	RestClient   *rest.RESTClient
+	RetryBackoff wait.Backoff
+	ShouldRetry  func(error) bool
 }
 
 // DefaultShouldRetry is a default implementation for the GenericWebhook ShouldRetry function property.
@@ -61,11 +72,11 @@ func DefaultShouldRetry(err error) bool {
 }
 
 // NewGenericWebhook creates a new GenericWebhook from the provided kubeconfig file.
-func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) {
-	return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout, customDial)
+func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, retryBackoff wait.Backoff, customDial utilnet.DialFunc) (*GenericWebhook, error) {
+	return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, retryBackoff, defaultRequestTimeout, customDial)
 }
 
-func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) {
+func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, retryBackoff wait.Backoff, requestTimeout time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) {
 	for _, groupVersion := range groupVersions {
 		if !scheme.IsVersionRegistered(groupVersion) {
 			return nil, fmt.Errorf("webhook plugin requires enabling extension resource: %s", groupVersion)
@@ -102,19 +113,20 @@ func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFact
 		return nil, err
 	}
 
-	return &GenericWebhook{restClient, initialBackoff, DefaultShouldRetry}, nil
+	return &GenericWebhook{restClient, retryBackoff, DefaultShouldRetry}, nil
 }
 
-// WithExponentialBackoff will retry webhookFn() up to 5 times with exponentially increasing backoff when
-// it returns an error for which this GenericWebhook's ShouldRetry function returns true, confirming it to
-// be retriable. If no ShouldRetry has been defined for the webhook, then the default one is used (DefaultShouldRetry).
+// WithExponentialBackoff will retry webhookFn() as specified by the given backoff parameters with exponentially
+// increasing backoff when it returns an error for which this GenericWebhook's ShouldRetry function returns true,
+// confirming it to be retriable. If no ShouldRetry has been defined for the webhook,
+// then the default one is used (DefaultShouldRetry).
 func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn func() rest.Result) rest.Result {
 	var result rest.Result
 	shouldRetry := g.ShouldRetry
 	if shouldRetry == nil {
 		shouldRetry = DefaultShouldRetry
 	}
-	WithExponentialBackoff(ctx, g.InitialBackoff, func() error {
+	WithExponentialBackoff(ctx, g.RetryBackoff, func() error {
 		result = webhookFn()
 		return result.Error()
 	}, shouldRetry)
@@ -123,28 +135,28 @@ func (g *GenericWebhook) WithExponentialBackoff(ctx context.Context, webhookFn f
 
 // WithExponentialBackoff will retry webhookFn up to 5 times with exponentially increasing backoff when
 // it returns an error for which shouldRetry returns true, confirming it to be retriable.
-func WithExponentialBackoff(ctx context.Context, initialBackoff time.Duration, webhookFn func() error, shouldRetry func(error) bool) error {
-	backoff := wait.Backoff{
-		Duration: initialBackoff,
-		Factor:   1.5,
-		Jitter:   0.2,
-		Steps:    5,
-	}
-
-	var err error
-	wait.ExponentialBackoff(backoff, func() (bool, error) {
-		err = webhookFn()
-		if ctx.Err() != nil {
-			// we timed out or were cancelled, we should not retry
-			return true, err
-		}
-		if shouldRetry(err) {
+func WithExponentialBackoff(ctx context.Context, retryBackoff wait.Backoff, webhookFn func() error, shouldRetry func(error) bool) error {
+	// having a webhook error allows us to track the last actual webhook error for requests that
+	// are later cancelled or time out.
+	var webhookErr error
+	err := wait.ExponentialBackoffWithContext(ctx, retryBackoff, func() (bool, error) {
+		webhookErr = webhookFn()
+		if shouldRetry(webhookErr) {
 			return false, nil
 		}
-		if err != nil {
-			return false, err
+		if webhookErr != nil {
+			return false, webhookErr
 		}
 		return true, nil
 	})
-	return err
+
+	switch {
+	// we check for webhookErr first, if webhookErr is set it's the most important error to return.
+	case webhookErr != nil:
+		return webhookErr
+	case err != nil:
+		return fmt.Errorf("webhook call failed: %s", err.Error())
+	default:
+		return nil
+	}
 }