vendor: Update vendor logic

vendor/k8s.io/apiserver/pkg/server/config.go

@@ -17,12 +17,11 @@ limitations under the License.
 package server
 
 import (
-	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"net"
 	"net/http"
 	goruntime "runtime"
+	"runtime/debug"
 	"sort"
 	"strconv"
 	"strings"
@@ -31,12 +30,12 @@ import (
 
 	jsonpatch "github.com/evanphx/json-patch"
 	"github.com/go-openapi/spec"
-	"github.com/pborman/uuid"
-	"k8s.io/klog"
+	"github.com/google/uuid"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
 	"k8s.io/apimachinery/pkg/version"
@@ -54,17 +53,17 @@ import (
 	genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
 	apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
 	apirequest "k8s.io/apiserver/pkg/endpoints/request"
-	"k8s.io/apiserver/pkg/features"
 	genericregistry "k8s.io/apiserver/pkg/registry/generic"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+	"k8s.io/apiserver/pkg/server/egressselector"
 	genericfilters "k8s.io/apiserver/pkg/server/filters"
 	"k8s.io/apiserver/pkg/server/healthz"
 	"k8s.io/apiserver/pkg/server/routes"
 	serverstore "k8s.io/apiserver/pkg/server/storage"
-	utilfeature "k8s.io/apiserver/pkg/util/feature"
 	"k8s.io/client-go/informers"
 	restclient "k8s.io/client-go/rest"
-	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/component-base/logs"
+	"k8s.io/klog"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 
 	// install apis
@@ -95,6 +94,11 @@ type Config struct {
 	// This is required for proper functioning of the PostStartHooks on a GenericAPIServer
 	// TODO: move into SecureServing(WithLoopback) as soon as insecure serving is gone
 	LoopbackClientConfig *restclient.Config
+
+	// EgressSelector provides a lookup mechanism for dialing outbound connections.
+	// It does so based on a EgressSelectorConfiguration which was read at startup.
+	EgressSelector *egressselector.EgressSelector
+
 	// RuleResolver is required to get the list of rules that apply to a given user
 	// in a given namespace
 	RuleResolver authorizer.RuleResolver
@@ -111,6 +115,8 @@ type Config struct {
 	EnableMetrics             bool
 
 	DisabledPostStartHooks sets.String
+	// done values in this values for this map are ignored.
+	PostStartHooks map[string]PostStartHookConfigEntry
 
 	// Version will enable the /version endpoint if non-nil
 	Version *version.Info
@@ -133,8 +139,12 @@ type Config struct {
 	// DiscoveryAddresses is used to build the IPs pass to discovery. If nil, the ExternalAddress is
 	// always reported
 	DiscoveryAddresses discovery.Addresses
-	// The default set of healthz checks. There might be more added via AddHealthzChecks dynamically.
-	HealthzChecks []healthz.HealthzChecker
+	// The default set of healthz checks. There might be more added via AddHealthChecks dynamically.
+	HealthzChecks []healthz.HealthChecker
+	// The default set of livez checks. There might be more added via AddHealthChecks dynamically.
+	LivezChecks []healthz.HealthChecker
+	// The default set of readyz-only checks. There might be more added via AddReadyzChecks dynamically.
+	ReadyzChecks []healthz.HealthChecker
 	// LegacyAPIGroupPrefixes is used to set up URL parsing for authorization and for validating requests
 	// to InstallLegacyAPIGroup. New API servers don't generally have legacy groups at all.
 	LegacyAPIGroupPrefixes sets.String
@@ -156,6 +166,17 @@ type Config struct {
 	// If specified, long running requests such as watch will be allocated a random timeout between this value, and
 	// twice this value.  Note that it is up to the request handlers to ignore or honor this timeout. In seconds.
 	MinRequestTimeout int
+
+	// This represents the maximum amount of time it should take for apiserver to complete its startup
+	// sequence and become healthy. From apiserver's start time to when this amount of time has
+	// elapsed, /livez will assume that unfinished post-start hooks will complete successfully and
+	// therefore return true.
+	LivezGracePeriod time.Duration
+	// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
+	// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
+	// but /readyz will return failure.
+	ShutdownDelayDuration time.Duration
+
 	// The limit on the total size increase all "copy" operations in a json
 	// patch may cause.
 	// This affects all places that applies json patch in the binary.
@@ -172,10 +193,6 @@ type Config struct {
 	// Predicate which is true for paths of long-running http requests
 	LongRunningFunc apirequest.LongRunningRequestCheck
 
-	// EnableAPIResponseCompression indicates whether API Responses should support compression
-	// if the client requests it via Accept-Encoding
-	EnableAPIResponseCompression bool
-
 	// MergedResourceConfig indicates which groupVersion enabled and its resources enabled/disabled.
 	// This is composed of genericapiserver defaultAPIResourceConfig and those parsed from flags.
 	// If not specify any in flags, then genericapiserver will only enable defaultAPIResourceConfig.
@@ -215,13 +232,13 @@ type SecureServingInfo struct {
 
 	// Cert is the main server cert which is used if SNI does not match. Cert must be non-nil and is
 	// allowed to be in SNICerts.
-	Cert *tls.Certificate
+	Cert dynamiccertificates.CertKeyContentProvider
 
-	// SNICerts are the TLS certificates by name used for SNI.
-	SNICerts map[string]*tls.Certificate
+	// SNICerts are the TLS certificates used for SNI.
+	SNICerts []dynamiccertificates.SNICertKeyContentProvider
 
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
-	ClientCA *x509.CertPool
+	ClientCA dynamiccertificates.CAContentProvider
 
 	// MinTLSVersion optionally overrides the minimum TLS version supported.
 	// Values are from tls package constants (https://golang.org/pkg/crypto/tls/#pkg-constants).
@@ -234,6 +251,9 @@ type SecureServingInfo struct {
 	// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
 	// A value of zero means to use the default provided by golang's HTTP/2 support.
 	HTTP2MaxStreamsPerConnection int
+
+	// DisableHTTP2 indicates that http2 should not be enabled.
+	DisableHTTP2 bool
 }
 
 type AuthenticationInfo struct {
@@ -256,13 +276,17 @@ type AuthorizationInfo struct {
 
 // NewConfig returns a Config struct with the default values
 func NewConfig(codecs serializer.CodecFactory) *Config {
+	defaultHealthChecks := []healthz.HealthChecker{healthz.PingHealthz, healthz.LogHealthz}
 	return &Config{
 		Serializer:                  codecs,
 		BuildHandlerChainFunc:       DefaultBuildHandlerChain,
 		HandlerChainWaitGroup:       new(utilwaitgroup.SafeWaitGroup),
 		LegacyAPIGroupPrefixes:      sets.NewString(DefaultLegacyAPIPrefix),
 		DisabledPostStartHooks:      sets.NewString(),
-		HealthzChecks:               []healthz.HealthzChecker{healthz.PingHealthz, healthz.LogHealthz},
+		PostStartHooks:              map[string]PostStartHookConfigEntry{},
+		HealthzChecks:               append([]healthz.HealthChecker{}, defaultHealthChecks...),
+		ReadyzChecks:                append([]healthz.HealthChecker{}, defaultHealthChecks...),
+		LivezChecks:                 append([]healthz.HealthChecker{}, defaultHealthChecks...),
 		EnableIndex:                 true,
 		EnableDiscovery:             true,
 		EnableProfiling:             true,
@@ -271,7 +295,9 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 		MaxMutatingRequestsInFlight: 200,
 		RequestTimeout:              time.Duration(60) * time.Second,
 		MinRequestTimeout:           1800,
-		// 1.5MB is the recommended client request size in byte
+		LivezGracePeriod:            time.Duration(0),
+		ShutdownDelayDuration:       time.Duration(0),
+		// 1.5MB is the default client request size in bytes
 		// the etcd server should accept. See
 		// https://github.com/etcd-io/etcd/blob/release-3.4/embed/config.go#L56.
 		// A request body might be encoded in json, and is converted to
@@ -284,8 +310,7 @@ func NewConfig(codecs serializer.CodecFactory) *Config {
 		// A request body might be encoded in json, and is converted to
 		// proto when persisted in etcd, so we allow 2x as the largest request
 		// body size to be accepted and decoded in a write request.
-		MaxRequestBodyBytes:          int64(3 * 1024 * 1024),
-		EnableAPIResponseCompression: utilfeature.DefaultFeatureGate.Enabled(features.APIResponseCompression),
+		MaxRequestBodyBytes: int64(3 * 1024 * 1024),
 
 		// Default to treating watch as a long-running operation
 		// Generic API servers have no inherent long-running subresources
@@ -320,22 +345,19 @@ func DefaultOpenAPIConfig(getDefinitions openapicommon.GetOpenAPIDefinitions, de
 	}
 }
 
-func (c *AuthenticationInfo) ApplyClientCert(clientCAFile string, servingInfo *SecureServingInfo) error {
-	if servingInfo != nil {
-		if len(clientCAFile) > 0 {
-			clientCAs, err := certutil.CertsFromFile(clientCAFile)
-			if err != nil {
-				return fmt.Errorf("unable to load client CA file: %v", err)
-			}
-			if servingInfo.ClientCA == nil {
-				servingInfo.ClientCA = x509.NewCertPool()
-			}
-			for _, cert := range clientCAs {
-				servingInfo.ClientCA.AddCert(cert)
-			}
-		}
+func (c *AuthenticationInfo) ApplyClientCert(clientCA dynamiccertificates.CAContentProvider, servingInfo *SecureServingInfo) error {
+	if servingInfo == nil {
+		return nil
+	}
+	if clientCA == nil {
+		return nil
+	}
+	if servingInfo.ClientCA == nil {
+		servingInfo.ClientCA = clientCA
+		return nil
 	}
 
+	servingInfo.ClientCA = dynamiccertificates.NewUnionCAContentProvider(servingInfo.ClientCA, clientCA)
 	return nil
 }
 
@@ -355,6 +377,47 @@ type CompletedConfig struct {
 	*completedConfig
 }
 
+// AddHealthChecks adds a health check to our config to be exposed by the health endpoints
+// of our configured apiserver. We should prefer this to adding healthChecks directly to
+// the config unless we explicitly want to add a healthcheck only to a specific health endpoint.
+func (c *Config) AddHealthChecks(healthChecks ...healthz.HealthChecker) {
+	for _, check := range healthChecks {
+		c.HealthzChecks = append(c.HealthzChecks, check)
+		c.LivezChecks = append(c.LivezChecks, check)
+		c.ReadyzChecks = append(c.ReadyzChecks, check)
+	}
+}
+
+// AddPostStartHook allows you to add a PostStartHook that will later be added to the server itself in a New call.
+// Name conflicts will cause an error.
+func (c *Config) AddPostStartHook(name string, hook PostStartHookFunc) error {
+	if len(name) == 0 {
+		return fmt.Errorf("missing name")
+	}
+	if hook == nil {
+		return fmt.Errorf("hook func may not be nil: %q", name)
+	}
+	if c.DisabledPostStartHooks.Has(name) {
+		klog.V(1).Infof("skipping %q because it was explicitly disabled", name)
+		return nil
+	}
+
+	if postStartHook, exists := c.PostStartHooks[name]; exists {
+		// this is programmer error, but it can be hard to debug
+		return fmt.Errorf("unable to add %q because it was already registered by: %s", name, postStartHook.originatingStack)
+	}
+	c.PostStartHooks[name] = PostStartHookConfigEntry{hook: hook, originatingStack: string(debug.Stack())}
+
+	return nil
+}
+
+// AddPostStartHookOrDie allows you to add a PostStartHook, but dies on failure.
+func (c *Config) AddPostStartHookOrDie(name string, hook PostStartHookFunc) {
+	if err := c.AddPostStartHook(name, hook); err != nil {
+		klog.Fatalf("Error registering PostStartHook %q: %v", name, err)
+	}
+}
+
 // Complete fills in any fields not set that are required to have valid data and can be derived
 // from other fields. If you're going to `ApplyOptions`, do that first. It's mutating the receiver.
 func (c *Config) Complete(informers informers.SharedInformerFactory) CompletedConfig {
@@ -475,11 +538,11 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		EquivalentResourceRegistry: c.EquivalentResourceRegistry,
 		HandlerChainWaitGroup:      c.HandlerChainWaitGroup,
 
-		minRequestTimeout: time.Duration(c.MinRequestTimeout) * time.Second,
-		ShutdownTimeout:   c.RequestTimeout,
-
-		SecureServingInfo: c.SecureServing,
-		ExternalAddress:   c.ExternalAddress,
+		minRequestTimeout:     time.Duration(c.MinRequestTimeout) * time.Second,
+		ShutdownTimeout:       c.RequestTimeout,
+		ShutdownDelayDuration: c.ShutdownDelayDuration,
+		SecureServingInfo:     c.SecureServing,
+		ExternalAddress:       c.ExternalAddress,
 
 		Handler: apiServerHandler,
 
@@ -491,12 +554,16 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		preShutdownHooks:       map[string]preShutdownHookEntry{},
 		disabledPostStartHooks: c.DisabledPostStartHooks,
 
-		healthzChecks: c.HealthzChecks,
+		healthzChecks:    c.HealthzChecks,
+		livezChecks:      c.LivezChecks,
+		readyzChecks:     c.ReadyzChecks,
+		readinessStopCh:  make(chan struct{}),
+		livezGracePeriod: c.LivezGracePeriod,
 
 		DiscoveryGroupManager: discovery.NewRootAPIsHandler(c.DiscoveryAddresses, c.Serializer),
 
-		enableAPIResponseCompression: c.EnableAPIResponseCompression,
-		maxRequestBodyBytes:          c.MaxRequestBodyBytes,
+		maxRequestBodyBytes: c.MaxRequestBodyBytes,
+		livezClock:          clock.RealClock{},
 	}
 
 	for {
@@ -512,6 +579,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		}
 	}
 
+	// first add poststarthooks from delegated targets
 	for k, v := range delegationTarget.PostStartHooks() {
 		s.postStartHooks[k] = v
 	}
@@ -520,6 +588,13 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		s.preShutdownHooks[k] = v
 	}
 
+	// add poststarthooks that were preconfigured.  Using the add method will give us an error if the same name has already been registered.
+	for name, preconfiguredPostStartHook := range c.PostStartHooks {
+		if err := s.AddPostStartHook(name, preconfiguredPostStartHook.hook); err != nil {
+			return nil, err
+		}
+	}
+
 	genericApiServerHookName := "generic-apiserver-start-informers"
 	if c.SharedInformerFactory != nil && !s.isPostStartHookRegistered(genericApiServerHookName) {
 		err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
@@ -542,8 +617,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
 		if skip {
 			continue
 		}
-
-		s.healthzChecks = append(s.healthzChecks, delegateCheck)
+		s.AddHealthChecks(delegateCheck)
 	}
 
 	s.listedPathProvider = routes.ListedPathProviders{s.listedPathProvider, delegationTarget}
@@ -644,6 +718,7 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
 	}
 	if authn == nil || authz == nil {
 		// prevent nil pointer panic
+		return
 	}
 	if authn.Authenticator == nil || authz.Authorizer == nil {
 		// authenticator or authorizer might be nil if we want to bypass authz/authn
@@ -652,7 +727,7 @@ func AuthorizeClientBearerToken(loopback *restclient.Config, authn *Authenticati
 	}
 
 	privilegedLoopbackToken := loopback.BearerToken
-	var uid = uuid.NewRandom().String()
+	var uid = uuid.New().String()
 	tokens := make(map[string]*user.DefaultInfo)
 	tokens[privilegedLoopbackToken] = &user.DefaultInfo{
 		Name:   user.APIServerUser,

vendor/k8s.io/apiserver/pkg/server/config_selfclient.go

@@ -21,6 +21,7 @@ import (
 	"net"
 
 	restclient "k8s.io/client-go/rest"
+	netutils "k8s.io/utils/net"
 )
 
 // LoopbackClientServerNameOverride is passed to the apiserver from the loopback client in order to
@@ -70,23 +71,27 @@ func LoopbackHostPort(bindAddress string) (string, string, error) {
 		return "", "", fmt.Errorf("invalid server bind address: %q", bindAddress)
 	}
 
-	isIPv6 := net.ParseIP(host).To4() == nil
+	isIPv6 := netutils.IsIPv6String(host)
 
 	// Value is expected to be an IP or DNS name, not "0.0.0.0".
 	if host == "0.0.0.0" || host == "::" {
-		host = "localhost"
 		// Get ip of local interface, but fall back to "localhost".
 		// Note that "localhost" is resolved with the external nameserver first with Go's stdlib.
 		// So if localhost.<yoursearchdomain> resolves, we don't get a 127.0.0.1 as expected.
-		addrs, err := net.InterfaceAddrs()
-		if err == nil {
-			for _, address := range addrs {
-				if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && isIPv6 == (ipnet.IP.To4() == nil) {
-					host = ipnet.IP.String()
-					break
-				}
-			}
-		}
+		host = getLoopbackAddress(isIPv6)
 	}
 	return host, port, nil
 }
+
+// getLoopbackAddress returns the ip address of local loopback interface. If any error occurs or loopback interface is not found, will fall back to "localhost"
+func getLoopbackAddress(wantIPv6 bool) string {
+	addrs, err := net.InterfaceAddrs()
+	if err == nil {
+		for _, address := range addrs {
+			if ipnet, ok := address.(*net.IPNet); ok && ipnet.IP.IsLoopback() && wantIPv6 == netutils.IsIPv6(ipnet.IP) {
+				return ipnet.IP.String()
+			}
+		}
+	}
+	return "localhost"
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/cert_key.go

@@ -0,0 +1,74 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+)
+
+// CertKeyContentProvider provides a certificate and matching private key
+type CertKeyContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCertKeyContent provides cert and key byte content
+	CurrentCertKeyContent() ([]byte, []byte)
+}
+
+// SNICertKeyContentProvider provides a certificate and matching private key as well as optional explicit names
+type SNICertKeyContentProvider interface {
+	CertKeyContentProvider
+	// SNINames provides names used for SNI. May return nil.
+	SNINames() []string
+}
+
+// certKeyContent holds the content for the cert and key
+type certKeyContent struct {
+	cert []byte
+	key  []byte
+}
+
+func (c *certKeyContent) Equal(rhs *certKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.key, rhs.key) && bytes.Equal(c.cert, rhs.cert)
+}
+
+// sniCertKeyContent holds the content for the cert and key as well as any explicit names
+type sniCertKeyContent struct {
+	certKeyContent
+	sniNames []string
+}
+
+func (c *sniCertKeyContent) Equal(rhs *sniCertKeyContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if len(c.sniNames) != len(rhs.sniNames) {
+		return false
+	}
+
+	for i := range c.sniNames {
+		if c.sniNames[i] != rhs.sniNames[i] {
+			return false
+		}
+	}
+
+	return c.certKeyContent.Equal(&rhs.certKeyContent)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/client_ca.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+)
+
+// CAContentProvider provides ca bundle byte content
+type CAContentProvider interface {
+	// Name is just an identifier
+	Name() string
+	// CurrentCABundleContent provides ca bundle byte content.  Errors can be contained to the controllers initializing
+	// the value.  By the time you get here, you should always be returning a value that won't fail.
+	CurrentCABundleContent() []byte
+	// VerifyOptions provides VerifyOptions for authenticators
+	VerifyOptions() (x509.VerifyOptions, bool)
+}
+
+// dynamicCertificateContent holds the content that overrides the baseTLSConfig
+type dynamicCertificateContent struct {
+	// clientCA holds the content for the clientCA bundle
+	clientCA    caBundleContent
+	servingCert certKeyContent
+	sniCerts    []sniCertKeyContent
+}
+
+// caBundleContent holds the content for the clientCA bundle.  Wrapping the bytes makes the Equals work nicely with the
+// method receiver.
+type caBundleContent struct {
+	caBundle []byte
+}
+
+func (c *dynamicCertificateContent) Equal(rhs *dynamicCertificateContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	if !c.clientCA.Equal(&rhs.clientCA) {
+		return false
+	}
+
+	if !c.servingCert.Equal(&rhs.servingCert) {
+		return false
+	}
+
+	if len(c.sniCerts) != len(rhs.sniCerts) {
+		return false
+	}
+
+	for i := range c.sniCerts {
+		if !c.sniCerts[i].Equal(&rhs.sniCerts[i]) {
+			return false
+		}
+	}
+
+	return true
+}
+
+func (c *caBundleContent) Equal(rhs *caBundleContent) bool {
+	if c == nil || rhs == nil {
+		return c == rhs
+	}
+
+	return bytes.Equal(c.caBundle, rhs.caBundle)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/configmap_cafile_content.go

@@ -0,0 +1,277 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/fields"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	corev1informers "k8s.io/client-go/informers/core/v1"
+	"k8s.io/client-go/kubernetes"
+	corev1listers "k8s.io/client-go/listers/core/v1"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes
+// It also fulfills the authenticator interface to provide verifyoptions
+type ConfigMapCAController struct {
+	name string
+
+	configmapLister    corev1listers.ConfigMapLister
+	configmapNamespace string
+	configmapName      string
+	configmapKey       string
+	// configMapInformer is tracked so that we can start these on Run
+	configMapInformer cache.SharedIndexInformer
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	queue workqueue.RateLimitingInterface
+	// preRunCaches are the caches to sync before starting the work of this control loop
+	preRunCaches []cache.InformerSynced
+}
+
+var _ Notifier = &ConfigMapCAController{}
+var _ CAContentProvider = &ConfigMapCAController{}
+var _ ControllerRunner = &ConfigMapCAController{}
+
+// NewDynamicCAFromConfigMapController returns a CAContentProvider based on a configmap that automatically reloads content.
+// It is near-realtime via an informer.
+func NewDynamicCAFromConfigMapController(purpose, namespace, name, key string, kubeClient kubernetes.Interface) (*ConfigMapCAController, error) {
+	if len(purpose) == 0 {
+		return nil, fmt.Errorf("missing purpose for ca bundle")
+	}
+	if len(namespace) == 0 {
+		return nil, fmt.Errorf("missing namespace for ca bundle")
+	}
+	if len(name) == 0 {
+		return nil, fmt.Errorf("missing name for ca bundle")
+	}
+	if len(key) == 0 {
+		return nil, fmt.Errorf("missing key for ca bundle")
+	}
+	caContentName := fmt.Sprintf("%s::%s::%s::%s", purpose, namespace, name, key)
+
+	// we construct our own informer because we need such a small subset of the information available.  Just one namespace.
+	uncastConfigmapInformer := corev1informers.NewFilteredConfigMapInformer(kubeClient, namespace, 12*time.Hour, cache.Indexers{cache.NamespaceIndex: cache.MetaNamespaceIndexFunc}, func(listOptions *v1.ListOptions) {
+		listOptions.FieldSelector = fields.OneTermEqualSelector("metadata.name", name).String()
+	})
+
+	configmapLister := corev1listers.NewConfigMapLister(uncastConfigmapInformer.GetIndexer())
+
+	c := &ConfigMapCAController{
+		name:               caContentName,
+		configmapNamespace: namespace,
+		configmapName:      name,
+		configmapKey:       key,
+		configmapLister:    configmapLister,
+		configMapInformer:  uncastConfigmapInformer,
+
+		queue:        workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicConfigMapCABundle-%s", purpose)),
+		preRunCaches: []cache.InformerSynced{uncastConfigmapInformer.HasSynced},
+	}
+	if err := c.loadCABundle(); err != nil {
+		// don't fail, but do print out a message
+		klog.Warningf("unable to load initial CA bundle for: %q due to: %s", c.name, err)
+	}
+	uncastConfigmapInformer.AddEventHandler(cache.FilteringResourceEventHandler{
+		FilterFunc: func(obj interface{}) bool {
+			if cast, ok := obj.(*corev1.ConfigMap); ok {
+				return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+			}
+			if tombstone, ok := obj.(cache.DeletedFinalStateUnknown); ok {
+				if cast, ok := tombstone.Obj.(*corev1.ConfigMap); ok {
+					return cast.Name == c.configmapName && cast.Namespace == c.configmapNamespace
+				}
+			}
+			return true // always return true just in case.  The checks are fairly cheap
+		},
+		Handler: cache.ResourceEventHandlerFuncs{
+			// we have a filter, so any time we're called, we may as well queue. We only ever check one configmap
+			// so we don't have to be choosy about our key.
+			AddFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			UpdateFunc: func(oldObj, newObj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+			DeleteFunc: func(obj interface{}) {
+				c.queue.Add(c.keyFn())
+			},
+		},
+	})
+
+	return c, nil
+}
+
+func (c *ConfigMapCAController) keyFn() string {
+	// this format matches DeletionHandlingMetaNamespaceKeyFunc for our single key
+	return c.configmapNamespace + "/" + c.configmapName
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *ConfigMapCAController) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *ConfigMapCAController) loadCABundle() error {
+	configMap, err := c.configmapLister.ConfigMaps(c.configmapNamespace).Get(c.configmapName)
+	if err != nil {
+		return err
+	}
+	caBundle := configMap.Data[c.configmapKey]
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged([]byte(caBundle)) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), []byte(caBundle))
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *ConfigMapCAController) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *ConfigMapCAController) RunOnce() error {
+	// Ignore the error when running once because when using a dynamically loaded ca file, because we think it's better to have nothing for
+	// a brief time than completely crash.  If crashing is necessary, higher order logic like a healthcheck and cause failures.
+	_ = c.loadCABundle()
+	return nil
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *ConfigMapCAController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// we have a personal informer that is narrowly scoped, start it.
+	go c.configMapInformer.Run(stopCh)
+
+	// wait for your secondary caches to fill before starting your work
+	if !cache.WaitForNamedCacheSync(c.name, stopCh, c.preRunCaches...) {
+		return
+	}
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	<-stopCh
+}
+
+func (c *ConfigMapCAController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *ConfigMapCAController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *ConfigMapCAController) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *ConfigMapCAController) CurrentCABundleContent() []byte {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return nil // this can happen if we've been unable load data from the apiserver for some reason
+	}
+
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *ConfigMapCAController) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		// This can happen if we've been unable load data from the apiserver for some reason.
+		// In this case, we should not accept any connections on the basis of this ca bundle.
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_cafile_content.go

@@ -0,0 +1,254 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	"k8s.io/client-go/util/cert"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.
+var FileRefreshDuration = 1 * time.Minute
+
+// Listener is an interface to use to notify interested parties of a change.
+type Listener interface {
+	// Enqueue should be called when an input may have changed
+	Enqueue()
+}
+
+// Notifier is a way to add listeners
+type Notifier interface {
+	// AddListener is adds a listener to be notified of potential input changes
+	AddListener(listener Listener)
+}
+
+// ControllerRunner is a generic interface for starting a controller
+type ControllerRunner interface {
+	// RunOnce runs the sync loop a single time.  This useful for synchronous priming
+	RunOnce() error
+
+	// Run should be called a go .Run
+	Run(workers int, stopCh <-chan struct{})
+}
+
+// DynamicFileCAContent provies a CAContentProvider that can dynamically react to new file content
+// It also fulfills the authenticator interface to provide verifyoptions
+type DynamicFileCAContent struct {
+	name string
+
+	// filename is the name the file to read.
+	filename string
+
+	// caBundle is a caBundleAndVerifier that contains the last read, non-zero length content of the file
+	caBundle atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileCAContent{}
+var _ CAContentProvider = &DynamicFileCAContent{}
+var _ ControllerRunner = &DynamicFileCAContent{}
+
+type caBundleAndVerifier struct {
+	caBundle      []byte
+	verifyOptions x509.VerifyOptions
+}
+
+// NewDynamicCAContentFromFile returns a CAContentProvider based on a filename that automatically reloads content
+func NewDynamicCAContentFromFile(purpose, filename string) (*DynamicFileCAContent, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+	name := fmt.Sprintf("%s::%s", purpose, filename)
+
+	ret := &DynamicFileCAContent{
+		name:     name,
+		filename: filename,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadCABundle(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c *DynamicFileCAContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadCABundle determines the next set of content for the file.
+func (c *DynamicFileCAContent) loadCABundle() error {
+	caBundle, err := ioutil.ReadFile(c.filename)
+	if err != nil {
+		return err
+	}
+	if len(caBundle) == 0 {
+		return fmt.Errorf("missing content for CA bundle %q", c.Name())
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	if !c.hasCAChanged(caBundle) {
+		return nil
+	}
+
+	caBundleAndVerifier, err := newCABundleAndVerifier(c.Name(), caBundle)
+	if err != nil {
+		return err
+	}
+	c.caBundle.Store(caBundleAndVerifier)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// hasCAChanged returns true if the caBundle is different than the current.
+func (c *DynamicFileCAContent) hasCAChanged(caBundle []byte) bool {
+	uncastExisting := c.caBundle.Load()
+	if uncastExisting == nil {
+		return true
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := uncastExisting.(*caBundleAndVerifier)
+	if !ok {
+		return true
+	}
+	if !bytes.Equal(existing.caBundle, caBundle) {
+		return true
+	}
+
+	return false
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileCAContent) RunOnce() error {
+	return c.loadCABundle()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicFileCAContent) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	// TODO this can be wired to an fsnotifier as well.
+
+	<-stopCh
+}
+
+func (c *DynamicFileCAContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileCAContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadCABundle()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *DynamicFileCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.Load().(*caBundleAndVerifier).caBundle
+}
+
+// VerifyOptions provides verifyoptions compatible with authenticators
+func (c *DynamicFileCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	uncastObj := c.caBundle.Load()
+	if uncastObj == nil {
+		return x509.VerifyOptions{}, false
+	}
+
+	return uncastObj.(*caBundleAndVerifier).verifyOptions, true
+}
+
+// newVerifyOptions creates a new verification func from a file.  It reads the content and then fails.
+// It will return a nil function if you pass an empty CA file.
+func newCABundleAndVerifier(name string, caBundle []byte) (*caBundleAndVerifier, error) {
+	if len(caBundle) == 0 {
+		return nil, fmt.Errorf("missing content for CA bundle %q", name)
+	}
+
+	// Wrap with an x509 verifier
+	var err error
+	verifyOptions := defaultVerifyOptions()
+	verifyOptions.Roots, err = cert.NewPoolFromBytes(caBundle)
+	if err != nil {
+		return nil, fmt.Errorf("error loading CA bundle for %q: %v", name, err)
+	}
+
+	return &caBundleAndVerifier{
+		caBundle:      caBundle,
+		verifyOptions: verifyOptions,
+	}, nil
+}
+
+// defaultVerifyOptions returns VerifyOptions that use the system root certificates, current time,
+// and requires certificates to be valid for client auth (x509.ExtKeyUsageClientAuth)
+func defaultVerifyOptions() x509.VerifyOptions {
+	return x509.VerifyOptions{
+		KeyUsages: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth},
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_serving_content.go

@@ -0,0 +1,179 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"fmt"
+	"io/ioutil"
+	"sync/atomic"
+	"time"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+// DynamicFileServingContent provides a CertKeyContentProvider that can dynamically react to new file content
+type DynamicFileServingContent struct {
+	name string
+
+	// certFile is the name of the certificate file to read.
+	certFile string
+	// keyFile is the name of the key file to read.
+	keyFile string
+
+	// servingCert is a certKeyContent that contains the last read, non-zero length content of the key and cert
+	servingCert atomic.Value
+
+	listeners []Listener
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue workqueue.RateLimitingInterface
+}
+
+var _ Notifier = &DynamicFileServingContent{}
+var _ CertKeyContentProvider = &DynamicFileServingContent{}
+var _ ControllerRunner = &DynamicFileServingContent{}
+
+// NewDynamicServingContentFromFiles returns a dynamic CertKeyContentProvider based on a cert and key filename
+func NewDynamicServingContentFromFiles(purpose, certFile, keyFile string) (*DynamicFileServingContent, error) {
+	if len(certFile) == 0 || len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for serving cert")
+	}
+	name := fmt.Sprintf("%s::%s::%s", purpose, certFile, keyFile)
+
+	ret := &DynamicFileServingContent{
+		name:     name,
+		certFile: certFile,
+		keyFile:  keyFile,
+		queue:    workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), fmt.Sprintf("DynamicCABundle-%s", purpose)),
+	}
+	if err := ret.loadServingCert(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// AddListener adds a listener to be notified when the serving cert content changes.
+func (c *DynamicFileServingContent) AddListener(listener Listener) {
+	c.listeners = append(c.listeners, listener)
+}
+
+// loadServingCert determines the next set of content for the file.
+func (c *DynamicFileServingContent) loadServingCert() error {
+	cert, err := ioutil.ReadFile(c.certFile)
+	if err != nil {
+		return err
+	}
+	key, err := ioutil.ReadFile(c.keyFile)
+	if err != nil {
+		return err
+	}
+	if len(cert) == 0 || len(key) == 0 {
+		return fmt.Errorf("missing content for serving cert %q", c.Name())
+	}
+
+	// Ensure that the key matches the cert and both are valid
+	_, err = tls.X509KeyPair(cert, key)
+	if err != nil {
+		return err
+	}
+
+	newCertKey := &certKeyContent{
+		cert: cert,
+		key:  key,
+	}
+
+	// check to see if we have a change. If the values are the same, do nothing.
+	existing, ok := c.servingCert.Load().(*certKeyContent)
+	if ok && existing != nil && existing.Equal(newCertKey) {
+		return nil
+	}
+
+	c.servingCert.Store(newCertKey)
+
+	for _, listener := range c.listeners {
+		listener.Enqueue()
+	}
+
+	return nil
+}
+
+// RunOnce runs a single sync loop
+func (c *DynamicFileServingContent) RunOnce() error {
+	return c.loadServingCert()
+}
+
+// Run starts the controller and blocks until stopCh is closed.
+func (c *DynamicFileServingContent) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting %s", c.name)
+	defer klog.Infof("Shutting down %s", c.name)
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	_ = wait.PollImmediateUntil(FileRefreshDuration, func() (bool, error) {
+		c.queue.Add(workItemKey)
+		return false, nil
+	}, stopCh)
+
+	// TODO this can be wired to an fsnotifier as well.
+
+	<-stopCh
+}
+
+func (c *DynamicFileServingContent) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicFileServingContent) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.loadServingCert()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Name is just an identifier
+func (c *DynamicFileServingContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides serving cert byte content
+func (c *DynamicFileServingContent) CurrentCertKeyContent() ([]byte, []byte) {
+	certKeyContent := c.servingCert.Load().(*certKeyContent)
+	return certKeyContent.cert, certKeyContent.key
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/dynamic_sni_content.go

@@ -0,0 +1,50 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+// DynamicFileSNIContent provides a SNICertKeyContentProvider that can dynamically react to new file content
+type DynamicFileSNIContent struct {
+	*DynamicFileServingContent
+	sniNames []string
+}
+
+var _ Notifier = &DynamicFileSNIContent{}
+var _ SNICertKeyContentProvider = &DynamicFileSNIContent{}
+var _ ControllerRunner = &DynamicFileSNIContent{}
+
+// NewDynamicSNIContentFromFiles returns a dynamic SNICertKeyContentProvider based on a cert and key filename and explicit names
+func NewDynamicSNIContentFromFiles(purpose, certFile, keyFile string, sniNames ...string) (*DynamicFileSNIContent, error) {
+	servingContent, err := NewDynamicServingContentFromFiles(purpose, certFile, keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	ret := &DynamicFileSNIContent{
+		DynamicFileServingContent: servingContent,
+		sniNames:                  sniNames,
+	}
+	if err := ret.loadServingCert(); err != nil {
+		return nil, err
+	}
+
+	return ret, nil
+}
+
+// SNINames returns explicitly set SNI names for the certificate. These are not dynamic.
+func (c *DynamicFileSNIContent) SNINames() []string {
+	return c.sniNames
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/named_certificates.go

@@ -0,0 +1,89 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"strings"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/klog"
+)
+
+// BuildNamedCertificates returns a map of *tls.Certificate by name. It's
+// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
+// is invalid. Returns nil if len(certs) == 0
+func (c *DynamicServingCertificateController) BuildNamedCertificates(sniCerts []sniCertKeyContent) (map[string]*tls.Certificate, error) {
+	nameToCertificate := map[string]*tls.Certificate{}
+	byNameExplicit := map[string]*tls.Certificate{}
+
+	// Iterate backwards so that earlier certs take precedence in the names map
+	for i := len(sniCerts) - 1; i >= 0; i-- {
+		cert, err := tls.X509KeyPair(sniCerts[i].cert, sniCerts[i].key)
+		if err != nil {
+			return nil, fmt.Errorf("invalid SNI cert keypair [%d/%q]: %v", i, c.sniCerts[i].Name(), err)
+		}
+
+		// error is not possible given above call to X509KeyPair
+		x509Cert, _ := x509.ParseCertificate(cert.Certificate[0])
+
+		names := sniCerts[i].sniNames
+		for _, name := range names {
+			byNameExplicit[name] = &cert
+		}
+
+		klog.V(2).Infof("loaded SNI cert [%d/%q]: %s", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "SNICertificateReload", "loaded SNI cert [%d/%q]: %s with explicit names %v", i, c.sniCerts[i].Name(), GetHumanCertDetail(x509Cert), names)
+		}
+
+		if len(names) == 0 {
+			names = getCertificateNames(x509Cert)
+			for _, name := range names {
+				nameToCertificate[name] = &cert
+			}
+		}
+	}
+
+	// Explicitly set names must override
+	for k, v := range byNameExplicit {
+		nameToCertificate[k] = v
+	}
+
+	return nameToCertificate, nil
+}
+
+// getCertificateNames returns names for an x509.Certificate. The names are
+// suitable for use in tls.Config#NamedCertificates.
+func getCertificateNames(cert *x509.Certificate) []string {
+	var names []string
+
+	cn := cert.Subject.CommonName
+	if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
+		names = append(names, cn)
+	}
+	for _, san := range cert.DNSNames {
+		names = append(names, san)
+	}
+	// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
+	// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
+
+	return names
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/static_content.go

@@ -0,0 +1,171 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+)
+
+type staticCAContent struct {
+	name     string
+	caBundle *caBundleAndVerifier
+}
+
+var _ CAContentProvider = &staticCAContent{}
+
+// NewStaticCAContentFromFile returns a CAContentProvider based on a filename
+func NewStaticCAContentFromFile(filename string) (CAContentProvider, error) {
+	if len(filename) == 0 {
+		return nil, fmt.Errorf("missing filename for ca bundle")
+	}
+
+	caBundle, err := ioutil.ReadFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	return NewStaticCAContent(filename, caBundle)
+}
+
+// NewStaticCAContent returns a CAContentProvider that always returns the same value
+func NewStaticCAContent(name string, caBundle []byte) (CAContentProvider, error) {
+	caBundleAndVerifier, err := newCABundleAndVerifier(name, caBundle)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCAContent{
+		name:     name,
+		caBundle: caBundleAndVerifier,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCAContent) Name() string {
+	return c.name
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c *staticCAContent) CurrentCABundleContent() (cabundle []byte) {
+	return c.caBundle.caBundle
+}
+
+func (c *staticCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	return c.caBundle.verifyOptions, true
+}
+
+type staticCertKeyContent struct {
+	name string
+	cert []byte
+	key  []byte
+}
+
+type staticSNICertKeyContent struct {
+	staticCertKeyContent
+	sniNames []string
+}
+
+// NewStaticCertKeyContentFromFiles returns a CertKeyContentProvider based on a filename
+func NewStaticCertKeyContentFromFiles(certFile, keyFile string) (CertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticCertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock)
+}
+
+// NewStaticSNICertKeyContentFromFiles returns a SNICertKeyContentProvider based on a filename
+func NewStaticSNICertKeyContentFromFiles(certFile, keyFile string, sniNames ...string) (SNICertKeyContentProvider, error) {
+	if len(certFile) == 0 {
+		return nil, fmt.Errorf("missing filename for certificate")
+	}
+	if len(keyFile) == 0 {
+		return nil, fmt.Errorf("missing filename for key")
+	}
+
+	certPEMBlock, err := ioutil.ReadFile(certFile)
+	if err != nil {
+		return nil, err
+	}
+	keyPEMBlock, err := ioutil.ReadFile(keyFile)
+	if err != nil {
+		return nil, err
+	}
+
+	return NewStaticSNICertKeyContent(fmt.Sprintf("cert: %s, key: %s", certFile, keyFile), certPEMBlock, keyPEMBlock, sniNames...)
+}
+
+// NewStaticCertKeyContent returns a CertKeyContentProvider that always returns the same value
+func NewStaticCertKeyContent(name string, cert, key []byte) (CertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticCertKeyContent{
+		name: name,
+		cert: cert,
+		key:  key,
+	}, nil
+}
+
+// NewStaticSNICertKeyContent returns a SNICertKeyContentProvider that always returns the same value
+func NewStaticSNICertKeyContent(name string, cert, key []byte, sniNames ...string) (SNICertKeyContentProvider, error) {
+	// Ensure that the key matches the cert and both are valid
+	_, err := tls.X509KeyPair(cert, key)
+	if err != nil {
+		return nil, err
+	}
+
+	return &staticSNICertKeyContent{
+		staticCertKeyContent: staticCertKeyContent{
+			name: name,
+			cert: cert,
+			key:  key,
+		},
+		sniNames: sniNames,
+	}, nil
+}
+
+// Name is just an identifier
+func (c *staticCertKeyContent) Name() string {
+	return c.name
+}
+
+// CurrentCertKeyContent provides cert and key content
+func (c *staticCertKeyContent) CurrentCertKeyContent() ([]byte, []byte) {
+	return c.cert, c.key
+}
+
+func (c *staticSNICertKeyContent) SNINames() []string {
+	return c.sniNames
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/tlsconfig.go

@@ -0,0 +1,263 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/tls"
+	"crypto/x509"
+	"errors"
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	v1 "k8s.io/api/core/v1"
+
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/client-go/tools/events"
+	"k8s.io/client-go/util/cert"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+)
+
+const workItemKey = "key"
+
+// DynamicServingCertificateController dynamically loads certificates and provides a golang tls compatible dynamic GetCertificate func.
+type DynamicServingCertificateController struct {
+	// baseTLSConfig is the static portion of the tlsConfig for serving to clients.  It is copied and the copy is mutated
+	// based on the dynamic cert state.
+	baseTLSConfig tls.Config
+
+	// clientCA provides the very latest content of the ca bundle
+	clientCA CAContentProvider
+	// servingCert provides the very latest content of the default serving certificate
+	servingCert CertKeyContentProvider
+	// sniCerts are a list of CertKeyContentProvider with associated names used for SNI
+	sniCerts []SNICertKeyContentProvider
+
+	// currentlyServedContent holds the original bytes that we are serving. This is used to decide if we need to set a
+	// new atomic value. The types used for efficient TLSConfig preclude using the processed value.
+	currentlyServedContent *dynamicCertificateContent
+	// currentServingTLSConfig holds a *tls.Config that will be used to serve requests
+	currentServingTLSConfig atomic.Value
+
+	// queue only ever has one item, but it has nice error handling backoff/retry semantics
+	queue         workqueue.RateLimitingInterface
+	eventRecorder events.EventRecorder
+}
+
+var _ Listener = &DynamicServingCertificateController{}
+
+// NewDynamicServingCertificateController returns a controller that can be used to keep a TLSConfig up to date.
+func NewDynamicServingCertificateController(
+	baseTLSConfig tls.Config,
+	clientCA CAContentProvider,
+	servingCert CertKeyContentProvider,
+	sniCerts []SNICertKeyContentProvider,
+	eventRecorder events.EventRecorder,
+) *DynamicServingCertificateController {
+	c := &DynamicServingCertificateController{
+		baseTLSConfig: baseTLSConfig,
+		clientCA:      clientCA,
+		servingCert:   servingCert,
+		sniCerts:      sniCerts,
+
+		queue:         workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "DynamicServingCertificateController"),
+		eventRecorder: eventRecorder,
+	}
+
+	return c
+}
+
+// GetConfigForClient is an implementation of tls.Config.GetConfigForClient
+func (c *DynamicServingCertificateController) GetConfigForClient(clientHello *tls.ClientHelloInfo) (*tls.Config, error) {
+	uncastObj := c.currentServingTLSConfig.Load()
+	if uncastObj == nil {
+		return nil, errors.New("dynamiccertificates: configuration not ready")
+	}
+	tlsConfig, ok := uncastObj.(*tls.Config)
+	if !ok {
+		return nil, errors.New("dynamiccertificates: unexpected config type")
+	}
+
+	return tlsConfig.Clone(), nil
+}
+
+// newTLSContent determines the next set of content for overriding the baseTLSConfig.
+func (c *DynamicServingCertificateController) newTLSContent() (*dynamicCertificateContent, error) {
+	newContent := &dynamicCertificateContent{}
+
+	if c.clientCA != nil {
+		currClientCABundle := c.clientCA.CurrentCABundleContent()
+		// we allow removing all client ca bundles because the server is still secure when this happens. it just means
+		// that there isn't a hint to clients about which client-cert to used.  this happens when there is no client-ca
+		// yet known for authentication, which can happen in aggregated apiservers and some kube-apiserver deployment modes.
+		newContent.clientCA = caBundleContent{caBundle: currClientCABundle}
+	}
+
+	if c.servingCert != nil {
+		currServingCert, currServingKey := c.servingCert.CurrentCertKeyContent()
+		if len(currServingCert) == 0 || len(currServingKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty serving certificate from %q", c.servingCert.Name())
+		}
+
+		newContent.servingCert = certKeyContent{cert: currServingCert, key: currServingKey}
+	}
+
+	for i, sniCert := range c.sniCerts {
+		currCert, currKey := sniCert.CurrentCertKeyContent()
+		if len(currCert) == 0 || len(currKey) == 0 {
+			return nil, fmt.Errorf("not loading an empty SNI certificate from %d/%q", i, sniCert.Name())
+		}
+
+		newContent.sniCerts = append(newContent.sniCerts, sniCertKeyContent{certKeyContent: certKeyContent{cert: currCert, key: currKey}, sniNames: sniCert.SNINames()})
+	}
+
+	return newContent, nil
+}
+
+// syncCerts gets newTLSContent, if it has changed from the existing, the content is parsed and stored for usage in
+// GetConfigForClient.
+func (c *DynamicServingCertificateController) syncCerts() error {
+	newContent, err := c.newTLSContent()
+	if err != nil {
+		return err
+	}
+	// if the content is the same as what we currently have, we can simply skip it.  This works because we are single
+	// threaded.  If you ever make this multi-threaded, add a lock.
+	if newContent.Equal(c.currentlyServedContent) {
+		return nil
+	}
+
+	// make a shallow copy and override the dynamic pieces which have changed.
+	newTLSConfigCopy := c.baseTLSConfig.Clone()
+
+	// parse new content to add to TLSConfig
+	if len(newContent.clientCA.caBundle) > 0 {
+		newClientCAPool := x509.NewCertPool()
+		newClientCAs, err := cert.ParseCertsPEM(newContent.clientCA.caBundle)
+		if err != nil {
+			return fmt.Errorf("unable to load client CA file %q: %v", string(newContent.clientCA.caBundle), err)
+		}
+		for i, cert := range newClientCAs {
+			klog.V(2).Infof("loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			if c.eventRecorder != nil {
+				c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "CACertificateReload", "loaded client CA [%d/%q]: %s", i, c.clientCA.Name(), GetHumanCertDetail(cert))
+			}
+
+			newClientCAPool.AddCert(cert)
+		}
+
+		newTLSConfigCopy.ClientCAs = newClientCAPool
+	}
+
+	if len(newContent.servingCert.cert) > 0 && len(newContent.servingCert.key) > 0 {
+		cert, err := tls.X509KeyPair(newContent.servingCert.cert, newContent.servingCert.key)
+		if err != nil {
+			return fmt.Errorf("invalid serving cert keypair: %v", err)
+		}
+
+		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
+		if err != nil {
+			return fmt.Errorf("invalid serving cert: %v", err)
+		}
+
+		klog.V(2).Infof("loaded serving cert [%q]: %s", c.servingCert.Name(), GetHumanCertDetail(x509Cert))
+		if c.eventRecorder != nil {
+			c.eventRecorder.Eventf(nil, nil, v1.EventTypeWarning, "TLSConfigChanged", "ServingCertificateReload", "loaded serving cert [%q]: %s", c.clientCA.Name(), GetHumanCertDetail(x509Cert))
+		}
+
+		newTLSConfigCopy.Certificates = []tls.Certificate{cert}
+	}
+
+	if len(newContent.sniCerts) > 0 {
+		newTLSConfigCopy.NameToCertificate, err = c.BuildNamedCertificates(newContent.sniCerts)
+		if err != nil {
+			return fmt.Errorf("unable to build named certificate map: %v", err)
+		}
+
+		// append all named certs. Otherwise, the go tls stack will think no SNI processing
+		// is necessary because there is only one cert anyway.
+		// Moreover, if servingCert is not set, the first SNI
+		// cert will become the default cert. That's what we expect anyway.
+		for _, sniCert := range newTLSConfigCopy.NameToCertificate {
+			newTLSConfigCopy.Certificates = append(newTLSConfigCopy.Certificates, *sniCert)
+		}
+	}
+
+	// store new values of content for serving.
+	c.currentServingTLSConfig.Store(newTLSConfigCopy)
+	c.currentlyServedContent = newContent // this is single threaded, so we have no locking issue
+
+	return nil
+}
+
+// RunOnce runs a single sync step to ensure that we have a valid starting configuration.
+func (c *DynamicServingCertificateController) RunOnce() error {
+	return c.syncCerts()
+}
+
+// Run starts the kube-apiserver and blocks until stopCh is closed.
+func (c *DynamicServingCertificateController) Run(workers int, stopCh <-chan struct{}) {
+	defer utilruntime.HandleCrash()
+	defer c.queue.ShutDown()
+
+	klog.Infof("Starting DynamicServingCertificateController")
+	defer klog.Infof("Shutting down DynamicServingCertificateController")
+
+	// synchronously load once.  We will trigger again, so ignoring any error is fine
+	_ = c.RunOnce()
+
+	// doesn't matter what workers say, only start one.
+	go wait.Until(c.runWorker, time.Second, stopCh)
+
+	// start timer that rechecks every minute, just in case.  this also serves to prime the controller quickly.
+	go wait.Until(func() {
+		c.Enqueue()
+	}, 1*time.Minute, stopCh)
+
+	<-stopCh
+}
+
+func (c *DynamicServingCertificateController) runWorker() {
+	for c.processNextWorkItem() {
+	}
+}
+
+func (c *DynamicServingCertificateController) processNextWorkItem() bool {
+	dsKey, quit := c.queue.Get()
+	if quit {
+		return false
+	}
+	defer c.queue.Done(dsKey)
+
+	err := c.syncCerts()
+	if err == nil {
+		c.queue.Forget(dsKey)
+		return true
+	}
+
+	utilruntime.HandleError(fmt.Errorf("%v failed with : %v", dsKey, err))
+	c.queue.AddRateLimited(dsKey)
+
+	return true
+}
+
+// Enqueue a method to allow separate control loops to cause the certificate controller to trigger and read content.
+func (c *DynamicServingCertificateController) Enqueue() {
+	c.queue.Add(workItemKey)
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/union_content.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"bytes"
+	"crypto/x509"
+	"strings"
+
+	utilerrors "k8s.io/apimachinery/pkg/util/errors"
+)
+
+type unionCAContent []CAContentProvider
+
+var _ Notifier = &unionCAContent{}
+var _ CAContentProvider = &unionCAContent{}
+var _ ControllerRunner = &unionCAContent{}
+
+// NewUnionCAContentProvider returns a CAContentProvider that is a union of other CAContentProviders
+func NewUnionCAContentProvider(caContentProviders ...CAContentProvider) CAContentProvider {
+	return unionCAContent(caContentProviders)
+}
+
+// Name is just an identifier
+func (c unionCAContent) Name() string {
+	names := []string{}
+	for _, curr := range c {
+		names = append(names, curr.Name())
+	}
+	return strings.Join(names, ",")
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) CurrentCABundleContent() []byte {
+	caBundles := [][]byte{}
+	for _, curr := range c {
+		if currCABytes := curr.CurrentCABundleContent(); len(currCABytes) > 0 {
+			caBundles = append(caBundles, []byte(strings.TrimSpace(string(currCABytes))))
+		}
+	}
+
+	return bytes.Join(caBundles, []byte("\n"))
+}
+
+// CurrentCABundleContent provides ca bundle byte content
+func (c unionCAContent) VerifyOptions() (x509.VerifyOptions, bool) {
+	currCABundle := c.CurrentCABundleContent()
+	if len(currCABundle) == 0 {
+		return x509.VerifyOptions{}, false
+	}
+
+	// TODO make more efficient.  This isn't actually used in any of our mainline paths.  It's called to build the TLSConfig
+	// TODO on file changes, but the actual authentication runs against the individual items, not the union.
+	ret, err := newCABundleAndVerifier(c.Name(), c.CurrentCABundleContent())
+	if err != nil {
+		// because we're made up of already vetted values, this indicates some kind of coding error
+		panic(err)
+	}
+
+	return ret.verifyOptions, true
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) AddListener(listener Listener) {
+	for _, curr := range c {
+		if notifier, ok := curr.(Notifier); ok {
+			notifier.AddListener(listener)
+		}
+	}
+}
+
+// AddListener adds a listener to be notified when the CA content changes.
+func (c unionCAContent) RunOnce() error {
+	errors := []error{}
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			if err := controller.RunOnce(); err != nil {
+				errors = append(errors, err)
+			}
+		}
+	}
+
+	return utilerrors.NewAggregate(errors)
+}
+
+// Run runs the controller
+func (c unionCAContent) Run(workers int, stopCh <-chan struct{}) {
+	for _, curr := range c {
+		if controller, ok := curr.(ControllerRunner); ok {
+			go controller.Run(workers, stopCh)
+		}
+	}
+}

vendor/k8s.io/apiserver/pkg/server/dynamiccertificates/util.go

@@ -0,0 +1,68 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package dynamiccertificates
+
+import (
+	"crypto/x509"
+	"fmt"
+	"strings"
+	"time"
+)
+
+// GetHumanCertDetail is a convenient method for printing compact details of certificate that helps when debugging
+// kube-apiserver usage of certs.
+func GetHumanCertDetail(certificate *x509.Certificate) string {
+	humanName := certificate.Subject.CommonName
+	signerHumanName := certificate.Issuer.CommonName
+	if certificate.Subject.CommonName == certificate.Issuer.CommonName {
+		signerHumanName = "<self>"
+	}
+
+	usages := []string{}
+	for _, curr := range certificate.ExtKeyUsage {
+		if curr == x509.ExtKeyUsageClientAuth {
+			usages = append(usages, "client")
+			continue
+		}
+		if curr == x509.ExtKeyUsageServerAuth {
+			usages = append(usages, "serving")
+			continue
+		}
+
+		usages = append(usages, fmt.Sprintf("%d", curr))
+	}
+
+	validServingNames := []string{}
+	for _, ip := range certificate.IPAddresses {
+		validServingNames = append(validServingNames, ip.String())
+	}
+	for _, dnsName := range certificate.DNSNames {
+		validServingNames = append(validServingNames, dnsName)
+	}
+	servingString := ""
+	if len(validServingNames) > 0 {
+		servingString = fmt.Sprintf(" validServingFor=[%s]", strings.Join(validServingNames, ","))
+	}
+
+	groupString := ""
+	if len(certificate.Subject.Organization) > 0 {
+		groupString = fmt.Sprintf(" groups=[%s]", strings.Join(certificate.Subject.Organization, ","))
+	}
+
+	return fmt.Sprintf("%q [%s]%s%s issuer=%q (%v to %v (now=%v))", humanName, strings.Join(usages, ","), groupString, servingString, signerHumanName, certificate.NotBefore.UTC(), certificate.NotAfter.UTC(),
+		time.Now().UTC())
+}

vendor/k8s.io/apiserver/pkg/server/egressselector/config.go

@@ -0,0 +1,176 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"fmt"
+	"io/ioutil"
+	"strings"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/validation/field"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/apiserver/pkg/apis/apiserver/install"
+	"k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
+	"k8s.io/utils/path"
+	"sigs.k8s.io/yaml"
+)
+
+var cfgScheme = runtime.NewScheme()
+
+func init() {
+	install.Install(cfgScheme)
+}
+
+// ReadEgressSelectorConfiguration reads the egress selector configuration at the specified path.
+// It returns the loaded egress selector configuration if the input file aligns with the required syntax.
+// If it does not align with the provided syntax, it returns a default configuration which should function as a no-op.
+// It does this by returning a nil configuration, which preserves backward compatibility.
+// This works because prior to this there was no egress selector configuration.
+// It returns an error if the file did not exist.
+func ReadEgressSelectorConfiguration(configFilePath string) (*apiserver.EgressSelectorConfiguration, error) {
+	if configFilePath == "" {
+		return nil, nil
+	}
+	// a file was provided, so we just read it.
+	data, err := ioutil.ReadFile(configFilePath)
+	if err != nil {
+		return nil, fmt.Errorf("unable to read egress selector configuration from %q [%v]", configFilePath, err)
+	}
+	var decodedConfig v1alpha1.EgressSelectorConfiguration
+	err = yaml.Unmarshal(data, &decodedConfig)
+	if err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	if decodedConfig.Kind != "EgressSelectorConfiguration" {
+		return nil, fmt.Errorf("invalid service configuration object %q", decodedConfig.Kind)
+	}
+	internalConfig := &apiserver.EgressSelectorConfiguration{}
+	if err := cfgScheme.Convert(&decodedConfig, internalConfig, nil); err != nil {
+		// we got an error where the decode wasn't related to a missing type
+		return nil, err
+	}
+	return internalConfig, nil
+}
+
+// ValidateEgressSelectorConfiguration checks the apiserver.EgressSelectorConfiguration for
+// common configuration errors. It will return error for problems such as configuring mtls/cert
+// settings for protocol which do not support security. It will also try to catch errors such as
+// incorrect file paths. It will return nil if it does not find anything wrong.
+func ValidateEgressSelectorConfiguration(config *apiserver.EgressSelectorConfiguration) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if config == nil {
+		return allErrs // Treating a nil configuration as valid
+	}
+	for _, service := range config.EgressSelections {
+		base := field.NewPath("service", "connection")
+		switch service.Connection.Type {
+		case "direct":
+			allErrs = append(allErrs, validateDirectConnection(service.Connection, base)...)
+		case "http-connect":
+			allErrs = append(allErrs, validateHTTPConnection(service.Connection, base)...)
+		default:
+			allErrs = append(allErrs, field.NotSupported(
+				base.Child("type"),
+				service.Connection.Type,
+				[]string{"direct", "http-connect"}))
+		}
+	}
+
+	return allErrs
+}
+
+func validateDirectConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	if connection.HTTPConnect != nil {
+		return field.ErrorList{field.Invalid(
+			fldPath.Child("httpConnect"),
+			"direct",
+			"httpConnect config should be absent for direct connect"),
+		}
+	}
+	return nil
+}
+
+func validateHTTPConnection(connection apiserver.Connection, fldPath *field.Path) field.ErrorList {
+	allErrs := field.ErrorList{}
+	if connection.HTTPConnect == nil {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect"),
+			"nil",
+			"httpConnect config should be present for http-connect"))
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "https://") {
+		if connection.HTTPConnect.CABundle == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				"nil",
+				"http-connect via https requires caBundle"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.CABundle); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect ca bundle does not exist"))
+		}
+		if connection.HTTPConnect.ClientCert == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				"nil",
+				"http-connect via https requires clientCert"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientCert); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect client cert does not exist"))
+		}
+		if connection.HTTPConnect.ClientKey == "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				"nil",
+				"http-connect via https requires clientKey"))
+		} else if exists, err := path.Exists(path.CheckFollowSymlink, connection.HTTPConnect.ClientKey); exists == false || err != nil {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect client key does not exist"))
+		}
+	} else if strings.HasPrefix(connection.HTTPConnect.URL, "http://") {
+		if connection.HTTPConnect.CABundle != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "caBundle"),
+				connection.HTTPConnect.CABundle,
+				"http-connect via http does not support caBundle"))
+		}
+		if connection.HTTPConnect.ClientCert != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientCert"),
+				connection.HTTPConnect.ClientCert,
+				"http-connect via http does not support clientCert"))
+		}
+		if connection.HTTPConnect.ClientKey != "" {
+			allErrs = append(allErrs, field.Invalid(
+				fldPath.Child("httpConnect", "clientKey"),
+				connection.HTTPConnect.ClientKey,
+				"http-connect via http does not support clientKey"))
+		}
+	} else {
+		allErrs = append(allErrs, field.Invalid(
+			fldPath.Child("httpConnect", "url"),
+			connection.HTTPConnect.URL,
+			"supported connection protocols are http:// and https://"))
+	}
+	return allErrs
+}

vendor/k8s.io/apiserver/pkg/server/egressselector/egress_selector.go

@@ -0,0 +1,199 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package egressselector
+
+import (
+	"bufio"
+	"context"
+	"crypto/tls"
+	"crypto/x509"
+	"fmt"
+	"io/ioutil"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	"k8s.io/apiserver/pkg/apis/apiserver"
+	"k8s.io/klog"
+	"net"
+	"net/http"
+	"net/url"
+	"strings"
+)
+
+var directDialer utilnet.DialFunc = http.DefaultTransport.(*http.Transport).DialContext
+
+// EgressSelector is the map of network context type to context dialer, for network egress.
+type EgressSelector struct {
+	egressToDialer map[EgressType]utilnet.DialFunc
+}
+
+// EgressType is an indicator of which egress selection should be used for sending traffic.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md#network-context
+type EgressType int
+
+const (
+	// Master is the EgressType for traffic intended to go to the control plane.
+	Master EgressType = iota
+	// Etcd is the EgressType for traffic intended to go to Kubernetes persistence store.
+	Etcd
+	// Cluster is the EgressType for traffic intended to go to the system being managed by Kubernetes.
+	Cluster
+)
+
+// NetworkContext is the struct used by Kubernetes API Server to indicate where it intends traffic to be sent.
+type NetworkContext struct {
+	// EgressSelectionName is the unique name of the
+	// EgressSelectorConfiguration which determines
+	// the network we route the traffic to.
+	EgressSelectionName EgressType
+}
+
+// Lookup is the interface to get the dialer function for the network context.
+type Lookup func(networkContext NetworkContext) (utilnet.DialFunc, error)
+
+// String returns the canonical string representation of the egress type
+func (s EgressType) String() string {
+	switch s {
+	case Master:
+		return "master"
+	case Etcd:
+		return "etcd"
+	case Cluster:
+		return "cluster"
+	default:
+		return "invalid"
+	}
+}
+
+// AsNetworkContext is a helper function to make it easy to get the basic NetworkContext objects.
+func (s EgressType) AsNetworkContext() NetworkContext {
+	return NetworkContext{EgressSelectionName: s}
+}
+
+func lookupServiceName(name string) (EgressType, error) {
+	switch strings.ToLower(name) {
+	case "master":
+		return Master, nil
+	case "etcd":
+		return Etcd, nil
+	case "cluster":
+		return Cluster, nil
+	}
+	return -1, fmt.Errorf("unrecognized service name %s", name)
+}
+
+func createConnectDialer(connectConfig *apiserver.HTTPConnectConfig) (utilnet.DialFunc, error) {
+	clientCert := connectConfig.ClientCert
+	clientKey := connectConfig.ClientKey
+	caCert := connectConfig.CABundle
+	proxyURL, err := url.Parse(connectConfig.URL)
+	if err != nil {
+		return nil, fmt.Errorf("invalid proxy server url %q: %v", connectConfig.URL, err)
+	}
+	proxyAddress := proxyURL.Host
+
+	clientCerts, err := tls.LoadX509KeyPair(clientCert, clientKey)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read key pair %s & %s, got %v", clientCert, clientKey, err)
+	}
+	certPool := x509.NewCertPool()
+	certBytes, err := ioutil.ReadFile(caCert)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read cert file %s, got %v", caCert, err)
+	}
+	ok := certPool.AppendCertsFromPEM(certBytes)
+	if !ok {
+		return nil, fmt.Errorf("failed to append CA cert to the cert pool")
+	}
+	contextDialer := func(ctx context.Context, network, addr string) (net.Conn, error) {
+		klog.V(4).Infof("Sending request to %q.", addr)
+		proxyConn, err := tls.Dial("tcp", proxyAddress,
+			&tls.Config{
+				Certificates: []tls.Certificate{clientCerts},
+				RootCAs:      certPool,
+			},
+		)
+		if err != nil {
+			return nil, fmt.Errorf("dialing proxy %q failed: %v", proxyAddress, err)
+		}
+		fmt.Fprintf(proxyConn, "CONNECT %s HTTP/1.1\r\nHost: %s\r\n\r\n", addr, "127.0.0.1")
+		br := bufio.NewReader(proxyConn)
+		res, err := http.ReadResponse(br, nil)
+		if err != nil {
+			proxyConn.Close()
+			return nil, fmt.Errorf("reading HTTP response from CONNECT to %s via proxy %s failed: %v",
+				addr, proxyAddress, err)
+		}
+		if res.StatusCode != 200 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("proxy error from %s while dialing %s, code %d: %v",
+				proxyAddress, addr, res.StatusCode, res.Status)
+		}
+
+		// It's safe to discard the bufio.Reader here and return the
+		// original TCP conn directly because we only use this for
+		// TLS, and in TLS the client speaks first, so we know there's
+		// no unbuffered data. But we can double-check.
+		if br.Buffered() > 0 {
+			proxyConn.Close()
+			return nil, fmt.Errorf("unexpected %d bytes of buffered data from CONNECT proxy %q",
+				br.Buffered(), proxyAddress)
+		}
+		klog.V(4).Infof("About to proxy request to %s over %s.", addr, proxyAddress)
+		return proxyConn, nil
+	}
+	return contextDialer, nil
+}
+
+// NewEgressSelector configures lookup mechanism for Lookup.
+// It does so based on a EgressSelectorConfiguration which was read at startup.
+func NewEgressSelector(config *apiserver.EgressSelectorConfiguration) (*EgressSelector, error) {
+	if config == nil || config.EgressSelections == nil {
+		// No Connection Services configured, leaving the serviceMap empty, will return default dialer.
+		return nil, nil
+	}
+	cs := &EgressSelector{
+		egressToDialer: make(map[EgressType]utilnet.DialFunc),
+	}
+	for _, service := range config.EgressSelections {
+		name, err := lookupServiceName(service.Name)
+		if err != nil {
+			return nil, err
+		}
+		switch service.Connection.Type {
+		case "http-connect":
+			contextDialer, err := createConnectDialer(service.Connection.HTTPConnect)
+			if err != nil {
+				return nil, fmt.Errorf("failed to create http-connect dialer: %v", err)
+			}
+			cs.egressToDialer[name] = contextDialer
+		case "direct":
+			cs.egressToDialer[name] = directDialer
+		default:
+			return nil, fmt.Errorf("unrecognized service connection type %q", service.Connection.Type)
+		}
+	}
+	return cs, nil
+}
+
+// Lookup gets the dialer function for the network context.
+// This is configured for the Kubernetes API Server at startup.
+func (cs *EgressSelector) Lookup(networkContext NetworkContext) (utilnet.DialFunc, error) {
+	if cs.egressToDialer == nil {
+		// The round trip wrapper will over-ride the dialContext method appropriately
+		return nil, nil
+	}
+	return cs.egressToDialer[networkContext.EgressSelectionName], nil
+}

vendor/k8s.io/apiserver/pkg/server/filters/compression.go

@@ -1,181 +0,0 @@
-/*
-Copyright 2017 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package filters
-
-import (
-	"compress/gzip"
-	"compress/zlib"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"strings"
-
-	"github.com/emicklei/go-restful"
-
-	"k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apiserver/pkg/endpoints/request"
-)
-
-// Compressor is an interface to compression writers
-type Compressor interface {
-	io.WriteCloser
-	Flush() error
-}
-
-const (
-	headerAcceptEncoding  = "Accept-Encoding"
-	headerContentEncoding = "Content-Encoding"
-
-	encodingGzip    = "gzip"
-	encodingDeflate = "deflate"
-)
-
-// WithCompression wraps an http.Handler with the Compression Handler
-func WithCompression(handler http.Handler) http.Handler {
-	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-		wantsCompression, encoding := wantsCompressedResponse(req)
-		w.Header().Set("Vary", "Accept-Encoding")
-		if wantsCompression {
-			compressionWriter, err := NewCompressionResponseWriter(w, encoding)
-			if err != nil {
-				handleError(w, req, err)
-				runtime.HandleError(fmt.Errorf("failed to compress HTTP response: %v", err))
-				return
-			}
-			compressionWriter.Header().Set("Content-Encoding", encoding)
-			handler.ServeHTTP(compressionWriter, req)
-			compressionWriter.(*compressionResponseWriter).Close()
-		} else {
-			handler.ServeHTTP(w, req)
-		}
-	})
-}
-
-// wantsCompressedResponse reads the Accept-Encoding header to see if and which encoding is requested.
-func wantsCompressedResponse(req *http.Request) (bool, string) {
-	// don't compress watches
-	ctx := req.Context()
-	info, ok := request.RequestInfoFrom(ctx)
-	if !ok {
-		return false, ""
-	}
-	if !info.IsResourceRequest {
-		return false, ""
-	}
-	if info.Verb == "watch" {
-		return false, ""
-	}
-	header := req.Header.Get(headerAcceptEncoding)
-	gi := strings.Index(header, encodingGzip)
-	zi := strings.Index(header, encodingDeflate)
-	// use in order of appearance
-	switch {
-	case gi == -1:
-		return zi != -1, encodingDeflate
-	case zi == -1:
-		return gi != -1, encodingGzip
-	case gi < zi:
-		return true, encodingGzip
-	default:
-		return true, encodingDeflate
-	}
-}
-
-type compressionResponseWriter struct {
-	writer     http.ResponseWriter
-	compressor Compressor
-	encoding   string
-}
-
-// NewCompressionResponseWriter returns wraps w with a compression ResponseWriter, using the given encoding
-func NewCompressionResponseWriter(w http.ResponseWriter, encoding string) (http.ResponseWriter, error) {
-	var compressor Compressor
-	switch encoding {
-	case encodingGzip:
-		compressor = gzip.NewWriter(w)
-	case encodingDeflate:
-		compressor = zlib.NewWriter(w)
-	default:
-		return nil, fmt.Errorf("%s is not a supported encoding type", encoding)
-	}
-	return &compressionResponseWriter{
-		writer:     w,
-		compressor: compressor,
-		encoding:   encoding,
-	}, nil
-}
-
-// compressionResponseWriter implements http.Responsewriter Interface
-var _ http.ResponseWriter = &compressionResponseWriter{}
-
-func (c *compressionResponseWriter) Header() http.Header {
-	return c.writer.Header()
-}
-
-// compress data according to compression method
-func (c *compressionResponseWriter) Write(p []byte) (int, error) {
-	if c.compressorClosed() {
-		return -1, errors.New("compressing error: tried to write data using closed compressor")
-	}
-	c.Header().Set(headerContentEncoding, c.encoding)
-	defer c.compressor.Flush()
-	return c.compressor.Write(p)
-}
-
-func (c *compressionResponseWriter) WriteHeader(status int) {
-	c.writer.WriteHeader(status)
-}
-
-// CloseNotify is part of http.CloseNotifier interface
-func (c *compressionResponseWriter) CloseNotify() <-chan bool {
-	return c.writer.(http.CloseNotifier).CloseNotify()
-}
-
-// Close the underlying compressor
-func (c *compressionResponseWriter) Close() error {
-	if c.compressorClosed() {
-		return errors.New("Compressing error: tried to close already closed compressor")
-	}
-
-	c.compressor.Close()
-	c.compressor = nil
-	return nil
-}
-
-func (c *compressionResponseWriter) Flush() {
-	if c.compressorClosed() {
-		return
-	}
-	c.compressor.Flush()
-}
-
-func (c *compressionResponseWriter) compressorClosed() bool {
-	return nil == c.compressor
-}
-
-// RestfulWithCompression wraps WithCompression to be compatible with go-restful
-func RestfulWithCompression(function restful.RouteFunction) restful.RouteFunction {
-	return restful.RouteFunction(func(request *restful.Request, response *restful.Response) {
-		handler := WithCompression(http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
-			response.ResponseWriter = w
-			request.Request = req
-			function(request, response)
-		}))
-		handler.ServeHTTP(response.ResponseWriter, request.Request)
-	})
-}

vendor/k8s.io/apiserver/pkg/server/filters/maxinflight.go

@@ -178,7 +178,7 @@ func WithMaxInFlightLimit(
 						}
 					}
 				}
-				metrics.Record(r, requestInfo, metrics.APIServerComponent, "", http.StatusTooManyRequests, 0, 0)
+				metrics.RecordRequestTermination(r, requestInfo, metrics.APIServerComponent, http.StatusTooManyRequests)
 				tooManyRequests(r, w)
 			}
 		}

vendor/k8s.io/apiserver/pkg/server/filters/timeout.go

@@ -59,7 +59,7 @@ func WithTimeoutForNonLongRunningRequests(handler http.Handler, longRunning apir
 
 		postTimeoutFn := func() {
 			cancel()
-			metrics.Record(req, requestInfo, metrics.APIServerComponent, "", http.StatusGatewayTimeout, 0, 0)
+			metrics.RecordRequestTermination(req, requestInfo, metrics.APIServerComponent, http.StatusGatewayTimeout)
 		}
 		return req, time.After(timeout), postTimeoutFn, apierrors.NewTimeoutError(fmt.Sprintf("request did not complete within %s", timeout), 0)
 	}
@@ -99,7 +99,8 @@ func (t *timeoutHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	go func() {
 		defer func() {
 			err := recover()
-			if err != nil {
+			// do not wrap the sentinel ErrAbortHandler panic value
+			if err != nil && err != http.ErrAbortHandler {
 				// Same as stdlib http server code. Manually allocate stack
 				// trace buffer size to prevent excessively large logs
 				const size = 64 << 10

vendor/k8s.io/apiserver/pkg/server/filters/wrap.go

@@ -25,23 +25,28 @@ import (
 	"k8s.io/apiserver/pkg/server/httplog"
 )
 
-// WithPanicRecovery wraps an http Handler to recover and log panics.
+// WithPanicRecovery wraps an http Handler to recover and log panics (except in the special case of http.ErrAbortHandler panics, which suppress logging).
 func WithPanicRecovery(handler http.Handler) http.Handler {
 	return withPanicRecovery(handler, func(w http.ResponseWriter, req *http.Request, err interface{}) {
+		if err == http.ErrAbortHandler {
+			// honor the http.ErrAbortHandler sentinel panic value:
+			//   ErrAbortHandler is a sentinel panic value to abort a handler.
+			//   While any panic from ServeHTTP aborts the response to the client,
+			//   panicking with ErrAbortHandler also suppresses logging of a stack trace to the server's error log.
+			return
+		}
 		http.Error(w, "This request caused apiserver to panic. Look in the logs for details.", http.StatusInternalServerError)
 		klog.Errorf("apiserver panic'd on %v %v", req.Method, req.RequestURI)
 	})
 }
 
 func withPanicRecovery(handler http.Handler, crashHandler func(http.ResponseWriter, *http.Request, interface{})) http.Handler {
+	handler = httplog.WithLogging(handler, httplog.DefaultStacktracePred)
 	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 		defer runtime.HandleCrash(func(err interface{}) {
 			crashHandler(w, req, err)
 		})
 
-		logger := httplog.NewLogged(req, &w)
-		defer logger.Log()
-
 		// Dispatch to the internal handler
 		handler.ServeHTTP(w, req)
 	})

vendor/k8s.io/apiserver/pkg/server/genericapiserver.go

@@ -26,13 +26,13 @@ import (
 
 	systemd "github.com/coreos/go-systemd/daemon"
 	"github.com/go-openapi/spec"
-	"k8s.io/klog"
 
 	"k8s.io/apimachinery/pkg/api/meta"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apimachinery/pkg/util/sets"
 	utilwaitgroup "k8s.io/apimachinery/pkg/util/waitgroup"
 	"k8s.io/apiserver/pkg/admission"
@@ -45,6 +45,7 @@ import (
 	"k8s.io/apiserver/pkg/server/routes"
 	utilopenapi "k8s.io/apiserver/pkg/util/openapi"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog"
 	openapibuilder "k8s.io/kube-openapi/pkg/builder"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 	"k8s.io/kube-openapi/pkg/handler"
@@ -76,6 +77,10 @@ type APIGroupInfo struct {
 	NegotiatedSerializer runtime.NegotiatedSerializer
 	// ParameterCodec performs conversions for query parameters passed to API calls
 	ParameterCodec runtime.ParameterCodec
+
+	// StaticOpenAPISpec is the spec derived from the definitions of all resources installed together.
+	// It is set during InstallAPIGroups, InstallAPIGroup, and InstallLegacyAPIGroup.
+	StaticOpenAPISpec *spec.Swagger
 }
 
 // GenericAPIServer contains state for a Kubernetes cluster api server.
@@ -145,9 +150,22 @@ type GenericAPIServer struct {
 	preShutdownHooksCalled bool
 
 	// healthz checks
-	healthzLock    sync.Mutex
-	healthzChecks  []healthz.HealthzChecker
-	healthzCreated bool
+	healthzLock            sync.Mutex
+	healthzChecks          []healthz.HealthChecker
+	healthzChecksInstalled bool
+	// livez checks
+	livezLock            sync.Mutex
+	livezChecks          []healthz.HealthChecker
+	livezChecksInstalled bool
+	// readyz checks
+	readyzLock            sync.Mutex
+	readyzChecks          []healthz.HealthChecker
+	readyzChecksInstalled bool
+	livezGracePeriod      time.Duration
+	livezClock            clock.Clock
+	// the readiness stop channel is used to signal that the apiserver has initiated a shutdown sequence, this
+	// will cause readyz to return unhealthy.
+	readinessStopCh chan struct{}
 
 	// auditing. The backend is started after the server starts listening.
 	AuditBackend audit.Backend
@@ -171,6 +189,11 @@ type GenericAPIServer struct {
 	// HandlerChainWaitGroup allows you to wait for all chain handlers finish after the server shutdown.
 	HandlerChainWaitGroup *utilwaitgroup.SafeWaitGroup
 
+	// ShutdownDelayDuration allows to block shutdown for some time, e.g. until endpoints pointing to this API server
+	// have converged on all node. During this time, the API server keeps serving, /healthz will return 200,
+	// but /readyz will return failure.
+	ShutdownDelayDuration time.Duration
+
 	// The limit on the request body size that would be accepted and decoded in a write request.
 	// 0 means no limit.
 	maxRequestBodyBytes int64
@@ -189,13 +212,16 @@ type DelegationTarget interface {
 	PreShutdownHooks() map[string]preShutdownHookEntry
 
 	// HealthzChecks returns the healthz checks that need to be combined
-	HealthzChecks() []healthz.HealthzChecker
+	HealthzChecks() []healthz.HealthChecker
 
 	// ListedPaths returns the paths for supporting an index
 	ListedPaths() []string
 
 	// NextDelegate returns the next delegationTarget in the chain of delegations
 	NextDelegate() DelegationTarget
+
+	// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
+	PrepareRun() preparedGenericAPIServer
 }
 
 func (s *GenericAPIServer) UnprotectedHandler() http.Handler {
@@ -208,7 +234,7 @@ func (s *GenericAPIServer) PostStartHooks() map[string]postStartHookEntry {
 func (s *GenericAPIServer) PreShutdownHooks() map[string]preShutdownHookEntry {
 	return s.preShutdownHooks
 }
-func (s *GenericAPIServer) HealthzChecks() []healthz.HealthzChecker {
+func (s *GenericAPIServer) HealthzChecks() []healthz.HealthChecker {
 	return s.healthzChecks
 }
 func (s *GenericAPIServer) ListedPaths() []string {
@@ -235,8 +261,8 @@ func (s emptyDelegate) PostStartHooks() map[string]postStartHookEntry {
 func (s emptyDelegate) PreShutdownHooks() map[string]preShutdownHookEntry {
 	return map[string]preShutdownHookEntry{}
 }
-func (s emptyDelegate) HealthzChecks() []healthz.HealthzChecker {
-	return []healthz.HealthzChecker{}
+func (s emptyDelegate) HealthzChecks() []healthz.HealthChecker {
+	return []healthz.HealthChecker{}
 }
 func (s emptyDelegate) ListedPaths() []string {
 	return []string{}
@@ -244,14 +270,19 @@ func (s emptyDelegate) ListedPaths() []string {
 func (s emptyDelegate) NextDelegate() DelegationTarget {
 	return nil
 }
+func (s emptyDelegate) PrepareRun() preparedGenericAPIServer {
+	return preparedGenericAPIServer{nil}
+}
 
 // preparedGenericAPIServer is a private wrapper that enforces a call of PrepareRun() before Run can be invoked.
 type preparedGenericAPIServer struct {
 	*GenericAPIServer
 }
 
-// PrepareRun does post API installation setup steps.
+// PrepareRun does post API installation setup steps. It calls recursively the same function of the delegates.
 func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
+	s.delegationTarget.PrepareRun()
+
 	if s.openAPIConfig != nil {
 		s.OpenAPIVersionedService, s.StaticOpenAPISpec = routes.OpenAPI{
 			Config: s.openAPIConfig,
@@ -259,6 +290,12 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 	}
 
 	s.installHealthz()
+	s.installLivez()
+	err := s.addReadyzShutdownCheck(s.readinessStopCh)
+	if err != nil {
+		klog.Errorf("Failed to install readyz shutdown check %s", err)
+	}
+	s.installReadyz()
 
 	// Register audit backend preShutdownHook.
 	if s.AuditBackend != nil {
@@ -277,18 +314,32 @@ func (s *GenericAPIServer) PrepareRun() preparedGenericAPIServer {
 // Run spawns the secure http server. It only returns if stopCh is closed
 // or the secure port cannot be listened on initially.
 func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
-	err := s.NonBlockingRun(stopCh)
+	delayedStopCh := make(chan struct{})
+
+	go func() {
+		defer close(delayedStopCh)
+		<-stopCh
+
+		time.Sleep(s.ShutdownDelayDuration)
+	}()
+
+	// close socket after delayed stopCh
+	err := s.NonBlockingRun(delayedStopCh)
 	if err != nil {
 		return err
 	}
 
 	<-stopCh
 
+	// run shutdown hooks directly. This includes deregistering from the kubernetes endpoint in case of kube-apiserver.
 	err = s.RunPreShutdownHooks()
 	if err != nil {
 		return err
 	}
 
+	// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
+	<-delayedStopCh
+
 	// Wait for all requests to finish, which are bounded by the RequestTimeout variable.
 	s.HandlerChainWaitGroup.Wait()
 
@@ -318,6 +369,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 		stoppedCh, err = s.SecureServingInfo.Serve(s.Handler, s.ShutdownTimeout, internalStopCh)
 		if err != nil {
 			close(internalStopCh)
+			close(auditStopCh)
 			return err
 		}
 	}
@@ -327,6 +379,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
 	// ensure cleanup.
 	go func() {
 		<-stopCh
+		close(s.readinessStopCh)
 		close(internalStopCh)
 		if stoppedCh != nil {
 			<-stoppedCh
@@ -473,10 +526,9 @@ func (s *GenericAPIServer) newAPIGroupVersion(apiGroupInfo *APIGroupInfo, groupV
 
 		EquivalentResourceRegistry: s.EquivalentResourceRegistry,
 
-		Admit:                        s.admissionControl,
-		MinRequestTimeout:            s.minRequestTimeout,
-		EnableAPIResponseCompression: s.enableAPIResponseCompression,
-		Authorizer:                   s.Authorizer,
+		Admit:             s.admissionControl,
+		MinRequestTimeout: s.minRequestTimeout,
+		Authorizer:        s.Authorizer,
 	}
 }
 
@@ -514,6 +566,9 @@ func (s *GenericAPIServer) getOpenAPIModels(apiPrefix string, apiGroupInfos ...*
 	if err != nil {
 		return nil, err
 	}
+	for _, apiGroupInfo := range apiGroupInfos {
+		apiGroupInfo.StaticOpenAPISpec = openAPISpec
+	}
 	return utilopenapi.ToProtoModels(openAPISpec)
 }
 

vendor/k8s.io/apiserver/pkg/server/healthz.go

@@ -18,28 +18,142 @@ package server
 
 import (
 	"fmt"
+	"net/http"
+	"time"
 
+	"k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apiserver/pkg/server/healthz"
 )
 
-// AddHealthzCheck allows you to add a HealthzCheck.
-func (s *GenericAPIServer) AddHealthzChecks(checks ...healthz.HealthzChecker) error {
+// AddHealthChecks adds HealthCheck(s) to health endpoints (healthz, livez, readyz) but
+// configures the liveness grace period to be zero, which means we expect this health check
+// to immediately indicate that the apiserver is unhealthy.
+func (s *GenericAPIServer) AddHealthChecks(checks ...healthz.HealthChecker) error {
+	// we opt for a delay of zero here, because this entrypoint adds generic health checks
+	// and not health checks which are specifically related to kube-apiserver boot-sequences.
+	return s.addHealthChecks(0, checks...)
+}
+
+// AddBootSequenceHealthChecks adds health checks to the old healthz endpoint (for backwards compatibility reasons)
+// as well as livez and readyz. The livez grace period is defined by the value of the
+// command-line flag --livez-grace-period; before the grace period elapses, the livez health checks
+// will default to healthy. One may want to set a grace period in order to prevent the kubelet from restarting
+// the kube-apiserver due to long-ish boot sequences. Readyz health checks, on the other hand, have no grace period,
+// since readyz should fail until boot fully completes.
+func (s *GenericAPIServer) AddBootSequenceHealthChecks(checks ...healthz.HealthChecker) error {
+	return s.addHealthChecks(s.livezGracePeriod, checks...)
+}
+
+// addHealthChecks adds health checks to healthz, livez, and readyz. The delay passed in will set
+// a corresponding grace period on livez.
+func (s *GenericAPIServer) addHealthChecks(livezGracePeriod time.Duration, checks ...healthz.HealthChecker) error {
 	s.healthzLock.Lock()
 	defer s.healthzLock.Unlock()
-
-	if s.healthzCreated {
+	if s.healthzChecksInstalled {
 		return fmt.Errorf("unable to add because the healthz endpoint has already been created")
 	}
-
 	s.healthzChecks = append(s.healthzChecks, checks...)
+	return s.addLivezChecks(livezGracePeriod, checks...)
+}
+
+// addReadyzChecks allows you to add a HealthCheck to readyz.
+func (s *GenericAPIServer) addReadyzChecks(checks ...healthz.HealthChecker) error {
+	s.readyzLock.Lock()
+	defer s.readyzLock.Unlock()
+	if s.readyzChecksInstalled {
+		return fmt.Errorf("unable to add because the readyz endpoint has already been created")
+	}
+	s.readyzChecks = append(s.readyzChecks, checks...)
 	return nil
 }
 
+// addLivezChecks allows you to add a HealthCheck to livez. It will also automatically add a check to readyz,
+// since we want to avoid being ready when we are not live.
+func (s *GenericAPIServer) addLivezChecks(delay time.Duration, checks ...healthz.HealthChecker) error {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	if s.livezChecksInstalled {
+		return fmt.Errorf("unable to add because the livez endpoint has already been created")
+	}
+	for _, check := range checks {
+		s.livezChecks = append(s.livezChecks, delayedHealthCheck(check, s.livezClock, delay))
+	}
+	return s.addReadyzChecks(checks...)
+}
+
+// addReadyzShutdownCheck is a convenience function for adding a readyz shutdown check, so
+// that we can register that the api-server is no longer ready while we attempt to gracefully
+// shutdown.
+func (s *GenericAPIServer) addReadyzShutdownCheck(stopCh <-chan struct{}) error {
+	return s.addReadyzChecks(shutdownCheck{stopCh})
+}
+
 // installHealthz creates the healthz endpoint for this server
 func (s *GenericAPIServer) installHealthz() {
 	s.healthzLock.Lock()
 	defer s.healthzLock.Unlock()
-	s.healthzCreated = true
-
+	s.healthzChecksInstalled = true
 	healthz.InstallHandler(s.Handler.NonGoRestfulMux, s.healthzChecks...)
 }
+
+// installReadyz creates the readyz endpoint for this server.
+func (s *GenericAPIServer) installReadyz() {
+	s.readyzLock.Lock()
+	defer s.readyzLock.Unlock()
+	s.readyzChecksInstalled = true
+	healthz.InstallReadyzHandler(s.Handler.NonGoRestfulMux, s.readyzChecks...)
+}
+
+// installLivez creates the livez endpoint for this server.
+func (s *GenericAPIServer) installLivez() {
+	s.livezLock.Lock()
+	defer s.livezLock.Unlock()
+	s.livezChecksInstalled = true
+	healthz.InstallLivezHandler(s.Handler.NonGoRestfulMux, s.livezChecks...)
+}
+
+// shutdownCheck fails if the embedded channel is closed. This is intended to allow for graceful shutdown sequences
+// for the apiserver.
+type shutdownCheck struct {
+	StopCh <-chan struct{}
+}
+
+func (shutdownCheck) Name() string {
+	return "shutdown"
+}
+
+func (c shutdownCheck) Check(req *http.Request) error {
+	select {
+	case <-c.StopCh:
+		return fmt.Errorf("process is shutting down")
+	default:
+	}
+	return nil
+}
+
+// delayedHealthCheck wraps a health check which will not fail until the explicitly defined delay has elapsed. This
+// is intended for use primarily for livez health checks.
+func delayedHealthCheck(check healthz.HealthChecker, clock clock.Clock, delay time.Duration) healthz.HealthChecker {
+	return delayedLivezCheck{
+		check,
+		clock.Now().Add(delay),
+		clock,
+	}
+}
+
+type delayedLivezCheck struct {
+	check      healthz.HealthChecker
+	startCheck time.Time
+	clock      clock.Clock
+}
+
+func (c delayedLivezCheck) Name() string {
+	return c.check.Name()
+}
+
+func (c delayedLivezCheck) Check(req *http.Request) error {
+	if c.clock.Now().After(c.startCheck) {
+		return c.check.Check(req)
+	}
+	return nil
+}

vendor/k8s.io/apiserver/pkg/server/healthz/healthz.go

@@ -25,20 +25,20 @@ import (
 	"sync/atomic"
 	"time"
 
-	"k8s.io/klog"
-
 	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
+	"k8s.io/apiserver/pkg/server/httplog"
+	"k8s.io/klog"
 )
 
-// HealthzChecker is a named healthz checker.
-type HealthzChecker interface {
+// HealthChecker is a named healthz checker.
+type HealthChecker interface {
 	Name() string
 	Check(req *http.Request) error
 }
 
 // PingHealthz returns true automatically when checked
-var PingHealthz HealthzChecker = ping{}
+var PingHealthz HealthChecker = ping{}
 
 // ping implements the simplest possible healthz checker.
 type ping struct{}
@@ -53,7 +53,7 @@ func (ping) Check(_ *http.Request) error {
 }
 
 // LogHealthz returns true if logging is not blocked
-var LogHealthz HealthzChecker = &log{}
+var LogHealthz HealthChecker = &log{}
 
 type log struct {
 	startOnce    sync.Once
@@ -81,7 +81,7 @@ func (l *log) Check(_ *http.Request) error {
 }
 
 // NamedCheck returns a healthz checker for the given name and function.
-func NamedCheck(name string, check func(r *http.Request) error) HealthzChecker {
+func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
 	return &healthzCheck{name, check}
 }
 
@@ -89,22 +89,38 @@ func NamedCheck(name string, check func(r *http.Request) error) HealthzChecker {
 // "/healthz" to mux. *All handlers* for mux must be specified in
 // exactly one call to InstallHandler. Calling InstallHandler more
 // than once for the same mux will result in a panic.
-func InstallHandler(mux mux, checks ...HealthzChecker) {
+func InstallHandler(mux mux, checks ...HealthChecker) {
 	InstallPathHandler(mux, "/healthz", checks...)
 }
 
+// InstallReadyzHandler registers handlers for health checking on the path
+// "/readyz" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallReadyzHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/readyz", checks...)
+}
+
+// InstallLivezHandler registers handlers for liveness checking on the path
+// "/livez" to mux. *All handlers* for mux must be specified in
+// exactly one call to InstallHandler. Calling InstallHandler more
+// than once for the same mux will result in a panic.
+func InstallLivezHandler(mux mux, checks ...HealthChecker) {
+	InstallPathHandler(mux, "/livez", checks...)
+}
+
 // InstallPathHandler registers handlers for health checking on
 // a specific path to mux. *All handlers* for the path must be
 // specified in exactly one call to InstallPathHandler. Calling
 // InstallPathHandler more than once for the same path and mux will
 // result in a panic.
-func InstallPathHandler(mux mux, path string, checks ...HealthzChecker) {
+func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
 	if len(checks) == 0 {
 		klog.V(5).Info("No default health checks specified. Installing the ping handler.")
-		checks = []HealthzChecker{PingHealthz}
+		checks = []HealthChecker{PingHealthz}
 	}
 
-	klog.V(5).Info("Installing healthz checkers:", formatQuoted(checkerNames(checks...)...))
+	klog.V(5).Infof("Installing health checkers for (%v): %v", path, formatQuoted(checkerNames(checks...)...))
 
 	mux.Handle(path, handleRootHealthz(checks...))
 	for _, check := range checks {
@@ -117,13 +133,13 @@ type mux interface {
 	Handle(pattern string, handler http.Handler)
 }
 
-// healthzCheck implements HealthzChecker on an arbitrary name and check function.
+// healthzCheck implements HealthChecker on an arbitrary name and check function.
 type healthzCheck struct {
 	name  string
 	check func(r *http.Request) error
 }
 
-var _ HealthzChecker = &healthzCheck{}
+var _ HealthChecker = &healthzCheck{}
 
 func (c *healthzCheck) Name() string {
 	return c.name
@@ -143,7 +159,7 @@ func getExcludedChecks(r *http.Request) sets.String {
 }
 
 // handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
-func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
+func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
 	return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		failed := false
 		excluded := getExcludedChecks(r)
@@ -173,7 +189,7 @@ func handleRootHealthz(checks ...HealthzChecker) http.HandlerFunc {
 		// always be verbose on failure
 		if failed {
 			klog.V(2).Infof("%vhealthz check failed", verboseOut.String())
-			http.Error(w, fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
+			http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
 			return
 		}
 
@@ -202,7 +218,7 @@ func adaptCheckToHandler(c func(r *http.Request) error) http.HandlerFunc {
 }
 
 // checkerNames returns the names of the checks in the same order as passed in.
-func checkerNames(checks ...HealthzChecker) []string {
+func checkerNames(checks ...HealthChecker) []string {
 	// accumulate the names of checks for printing them out.
 	checkerNames := make([]string, 0, len(checks))
 	for _, check := range checks {

vendor/k8s.io/apiserver/pkg/server/hooks.go

@@ -22,12 +22,11 @@ import (
 	"net/http"
 	"runtime/debug"
 
-	"k8s.io/klog"
-
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
 	"k8s.io/apiserver/pkg/server/healthz"
 	restclient "k8s.io/client-go/rest"
+	"k8s.io/klog"
 )
 
 // PostStartHookFunc is a function that is called after the server has started.
@@ -67,6 +66,13 @@ type postStartHookEntry struct {
 	done chan struct{}
 }
 
+type PostStartHookConfigEntry struct {
+	hook PostStartHookFunc
+	// originatingStack holds the stack that registered postStartHooks. This allows us to show a more helpful message
+	// for duplicate registration.
+	originatingStack string
+}
+
 type preShutdownHookEntry struct {
 	hook PreShutdownHookFunc
 }
@@ -77,9 +83,10 @@ func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc)
 		return fmt.Errorf("missing name")
 	}
 	if hook == nil {
-		return nil
+		return fmt.Errorf("hook func may not be nil: %q", name)
 	}
 	if s.disabledPostStartHooks.Has(name) {
+		klog.V(1).Infof("skipping %q because it was explicitly disabled", name)
 		return nil
 	}
 
@@ -97,7 +104,7 @@ func (s *GenericAPIServer) AddPostStartHook(name string, hook PostStartHookFunc)
 	// done is closed when the poststarthook is finished.  This is used by the health check to be able to indicate
 	// that the poststarthook is finished
 	done := make(chan struct{})
-	if err := s.AddHealthzChecks(postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
+	if err := s.AddBootSequenceHealthChecks(postStartHookHealthz{name: "poststarthook/" + name, done: done}); err != nil {
 		return err
 	}
 	s.postStartHooks[name] = postStartHookEntry{hook: hook, originatingStack: string(debug.Stack()), done: done}
@@ -219,7 +226,7 @@ type postStartHookHealthz struct {
 	done chan struct{}
 }
 
-var _ healthz.HealthzChecker = postStartHookHealthz{}
+var _ healthz.HealthChecker = postStartHookHealthz{}
 
 func (h postStartHookHealthz) Name() string {
 	return h.name

vendor/k8s.io/apiserver/pkg/server/httplog/httplog.go

@@ -18,6 +18,7 @@ package httplog
 
 import (
 	"bufio"
+	"context"
 	"fmt"
 	"net"
 	"net/http"
@@ -34,6 +35,11 @@ type logger interface {
 	Addf(format string, data ...interface{})
 }
 
+type respLoggerContextKeyType int
+
+// respLoggerContextKey is used to store the respLogger pointer in the request context.
+const respLoggerContextKey respLoggerContextKeyType = iota
+
 // Add a layer on top of ResponseWriter, so we can track latency and error
 // message sources.
 //
@@ -69,47 +75,54 @@ func DefaultStacktracePred(status int) bool {
 	return (status < http.StatusOK || status >= http.StatusInternalServerError) && status != http.StatusSwitchingProtocols
 }
 
-// NewLogged turns a normal response writer into a logged response writer.
-//
-// Usage:
-//
-// defer NewLogged(req, &w).StacktraceWhen(StatusIsNot(200, 202)).Log()
-//
-// (Only the call to Log() is deferred, so you can set everything up in one line!)
-//
-// Note that this *changes* your writer, to route response writing actions
-// through the logger.
-//
-// Use LogOf(w).Addf(...) to log something along with the response result.
-func NewLogged(req *http.Request, w *http.ResponseWriter) *respLogger {
-	if _, ok := (*w).(*respLogger); ok {
-		// Don't double-wrap!
-		panic("multiple NewLogged calls!")
+// WithLogging wraps the handler with logging.
+func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
+	return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
+		ctx := req.Context()
+		if old := respLoggerFromContext(req); old != nil {
+			panic("multiple WithLogging calls!")
+		}
+		rl := newLogged(req, w).StacktraceWhen(pred)
+		req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
+
+		defer rl.Log()
+		handler.ServeHTTP(rl, req)
+	})
+}
+
+// respLoggerFromContext returns the respLogger or nil.
+func respLoggerFromContext(req *http.Request) *respLogger {
+	ctx := req.Context()
+	val := ctx.Value(respLoggerContextKey)
+	if rl, ok := val.(*respLogger); ok {
+		return rl
 	}
-	rl := &respLogger{
+	return nil
+}
+
+// newLogged turns a normal response writer into a logged response writer.
+func newLogged(req *http.Request, w http.ResponseWriter) *respLogger {
+	return &respLogger{
 		startTime:         time.Now(),
 		req:               req,
-		w:                 *w,
+		w:                 w,
 		logStacktracePred: DefaultStacktracePred,
 	}
-	*w = rl // hijack caller's writer!
-	return rl
 }
 
 // LogOf returns the logger hiding in w. If there is not an existing logger
 // then a passthroughLogger will be created which will log to stdout immediately
 // when Addf is called.
 func LogOf(req *http.Request, w http.ResponseWriter) logger {
-	if rl, ok := w.(*respLogger); ok {
+	if rl := respLoggerFromContext(req); rl != nil {
 		return rl
 	}
-
 	return &passthroughLogger{}
 }
 
 // Unlogged returns the original ResponseWriter, or w if it is not our inserted logger.
-func Unlogged(w http.ResponseWriter) http.ResponseWriter {
-	if rl, ok := w.(*respLogger); ok {
+func Unlogged(req *http.Request, w http.ResponseWriter) http.ResponseWriter {
+	if rl := respLoggerFromContext(req); rl != nil {
 		return rl.w
 	}
 	return w

vendor/k8s.io/apiserver/pkg/server/options/admission.go

@@ -32,11 +32,13 @@ import (
 	mutatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/mutating"
 	validatingwebhook "k8s.io/apiserver/pkg/admission/plugin/webhook/validating"
 	apiserverapi "k8s.io/apiserver/pkg/apis/apiserver"
+	apiserverapiv1 "k8s.io/apiserver/pkg/apis/apiserver/v1"
 	apiserverapiv1alpha1 "k8s.io/apiserver/pkg/apis/apiserver/v1alpha1"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
+	"k8s.io/component-base/featuregate"
 )
 
 var configScheme = runtime.NewScheme()
@@ -44,6 +46,7 @@ var configScheme = runtime.NewScheme()
 func init() {
 	utilruntime.Must(apiserverapi.AddToScheme(configScheme))
 	utilruntime.Must(apiserverapiv1alpha1.AddToScheme(configScheme))
+	utilruntime.Must(apiserverapiv1.AddToScheme(configScheme))
 }
 
 // AdmissionOptions holds the admission options
@@ -117,6 +120,7 @@ func (a *AdmissionOptions) ApplyTo(
 	c *server.Config,
 	informers informers.SharedInformerFactory,
 	kubeAPIServerClientConfig *rest.Config,
+	features featuregate.FeatureGate,
 	pluginInitializers ...admission.PluginInitializer,
 ) error {
 	if a == nil {
@@ -139,7 +143,7 @@ func (a *AdmissionOptions) ApplyTo(
 	if err != nil {
 		return err
 	}
-	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer)
+	genericInitializer := initializer.New(clientset, informers, c.Authorization.Authorizer, features)
 	initializersChain := admission.PluginInitializers{}
 	pluginInitializers = append(pluginInitializers, genericInitializer)
 	initializersChain = append(initializersChain, pluginInitializers...)

vendor/k8s.io/apiserver/pkg/server/options/api_enablement.go

@@ -43,11 +43,14 @@ func NewAPIEnablementOptions() *APIEnablementOptions {
 // AddFlags adds flags for a specific APIServer to the specified FlagSet
 func (s *APIEnablementOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.Var(&s.RuntimeConfig, "runtime-config", ""+
-		"A set of key=value pairs that describe runtime configuration that may be passed "+
-		"to apiserver. <group>/<version> (or <version> for the core group) key can be used to "+
-		"turn on/off specific api versions. api/all is special key to control all api versions, "+
-		"be careful setting it false, unless you know what you do. api/legacy is deprecated, "+
-		"we will remove it in the future, so stop using it.")
+		"A set of key=value pairs that enable or disable built-in APIs. Supported options are:\n"+
+		"v1=true|false for the core API group\n"+
+		"<group>/<version>=true|false for a specific API group and version (e.g. apps/v1=true)\n"+
+		"api/all=true|false controls all API versions\n"+
+		"api/ga=true|false controls all API versions of the form v[0-9]+\n"+
+		"api/beta=true|false controls all API versions of the form v[0-9]+beta[0-9]+\n"+
+		"api/alpha=true|false controls all API versions of the form v[0-9]+alpha[0-9]+\n"+
+		"api/legacy is deprecated, and will be removed in a future version")
 }
 
 // Validate validates RuntimeConfig with a list of registries.
@@ -61,9 +64,9 @@ func (s *APIEnablementOptions) Validate(registries ...GroupRegisty) []error {
 	}
 
 	errors := []error{}
-	if s.RuntimeConfig["api/all"] == "false" && len(s.RuntimeConfig) == 1 {
+	if s.RuntimeConfig[resourceconfig.APIAll] == "false" && len(s.RuntimeConfig) == 1 {
 		// Do not allow only set api/all=false, in such case apiserver startup has no meaning.
-		return append(errors, fmt.Errorf("invalid key with only api/all=false"))
+		return append(errors, fmt.Errorf("invalid key with only %v=false", resourceconfig.APIAll))
 	}
 
 	groups, err := resourceconfig.ParseGroups(s.RuntimeConfig)

vendor/k8s.io/apiserver/pkg/server/options/authentication.go

@@ -19,20 +19,22 @@ package options
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
+	"strings"
 	"time"
 
-	"github.com/spf13/pflag"
-	"k8s.io/klog"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
+
+	"github.com/spf13/pflag"
 
-	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
+	"k8s.io/apiserver/pkg/authentication/request/headerrequest"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
+	"k8s.io/klog"
 	openapicommon "k8s.io/kube-openapi/pkg/common"
 )
 
@@ -47,6 +49,35 @@ type RequestHeaderAuthenticationOptions struct {
 	AllowedNames        []string
 }
 
+func (s *RequestHeaderAuthenticationOptions) Validate() []error {
+	allErrors := []error{}
+
+	if err := checkForWhiteSpaceOnly("requestheader-username-headers", s.UsernameHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-group-headers", s.GroupHeaders...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-extra-headers-prefix", s.ExtraHeaderPrefixes...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+	if err := checkForWhiteSpaceOnly("requestheader-allowed-names", s.AllowedNames...); err != nil {
+		allErrors = append(allErrors, err)
+	}
+
+	return allErrors
+}
+
+func checkForWhiteSpaceOnly(flag string, headerNames ...string) error {
+	for _, headerName := range headerNames {
+		if len(strings.TrimSpace(headerName)) == 0 {
+			return fmt.Errorf("empty value in %q", flag)
+		}
+	}
+
+	return nil
+}
+
 func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 	if s == nil {
 		return
@@ -74,23 +105,48 @@ func (s *RequestHeaderAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 
 // ToAuthenticationRequestHeaderConfig returns a RequestHeaderConfig config object for these options
 // if necessary, nil otherwise.
-func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() *authenticatorfactory.RequestHeaderConfig {
+func (s *RequestHeaderAuthenticationOptions) ToAuthenticationRequestHeaderConfig() (*authenticatorfactory.RequestHeaderConfig, error) {
 	if len(s.ClientCAFile) == 0 {
-		return nil
+		return nil, nil
+	}
+
+	caBundleProvider, err := dynamiccertificates.NewDynamicCAContentFromFile("request-header", s.ClientCAFile)
+	if err != nil {
+		return nil, err
 	}
 
 	return &authenticatorfactory.RequestHeaderConfig{
-		UsernameHeaders:     s.UsernameHeaders,
-		GroupHeaders:        s.GroupHeaders,
-		ExtraHeaderPrefixes: s.ExtraHeaderPrefixes,
-		ClientCA:            s.ClientCAFile,
-		AllowedClientNames:  s.AllowedNames,
-	}
+		UsernameHeaders:     headerrequest.StaticStringSlice(s.UsernameHeaders),
+		GroupHeaders:        headerrequest.StaticStringSlice(s.GroupHeaders),
+		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(s.ExtraHeaderPrefixes),
+		CAContentProvider:   caBundleProvider,
+		AllowedClientNames:  headerrequest.StaticStringSlice(s.AllowedNames),
+	}, nil
 }
 
+// ClientCertAuthenticationOptions provides different options for client cert auth. You should use `GetClientVerifyOptionFn` to
+// get the verify options for your authenticator.
 type ClientCertAuthenticationOptions struct {
 	// ClientCA is the certificate bundle for all the signers that you'll recognize for incoming client certificates
 	ClientCA string
+
+	// CAContentProvider are the options for verifying incoming connections using mTLS and directly assigning to users.
+	// Generally this is the CA bundle file used to authenticate client certificates
+	// If non-nil, this takes priority over the ClientCA file.
+	CAContentProvider dynamiccertificates.CAContentProvider
+}
+
+// GetClientVerifyOptionFn provides verify options for your authenticator while respecting the preferred order of verifiers.
+func (s *ClientCertAuthenticationOptions) GetClientCAContentProvider() (dynamiccertificates.CAContentProvider, error) {
+	if s.CAContentProvider != nil {
+		return s.CAContentProvider, nil
+	}
+
+	if len(s.ClientCA) == 0 {
+		return nil, nil
+	}
+
+	return dynamiccertificates.NewDynamicCAContentFromFile("client-ca-bundle", s.ClientCA)
 }
 
 func (s *ClientCertAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
@@ -140,6 +196,8 @@ func NewDelegatingAuthenticationOptions() *DelegatingAuthenticationOptions {
 
 func (s *DelegatingAuthenticationOptions) Validate() []error {
 	allErrors := []error{}
+	allErrors = append(allErrors, s.RequestHeader.Validate()...)
+
 	return allErrors
 }
 
@@ -170,9 +228,9 @@ func (s *DelegatingAuthenticationOptions) AddFlags(fs *pflag.FlagSet) {
 		"Note that this can result in authentication that treats all requests as anonymous.")
 }
 
-func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
+func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.AuthenticationInfo, servingInfo *server.SecureServingInfo, openAPIConfig *openapicommon.Config) error {
 	if s == nil {
-		c.Authenticator = nil
+		authenticationInfo.Authenticator = nil
 		return nil
 	}
 
@@ -188,32 +246,67 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 
 	// configure token review
 	if client != nil {
-		cfg.TokenAccessReviewClient = client.AuthenticationV1beta1().TokenReviews()
+		cfg.TokenAccessReviewClient = client.AuthenticationV1().TokenReviews()
 	}
 
-	// look into configmaps/external-apiserver-authentication for missing authn info
-	if !s.SkipInClusterLookup {
-		err := s.lookupMissingConfigInCluster(client)
+	// get the clientCA information
+	clientCAFileSpecified := len(s.ClientCert.ClientCA) > 0
+	var clientCAProvider dynamiccertificates.CAContentProvider
+	if clientCAFileSpecified {
+		clientCAProvider, err = s.ClientCert.GetClientCAContentProvider()
 		if err != nil {
-			if s.TolerateInClusterLookupFailure {
-				klog.Warningf("Error looking up in-cluster authentication configuration: %v", err)
-				klog.Warningf("Continuing without authentication configuration. This may treat all requests as anonymous.")
-				klog.Warningf("To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false")
-			} else {
-				return err
+			return fmt.Errorf("unable to load client CA file %q: %v", s.ClientCert.ClientCA, err)
+		}
+		cfg.ClientCertificateCAContentProvider = clientCAProvider
+		if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to assign  client CA file: %v", err)
+		}
+
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			clientCAProvider, err = dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "client-ca-file", client)
+			if err != nil {
+				return fmt.Errorf("unable to load configmap based client CA file: %v", err)
 			}
+			cfg.ClientCertificateCAContentProvider = clientCAProvider
+			if err = authenticationInfo.ApplyClientCert(cfg.ClientCertificateCAContentProvider, servingInfo); err != nil {
+				return fmt.Errorf("unable to assign configmap based client CA file: %v", err)
+			}
+
 		}
 	}
 
-	// configure AuthenticationInfo config
-	cfg.ClientCAFile = s.ClientCert.ClientCA
-	if err = c.ApplyClientCert(s.ClientCert.ClientCA, servingInfo); err != nil {
-		return fmt.Errorf("unable to load client CA file: %v", err)
-	}
+	requestHeaderCAFileSpecified := len(s.RequestHeader.ClientCAFile) > 0
+	var requestHeaderConfig *authenticatorfactory.RequestHeaderConfig
+	if requestHeaderCAFileSpecified {
+		requestHeaderConfig, err = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
+		if err != nil {
+			return fmt.Errorf("unable to create request header authentication config: %v", err)
+		}
 
-	cfg.RequestHeaderConfig = s.RequestHeader.ToAuthenticationRequestHeaderConfig()
-	if err = c.ApplyClientCert(s.RequestHeader.ClientCAFile, servingInfo); err != nil {
-		return fmt.Errorf("unable to load client CA file: %v", err)
+	} else if !s.SkipInClusterLookup {
+		if client == nil {
+			klog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
+		} else {
+			requestHeaderConfig, err = s.createRequestHeaderConfig(client)
+			if err != nil {
+				if s.TolerateInClusterLookupFailure {
+					klog.Warningf("Error looking up in-cluster authentication configuration: %v", err)
+					klog.Warningf("Continuing without authentication configuration. This may treat all requests as anonymous.")
+					klog.Warningf("To require authentication configuration lookup to succeed, set --authentication-tolerate-lookup-failure=false")
+				} else {
+					return fmt.Errorf("unable to load configmap based request-header-client-ca-file: %v", err)
+				}
+			}
+		}
+	}
+	if requestHeaderConfig != nil {
+		cfg.RequestHeaderConfig = requestHeaderConfig
+		if err = authenticationInfo.ApplyClientCert(cfg.RequestHeaderConfig.CAContentProvider, servingInfo); err != nil {
+			return fmt.Errorf("unable to load request-header-client-ca-file: %v", err)
+		}
 	}
 
 	// create authenticator
@@ -221,11 +314,11 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(c *server.AuthenticationInfo,
 	if err != nil {
 		return err
 	}
-	c.Authenticator = authenticator
+	authenticationInfo.Authenticator = authenticator
 	if openAPIConfig != nil {
 		openAPIConfig.SecurityDefinitions = securityDefinitions
 	}
-	c.SupportsBasicAuth = false
+	authenticationInfo.SupportsBasicAuth = false
 
 	return nil
 }
@@ -240,97 +333,26 @@ const (
 	authenticationRoleName      = "extension-apiserver-authentication-reader"
 )
 
-func (s *DelegatingAuthenticationOptions) lookupMissingConfigInCluster(client kubernetes.Interface) error {
-	if len(s.ClientCert.ClientCA) > 0 && len(s.RequestHeader.ClientCAFile) > 0 {
-		return nil
-	}
-	if client == nil {
-		if len(s.ClientCert.ClientCA) == 0 {
-			klog.Warningf("No authentication-kubeconfig provided in order to lookup client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
-		}
-		if len(s.RequestHeader.ClientCAFile) == 0 {
-			klog.Warningf("No authentication-kubeconfig provided in order to lookup requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
-		}
-		return nil
+func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
+	requestHeaderCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "requestheader-client-ca-file", client)
+	if err != nil {
+		return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
 	}
 
 	authConfigMap, err := client.CoreV1().ConfigMaps(authenticationConfigMapNamespace).Get(authenticationConfigMapName, metav1.GetOptions{})
 	switch {
 	case errors.IsNotFound(err):
 		// ignore, authConfigMap is nil now
+		return nil, nil
 	case errors.IsForbidden(err):
 		klog.Warningf("Unable to get configmap/%s in %s.  Usually fixed by "+
 			"'kubectl create rolebinding -n %s ROLEBINDING_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
 			authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
-		return err
+		return nil, err
 	case err != nil:
-		return err
-	}
-
-	if len(s.ClientCert.ClientCA) == 0 {
-		if authConfigMap != nil {
-			opt, err := inClusterClientCA(authConfigMap)
-			if err != nil {
-				return err
-			}
-			if opt != nil {
-				s.ClientCert = *opt
-			}
-		}
-		if len(s.ClientCert.ClientCA) == 0 {
-			klog.Warningf("Cluster doesn't provide client-ca-file in configmap/%s in %s, so client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
-		}
-	}
-
-	if len(s.RequestHeader.ClientCAFile) == 0 {
-		if authConfigMap != nil {
-			opt, err := inClusterRequestHeader(authConfigMap)
-			if err != nil {
-				return err
-			}
-			if opt != nil {
-				s.RequestHeader = *opt
-			}
-		}
-		if len(s.RequestHeader.ClientCAFile) == 0 {
-			klog.Warningf("Cluster doesn't provide requestheader-client-ca-file in configmap/%s in %s, so request-header client certificate authentication won't work.", authenticationConfigMapName, authenticationConfigMapNamespace)
-		}
-	}
-
-	return nil
-}
-
-func inClusterClientCA(authConfigMap *v1.ConfigMap) (*ClientCertAuthenticationOptions, error) {
-	clientCA, ok := authConfigMap.Data["client-ca-file"]
-	if !ok {
-		// not having a client-ca is fine, return nil
-		return nil, nil
-	}
-
-	f, err := ioutil.TempFile("", "client-ca-file")
-	if err != nil {
 		return nil, err
 	}
-	if err := ioutil.WriteFile(f.Name(), []byte(clientCA), 0600); err != nil {
-		return nil, err
-	}
-	return &ClientCertAuthenticationOptions{ClientCA: f.Name()}, nil
-}
 
-func inClusterRequestHeader(authConfigMap *v1.ConfigMap) (*RequestHeaderAuthenticationOptions, error) {
-	requestHeaderCA, ok := authConfigMap.Data["requestheader-client-ca-file"]
-	if !ok {
-		// not having a requestheader-client-ca is fine, return nil
-		return nil, nil
-	}
-
-	f, err := ioutil.TempFile("", "requestheader-client-ca-file")
-	if err != nil {
-		return nil, err
-	}
-	if err := ioutil.WriteFile(f.Name(), []byte(requestHeaderCA), 0600); err != nil {
-		return nil, err
-	}
 	usernameHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-username-headers"])
 	if err != nil {
 		return nil, err
@@ -348,12 +370,12 @@ func inClusterRequestHeader(authConfigMap *v1.ConfigMap) (*RequestHeaderAuthenti
 		return nil, err
 	}
 
-	return &RequestHeaderAuthenticationOptions{
-		UsernameHeaders:     usernameHeaders,
-		GroupHeaders:        groupHeaders,
-		ExtraHeaderPrefixes: extraHeaderPrefixes,
-		ClientCAFile:        f.Name(),
-		AllowedNames:        allowedNames,
+	return &authenticatorfactory.RequestHeaderConfig{
+		CAContentProvider:   requestHeaderCAProvider,
+		UsernameHeaders:     headerrequest.StaticStringSlice(usernameHeaders),
+		GroupHeaders:        headerrequest.StaticStringSlice(groupHeaders),
+		ExtraHeaderPrefixes: headerrequest.StaticStringSlice(extraHeaderPrefixes),
+		AllowedClientNames:  headerrequest.StaticStringSlice(allowedNames),
 	}, nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/authorization.go

@@ -146,7 +146,7 @@ func (s *DelegatingAuthorizationOptions) toAuthorizer(client kubernetes.Interfac
 		klog.Warningf("No authorization-kubeconfig provided, so SubjectAccessReview of authorization tokens won't work.")
 	} else {
 		cfg := authorizerfactory.DelegatingAuthorizerConfig{
-			SubjectAccessReviewClient: client.AuthorizationV1beta1().SubjectAccessReviews(),
+			SubjectAccessReviewClient: client.AuthorizationV1().SubjectAccessReviews(),
 			AllowCacheTTL:             s.AllowCacheTTL,
 			DenyCacheTTL:              s.DenyCacheTTL,
 		}

vendor/k8s.io/apiserver/pkg/server/options/deprecated_insecure_serving.go

@@ -54,8 +54,8 @@ func (s *DeprecatedInsecureServingOptions) Validate() []error {
 
 	errors := []error{}
 
-	if s.BindPort < 0 || s.BindPort > 65335 {
-		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65335, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
+	if s.BindPort < 0 || s.BindPort > 65535 {
+		errors = append(errors, fmt.Errorf("insecure port %v must be between 0 and 65535, inclusive. 0 for turning off insecure (HTTP) port", s.BindPort))
 	}
 
 	return errors

vendor/k8s.io/apiserver/pkg/server/options/egress_selector.go

@@ -0,0 +1,92 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package options
+
+import (
+	"fmt"
+	"github.com/spf13/pflag"
+	"k8s.io/utils/path"
+
+	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/egressselector"
+)
+
+// EgressSelectorOptions holds the api server egress selector options.
+// See https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190226-network-proxy.md
+type EgressSelectorOptions struct {
+	// ConfigFile is the file path with api-server egress selector configuration.
+	ConfigFile string
+}
+
+// NewEgressSelectorOptions creates a new instance of EgressSelectorOptions
+//
+// The option is to point to a configuration file for egress/konnectivity.
+// This determines which types of requests use egress/konnectivity and how they use it.
+// If empty the API Server will attempt to connect directly using the network.
+func NewEgressSelectorOptions() *EgressSelectorOptions {
+	return &EgressSelectorOptions{}
+}
+
+// AddFlags adds flags related to admission for a specific APIServer to the specified FlagSet
+func (o *EgressSelectorOptions) AddFlags(fs *pflag.FlagSet) {
+	if o == nil {
+		return
+	}
+
+	fs.StringVar(&o.ConfigFile, "egress-selector-config-file", o.ConfigFile,
+		"File with apiserver egress selector configuration.")
+}
+
+// ApplyTo adds the egress selector settings to the server configuration.
+// In case egress selector settings were not provided by a cluster-admin
+// they will be prepared from the recommended/default/no-op values.
+func (o *EgressSelectorOptions) ApplyTo(c *server.Config) error {
+	if o == nil {
+		return nil
+	}
+
+	npConfig, err := egressselector.ReadEgressSelectorConfiguration(o.ConfigFile)
+	if err != nil {
+		return fmt.Errorf("failed to read egress selector config: %v", err)
+	}
+	errs := egressselector.ValidateEgressSelectorConfiguration(npConfig)
+	if len(errs) > 0 {
+		return fmt.Errorf("failed to validate egress selector configuration: %v", errs.ToAggregate())
+	}
+
+	cs, err := egressselector.NewEgressSelector(npConfig)
+	if err != nil {
+		return fmt.Errorf("failed to setup egress selector with config %#v: %v", npConfig, err)
+	}
+	c.EgressSelector = cs
+	return nil
+}
+
+// Validate verifies flags passed to EgressSelectorOptions.
+func (o *EgressSelectorOptions) Validate() []error {
+	if o == nil || o.ConfigFile == "" {
+		return nil
+	}
+
+	errs := []error{}
+
+	if exists, err := path.Exists(path.CheckFollowSymlink, o.ConfigFile); exists == false || err != nil {
+		errs = append(errs, fmt.Errorf("egress-selector-config-file %s does not exist", o.ConfigFile))
+	}
+
+	return errs
+}

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/OWNERS

@@ -0,0 +1,9 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- sig-auth-encryption-at-rest-approvers
+reviewers:
+- sig-auth-encryption-at-rest-reviewers
+labels:
+- sig/auth
+

vendor/k8s.io/apiserver/pkg/server/options/encryptionconfig/config.go

@@ -0,0 +1,428 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package encryptionconfig
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"encoding/base64"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"sync"
+	"time"
+
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/runtime/serializer"
+	apiserverconfig "k8s.io/apiserver/pkg/apis/config"
+	apiserverconfigv1 "k8s.io/apiserver/pkg/apis/config/v1"
+	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/storage/value"
+	aestransformer "k8s.io/apiserver/pkg/storage/value/encrypt/aes"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/envelope"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/identity"
+	"k8s.io/apiserver/pkg/storage/value/encrypt/secretbox"
+)
+
+const (
+	aesCBCTransformerPrefixV1    = "k8s:enc:aescbc:v1:"
+	aesGCMTransformerPrefixV1    = "k8s:enc:aesgcm:v1:"
+	secretboxTransformerPrefixV1 = "k8s:enc:secretbox:v1:"
+	kmsTransformerPrefixV1       = "k8s:enc:kms:v1:"
+	kmsPluginConnectionTimeout   = 3 * time.Second
+	kmsPluginHealthzTTL          = 3 * time.Second
+)
+
+type kmsPluginHealthzResponse struct {
+	err      error
+	received time.Time
+}
+
+type kmsPluginProbe struct {
+	name string
+	envelope.Service
+	lastResponse *kmsPluginHealthzResponse
+	l            *sync.Mutex
+}
+
+func (h *kmsPluginProbe) toHealthzCheck(idx int) healthz.HealthChecker {
+	return healthz.NamedCheck(fmt.Sprintf("kms-provider-%d", idx), func(r *http.Request) error {
+		return h.Check()
+	})
+}
+
+// GetKMSPluginHealthzCheckers extracts KMSPluginProbes from the EncryptionConfig.
+func GetKMSPluginHealthzCheckers(filepath string) ([]healthz.HealthChecker, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
+	}
+	defer f.Close()
+	var result []healthz.HealthChecker
+	probes, err := getKMSPluginProbes(f)
+	if err != nil {
+		return nil, err
+	}
+
+	for i, p := range probes {
+		probe := p
+		result = append(result, probe.toHealthzCheck(i))
+	}
+	return result, nil
+}
+
+func getKMSPluginProbes(reader io.Reader) ([]*kmsPluginProbe, error) {
+	var result []*kmsPluginProbe
+
+	configFileContents, err := ioutil.ReadAll(reader)
+	if err != nil {
+		return result, fmt.Errorf("could not read content of encryption provider configuration: %v", err)
+	}
+
+	config, err := loadConfig(configFileContents)
+	if err != nil {
+		return result, fmt.Errorf("error while parsing encrypiton provider configuration: %v", err)
+	}
+
+	for _, r := range config.Resources {
+		for _, p := range r.Providers {
+			if p.KMS != nil {
+				timeout := kmsPluginConnectionTimeout
+				if p.KMS.Timeout != nil {
+					if p.KMS.Timeout.Duration <= 0 {
+						return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, timeout should be a positive value", p.KMS.Name)
+					}
+					timeout = p.KMS.Timeout.Duration
+				}
+
+				s, err := envelope.NewGRPCService(p.KMS.Endpoint, timeout)
+				if err != nil {
+					return nil, fmt.Errorf("could not configure KMS-Plugin's probe %q, error: %v", p.KMS.Name, err)
+				}
+
+				result = append(result, &kmsPluginProbe{
+					name:         p.KMS.Name,
+					Service:      s,
+					l:            &sync.Mutex{},
+					lastResponse: &kmsPluginHealthzResponse{},
+				})
+			}
+		}
+	}
+
+	return result, nil
+}
+
+// Check encrypts and decrypts test data against KMS-Plugin's gRPC endpoint.
+func (h *kmsPluginProbe) Check() error {
+	h.l.Lock()
+	defer h.l.Unlock()
+
+	if (time.Since(h.lastResponse.received)) < kmsPluginHealthzTTL {
+		return h.lastResponse.err
+	}
+
+	p, err := h.Service.Encrypt([]byte("ping"))
+	if err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		return fmt.Errorf("failed to perform encrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+	}
+
+	if _, err := h.Service.Decrypt(p); err != nil {
+		h.lastResponse = &kmsPluginHealthzResponse{err: err, received: time.Now()}
+		return fmt.Errorf("failed to perform decrypt section of the healthz check for KMS Provider %s, error: %v", h.name, err)
+	}
+
+	h.lastResponse = &kmsPluginHealthzResponse{err: nil, received: time.Now()}
+	return nil
+}
+
+// GetTransformerOverrides returns the transformer overrides by reading and parsing the encryption provider configuration file
+func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Transformer, error) {
+	f, err := os.Open(filepath)
+	if err != nil {
+		return nil, fmt.Errorf("error opening encryption provider configuration file %q: %v", filepath, err)
+	}
+	defer f.Close()
+
+	result, err := ParseEncryptionConfiguration(f)
+	if err != nil {
+		return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
+	}
+	return result, nil
+}
+
+// ParseEncryptionConfiguration parses configuration data and returns the transformer overrides
+func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
+	configFileContents, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, fmt.Errorf("could not read contents: %v", err)
+	}
+
+	config, err := loadConfig(configFileContents)
+	if err != nil {
+		return nil, fmt.Errorf("error while parsing file: %v", err)
+	}
+
+	resourceToPrefixTransformer := map[schema.GroupResource][]value.PrefixTransformer{}
+
+	// For each entry in the configuration
+	for _, resourceConfig := range config.Resources {
+		transformers, err := GetPrefixTransformers(&resourceConfig)
+		if err != nil {
+			return nil, err
+		}
+
+		// For each resource, create a list of providers to use
+		for _, resource := range resourceConfig.Resources {
+			gr := schema.ParseGroupResource(resource)
+			resourceToPrefixTransformer[gr] = append(
+				resourceToPrefixTransformer[gr], transformers...)
+		}
+	}
+
+	result := map[schema.GroupResource]value.Transformer{}
+	for gr, transList := range resourceToPrefixTransformer {
+		result[gr] = value.NewMutableTransformer(value.NewPrefixTransformers(fmt.Errorf("no matching prefix found"), transList...))
+	}
+	return result, nil
+
+}
+
+// loadConfig decodes data as a EncryptionConfiguration object.
+func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
+	scheme := runtime.NewScheme()
+	codecs := serializer.NewCodecFactory(scheme)
+	apiserverconfig.AddToScheme(scheme)
+	apiserverconfigv1.AddToScheme(scheme)
+
+	configObj, gvk, err := codecs.UniversalDecoder().Decode(data, nil, nil)
+	if err != nil {
+		return nil, err
+	}
+	config, ok := configObj.(*apiserverconfig.EncryptionConfiguration)
+	if !ok {
+		return nil, fmt.Errorf("got unexpected config type: %v", gvk)
+	}
+	return config, nil
+}
+
+// The factory to create kms service. This is to make writing test easier.
+var envelopeServiceFactory = envelope.NewGRPCService
+
+// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration.
+func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
+	var result []value.PrefixTransformer
+	for _, provider := range config.Providers {
+		found := false
+
+		var transformer value.PrefixTransformer
+		var err error
+
+		if provider.AESGCM != nil {
+			transformer, err = GetAESPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
+			if err != nil {
+				return result, err
+			}
+			found = true
+		}
+
+		if provider.AESCBC != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer, err = GetAESPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
+			found = true
+		}
+
+		if provider.Secretbox != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer, err = GetSecretboxPrefixTransformer(provider.Secretbox)
+			found = true
+		}
+
+		if provider.Identity != nil {
+			if found == true {
+				return result, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+			transformer = value.PrefixTransformer{
+				Transformer: identity.NewEncryptCheckTransformer(),
+				Prefix:      []byte{},
+			}
+			found = true
+		}
+
+		if provider.KMS != nil {
+			if found == true {
+				return nil, fmt.Errorf("more than one provider specified in a single element, should split into different list elements")
+			}
+
+			// Ensure the endpoint is provided.
+			if len(provider.KMS.Endpoint) == 0 {
+				return nil, fmt.Errorf("remote KMS provider can't use empty string as endpoint")
+			}
+
+			timeout := kmsPluginConnectionTimeout
+			if provider.KMS.Timeout != nil {
+				if provider.KMS.Timeout.Duration <= 0 {
+					return nil, fmt.Errorf("could not configure KMS plugin %q, timeout should be a positive value", provider.KMS.Name)
+				}
+				timeout = provider.KMS.Timeout.Duration
+			}
+
+			// Get gRPC client service with endpoint.
+			envelopeService, err := envelopeServiceFactory(provider.KMS.Endpoint, timeout)
+			if err != nil {
+				return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
+			}
+
+			transformer, err = getEnvelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
+			found = true
+		}
+
+		if err != nil {
+			return result, err
+		}
+		result = append(result, transformer)
+
+		if found == false {
+			return result, fmt.Errorf("invalid provider configuration: at least one provider must be specified")
+		}
+	}
+	return result, nil
+}
+
+// BlockTransformerFunc takes an AES cipher block and returns a value transformer.
+type BlockTransformerFunc func(cipher.Block) value.Transformer
+
+// GetAESPrefixTransformer returns a prefix transformer from the provided configuration.
+// Returns an AES transformer based on the provided prefix and block transformer.
+func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("aes provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+		block, err := aes.NewCipher(key)
+		if err != nil {
+			return result, fmt.Errorf("error while creating cipher for named key %s: %s", keyData.Name, err)
+		}
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: fn(block),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided AES transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(prefix),
+	}
+	return result, nil
+}
+
+// GetSecretboxPrefixTransformer returns a prefix transformer from the provided configuration
+func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
+	var result value.PrefixTransformer
+
+	if len(config.Keys) == 0 {
+		return result, fmt.Errorf("secretbox provider has no valid keys")
+	}
+	for _, key := range config.Keys {
+		if key.Name == "" {
+			return result, fmt.Errorf("key with invalid name provided")
+		}
+		if key.Secret == "" {
+			return result, fmt.Errorf("key %v has no provided secret", key.Name)
+		}
+	}
+
+	keyTransformers := []value.PrefixTransformer{}
+
+	for _, keyData := range config.Keys {
+		key, err := base64.StdEncoding.DecodeString(keyData.Secret)
+		if err != nil {
+			return result, fmt.Errorf("could not obtain secret for named key %s: %s", keyData.Name, err)
+		}
+
+		if len(key) != 32 {
+			return result, fmt.Errorf("expected key size 32 for secretbox provider, got %v", len(key))
+		}
+
+		keyArray := [32]byte{}
+		copy(keyArray[:], key)
+
+		// Create a new PrefixTransformer for this key
+		keyTransformers = append(keyTransformers,
+			value.PrefixTransformer{
+				Transformer: secretbox.NewSecretboxTransformer(keyArray),
+				Prefix:      []byte(keyData.Name + ":"),
+			})
+	}
+
+	// Create a prefixTransformer which can choose between these keys
+	keyTransformer := value.NewPrefixTransformers(
+		fmt.Errorf("no matching key was found for the provided Secretbox transformer"), keyTransformers...)
+
+	// Create a PrefixTransformer which shall later be put in a list with other providers
+	result = value.PrefixTransformer{
+		Transformer: keyTransformer,
+		Prefix:      []byte(secretboxTransformerPrefixV1),
+	}
+	return result, nil
+}
+
+// getEnvelopePrefixTransformer returns a prefix transformer from the provided config.
+// envelopeService is used as the root of trust.
+func getEnvelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
+	envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(config.CacheSize), aestransformer.NewCBCTransformer)
+	if err != nil {
+		return value.PrefixTransformer{}, err
+	}
+	return value.PrefixTransformer{
+		Transformer: envelopeTransformer,
+		Prefix:      []byte(prefix + config.Name + ":"),
+	}, nil
+}

vendor/k8s.io/apiserver/pkg/server/options/etcd.go

@@ -31,6 +31,7 @@ import (
 	genericregistry "k8s.io/apiserver/pkg/registry/generic/registry"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/server/healthz"
+	"k8s.io/apiserver/pkg/server/options/encryptionconfig"
 	serverstorage "k8s.io/apiserver/pkg/server/storage"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
 	storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
@@ -160,7 +161,7 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
 	fs.StringVar(&s.StorageConfig.Transport.CertFile, "etcd-certfile", s.StorageConfig.Transport.CertFile,
 		"SSL certification file used to secure etcd communication.")
 
-	fs.StringVar(&s.StorageConfig.Transport.CAFile, "etcd-cafile", s.StorageConfig.Transport.CAFile,
+	fs.StringVar(&s.StorageConfig.Transport.TrustedCAFile, "etcd-cafile", s.StorageConfig.Transport.TrustedCAFile,
 		"SSL Certificate Authority file used to secure etcd communication.")
 
 	fs.StringVar(&s.EncryptionProviderConfigFilepath, "experimental-encryption-provider-config", s.EncryptionProviderConfigFilepath,
@@ -201,9 +202,18 @@ func (s *EtcdOptions) addEtcdHealthEndpoint(c *server.Config) error {
 	if err != nil {
 		return err
 	}
-	c.HealthzChecks = append(c.HealthzChecks, healthz.NamedCheck("etcd", func(r *http.Request) error {
+	c.AddHealthChecks(healthz.NamedCheck("etcd", func(r *http.Request) error {
 		return healthCheck()
 	}))
+
+	if s.EncryptionProviderConfigFilepath != "" {
+		kmsPluginHealthzChecks, err := encryptionconfig.GetKMSPluginHealthzCheckers(s.EncryptionProviderConfigFilepath)
+		if err != nil {
+			return err
+		}
+		c.AddHealthChecks(kmsPluginHealthzChecks...)
+	}
+
 	return nil
 }
 

vendor/k8s.io/apiserver/pkg/server/options/recommended.go

@@ -18,11 +18,13 @@ package options
 
 import (
 	"github.com/spf13/pflag"
+	"k8s.io/apiserver/pkg/util/feature"
 
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/admission"
 	"k8s.io/apiserver/pkg/server"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
+	"k8s.io/component-base/featuregate"
 )
 
 // RecommendedOptions contains the recommended options for running an API server.
@@ -37,6 +39,8 @@ type RecommendedOptions struct {
 	Features       *FeatureOptions
 	CoreAPI        *CoreAPIOptions
 
+	// FeatureGate is a way to plumb feature gate through if you have them.
+	FeatureGate featuregate.FeatureGate
 	// ExtraAdmissionInitializers is called once after all ApplyTo from the options above, to pass the returned
 	// admission plugin initializers to Admission.ApplyTo.
 	ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
@@ -44,6 +48,8 @@ type RecommendedOptions struct {
 	// ProcessInfo is used to identify events created by the server.
 	ProcessInfo *ProcessInfo
 	Webhook     *WebhookOptions
+	// API Server Egress Selector is used to control outbound traffic from the API Server
+	EgressSelector *EgressSelectorOptions
 }
 
 func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
@@ -56,17 +62,22 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc
 	sso.HTTP2MaxStreamsPerConnection = 1000
 
 	return &RecommendedOptions{
-		Etcd:                       NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)),
-		SecureServing:              sso.WithLoopback(),
-		Authentication:             NewDelegatingAuthenticationOptions(),
-		Authorization:              NewDelegatingAuthorizationOptions(),
-		Audit:                      NewAuditOptions(),
-		Features:                   NewFeatureOptions(),
-		CoreAPI:                    NewCoreAPIOptions(),
+		Etcd:           NewEtcdOptions(storagebackend.NewDefaultConfig(prefix, codec)),
+		SecureServing:  sso.WithLoopback(),
+		Authentication: NewDelegatingAuthenticationOptions(),
+		Authorization:  NewDelegatingAuthorizationOptions(),
+		Audit:          NewAuditOptions(),
+		Features:       NewFeatureOptions(),
+		CoreAPI:        NewCoreAPIOptions(),
+		// Wired a global by default that sadly people will abuse to have different meanings in different repos.
+		// Please consider creating your own FeatureGate so you can have a consistent meaning for what a variable contains
+		// across different repos.  Future you will thank you.
+		FeatureGate:                feature.DefaultFeatureGate,
 		ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
 		Admission:                  NewAdmissionOptions(),
 		ProcessInfo:                processInfo,
 		Webhook:                    NewWebhookOptions(),
+		EgressSelector:             NewEgressSelectorOptions(),
 	}
 }
 
@@ -79,6 +90,7 @@ func (o *RecommendedOptions) AddFlags(fs *pflag.FlagSet) {
 	o.Features.AddFlags(fs)
 	o.CoreAPI.AddFlags(fs)
 	o.Admission.AddFlags(fs)
+	o.EgressSelector.AddFlags(fs)
 }
 
 // ApplyTo adds RecommendedOptions to the server configuration.
@@ -107,7 +119,10 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
 	}
 	if initializers, err := o.ExtraAdmissionInitializers(config); err != nil {
 		return err
-	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, initializers...); err != nil {
+	} else if err := o.Admission.ApplyTo(&config.Config, config.SharedInformerFactory, config.ClientConfig, o.FeatureGate, initializers...); err != nil {
+		return err
+	}
+	if err := o.EgressSelector.ApplyTo(&config.Config); err != nil {
 		return err
 	}
 
@@ -124,6 +139,7 @@ func (o *RecommendedOptions) Validate() []error {
 	errors = append(errors, o.Features.Validate()...)
 	errors = append(errors, o.CoreAPI.Validate()...)
 	errors = append(errors, o.Admission.Validate()...)
+	errors = append(errors, o.EgressSelector.Validate()...)
 
 	return errors
 }

vendor/k8s.io/apiserver/pkg/server/options/server_run_options.go

@@ -41,7 +41,9 @@ type ServerRunOptions struct {
 	MaxRequestsInFlight         int
 	MaxMutatingRequestsInFlight int
 	RequestTimeout              time.Duration
+	LivezGracePeriod            time.Duration
 	MinRequestTimeout           int
+	ShutdownDelayDuration       time.Duration
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
 	JSONPatchMaxCopyBytes int64
@@ -49,9 +51,9 @@ type ServerRunOptions struct {
 	// decoded in a write request. 0 means no limit.
 	// We intentionally did not add a flag for this option. Users of the
 	// apiserver library can wire it to a flag.
-	MaxRequestBodyBytes       int64
-	TargetRAMMB               int
-	EnableInfightQuotaHandler bool
+	MaxRequestBodyBytes        int64
+	TargetRAMMB                int
+	EnableInflightQuotaHandler bool
 }
 
 func NewServerRunOptions() *ServerRunOptions {
@@ -60,7 +62,9 @@ func NewServerRunOptions() *ServerRunOptions {
 		MaxRequestsInFlight:         defaults.MaxRequestsInFlight,
 		MaxMutatingRequestsInFlight: defaults.MaxMutatingRequestsInFlight,
 		RequestTimeout:              defaults.RequestTimeout,
+		LivezGracePeriod:            defaults.LivezGracePeriod,
 		MinRequestTimeout:           defaults.MinRequestTimeout,
+		ShutdownDelayDuration:       defaults.ShutdownDelayDuration,
 		JSONPatchMaxCopyBytes:       defaults.JSONPatchMaxCopyBytes,
 		MaxRequestBodyBytes:         defaults.MaxRequestBodyBytes,
 	}
@@ -72,8 +76,10 @@ func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
 	c.ExternalAddress = s.ExternalHost
 	c.MaxRequestsInFlight = s.MaxRequestsInFlight
 	c.MaxMutatingRequestsInFlight = s.MaxMutatingRequestsInFlight
+	c.LivezGracePeriod = s.LivezGracePeriod
 	c.RequestTimeout = s.RequestTimeout
 	c.MinRequestTimeout = s.MinRequestTimeout
+	c.ShutdownDelayDuration = s.ShutdownDelayDuration
 	c.JSONPatchMaxCopyBytes = s.JSONPatchMaxCopyBytes
 	c.MaxRequestBodyBytes = s.MaxRequestBodyBytes
 	c.PublicAddress = s.AdvertiseAddress
@@ -106,10 +112,14 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
 	}
 
-	if s.EnableInfightQuotaHandler {
-		if !utilfeature.DefaultFeatureGate.Enabled(features.RequestManagement) {
+	if s.LivezGracePeriod < 0 {
+		errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
+	}
+
+	if s.EnableInflightQuotaHandler {
+		if !utilfeature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
 			errors = append(errors, fmt.Errorf("--enable-inflight-quota-handler can not be set if feature "+
-				"gate RequestManagement is disabled"))
+				"gate APIPriorityAndFairness is disabled"))
 		}
 		if s.MaxMutatingRequestsInFlight != 0 {
 			errors = append(errors, fmt.Errorf("--max-mutating-requests-inflight=%v "+
@@ -136,6 +146,10 @@ func (s *ServerRunOptions) Validate() []error {
 		errors = append(errors, fmt.Errorf("--min-request-timeout can not be negative value"))
 	}
 
+	if s.ShutdownDelayDuration < 0 {
+		errors = append(errors, fmt.Errorf("--shutdown-delay-duration can not be negative value"))
+	}
+
 	if s.JSONPatchMaxCopyBytes < 0 {
 		errors = append(errors, fmt.Errorf("--json-patch-max-copy-bytes can not be negative value"))
 	}
@@ -185,14 +199,24 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
 		"it out. This is the default request timeout for requests but may be overridden by flags such as "+
 		"--min-request-timeout for specific types of requests.")
 
+	fs.DurationVar(&s.LivezGracePeriod, "livez-grace-period", s.LivezGracePeriod, ""+
+		"This option represents the maximum amount of time it should take for apiserver to complete its startup sequence "+
+		"and become live. From apiserver's start time to when this amount of time has elapsed, /livez will assume "+
+		"that unfinished post-start hooks will complete successfully and therefore return true.")
+
 	fs.IntVar(&s.MinRequestTimeout, "min-request-timeout", s.MinRequestTimeout, ""+
 		"An optional field indicating the minimum number of seconds a handler must keep "+
 		"a request open before timing it out. Currently only honored by the watch request "+
 		"handler, which picks a randomized value above this number as the connection timeout, "+
 		"to spread out load.")
 
-	fs.BoolVar(&s.EnableInfightQuotaHandler, "enable-inflight-quota-handler", s.EnableInfightQuotaHandler, ""+
+	fs.BoolVar(&s.EnableInflightQuotaHandler, "enable-inflight-quota-handler", s.EnableInflightQuotaHandler, ""+
 		"If true, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
 
+	fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
+		"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
+		"returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
+		"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
+
 	utilfeature.DefaultMutableFeatureGate.AddFlag(fs)
 }

vendor/k8s.io/apiserver/pkg/server/options/serving.go

@@ -17,7 +17,6 @@ limitations under the License.
 package options
 
 import (
-	"crypto/tls"
 	"fmt"
 	"net"
 	"path"
@@ -29,6 +28,7 @@ import (
 
 	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	certutil "k8s.io/client-go/util/cert"
 	"k8s.io/client-go/util/keyutil"
 	cliflag "k8s.io/component-base/cli/flag"
@@ -88,7 +88,7 @@ type GeneratableKeyCert struct {
 	PairName string
 
 	// GeneratedCert holds an in-memory generated certificate if CertFile/KeyFile aren't explicitly set, and CertDirectory/PairName are not set.
-	GeneratedCert *tls.Certificate
+	GeneratedCert dynamiccertificates.CertKeyContentProvider
 
 	// FixtureDirectory is a directory that contains test fixture used to avoid regeneration of certs during tests.
 	// The format is:
@@ -109,10 +109,10 @@ func NewSecureServingOptions() *SecureServingOptions {
 }
 
 func (s *SecureServingOptions) DefaultExternalAddress() (net.IP, error) {
-	if !s.ExternalAddress.IsUnspecified() {
+	if s.ExternalAddress != nil && !s.ExternalAddress.IsUnspecified() {
 		return s.ExternalAddress, nil
 	}
-	return utilnet.ChooseBindAddress(s.BindAddress)
+	return utilnet.ResolveBindAddress(s.BindAddress)
 }
 
 func (s *SecureServingOptions) Validate() []error {
@@ -225,11 +225,11 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	serverCertFile, serverKeyFile := s.ServerCert.CertKey.CertFile, s.ServerCert.CertKey.KeyFile
 	// load main cert
 	if len(serverCertFile) != 0 || len(serverKeyFile) != 0 {
-		tlsCert, err := tls.LoadX509KeyPair(serverCertFile, serverKeyFile)
+		var err error
+		c.Cert, err = dynamiccertificates.NewDynamicServingContentFromFiles("serving-cert", serverCertFile, serverKeyFile)
 		if err != nil {
-			return fmt.Errorf("unable to load server certificate: %v", err)
+			return err
 		}
-		c.Cert = &tlsCert
 	} else if s.ServerCert.GeneratedCert != nil {
 		c.Cert = s.ServerCert.GeneratedCert
 	}
@@ -249,21 +249,15 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
 	}
 
 	// load SNI certs
-	namedTLSCerts := make([]server.NamedTLSCert, 0, len(s.SNICertKeys))
+	namedTLSCerts := make([]dynamiccertificates.SNICertKeyContentProvider, 0, len(s.SNICertKeys))
 	for _, nck := range s.SNICertKeys {
-		tlsCert, err := tls.LoadX509KeyPair(nck.CertFile, nck.KeyFile)
-		namedTLSCerts = append(namedTLSCerts, server.NamedTLSCert{
-			TLSCert: tlsCert,
-			Names:   nck.Names,
-		})
+		tlsCert, err := dynamiccertificates.NewDynamicSNIContentFromFiles("sni-serving-cert", nck.CertFile, nck.KeyFile, nck.Names...)
+		namedTLSCerts = append(namedTLSCerts, tlsCert)
 		if err != nil {
 			return fmt.Errorf("failed to load SNI cert and key: %v", err)
 		}
 	}
-	c.SNICerts, err = server.GetNamedCertificateMap(namedTLSCerts)
-	if err != nil {
-		return err
-	}
+	c.SNICerts = namedTLSCerts
 
 	return nil
 }
@@ -311,11 +305,10 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
 			}
 			klog.Infof("Generated self-signed cert (%s, %s)", keyCert.CertFile, keyCert.KeyFile)
 		} else {
-			tlsCert, err := tls.X509KeyPair(cert, key)
+			s.ServerCert.GeneratedCert, err = dynamiccertificates.NewStaticCertKeyContent("Generated self signed cert", cert, key)
 			if err != nil {
-				return fmt.Errorf("unable to generate self signed cert: %v", err)
+				return err
 			}
-			s.ServerCert.GeneratedCert = &tlsCert
 			klog.Infof("Generated self-signed cert in-memory")
 		}
 	}

vendor/k8s.io/apiserver/pkg/server/options/serving_with_loopback.go

@@ -17,12 +17,12 @@ limitations under the License.
 package options
 
 import (
-	"crypto/tls"
 	"fmt"
 
-	"github.com/pborman/uuid"
+	"github.com/google/uuid"
 
 	"k8s.io/apiserver/pkg/server"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 	"k8s.io/client-go/rest"
 	certutil "k8s.io/client-go/util/cert"
 )
@@ -55,12 +55,12 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 	if err != nil {
 		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
 	}
-	tlsCert, err := tls.X509KeyPair(certPem, keyPem)
+	certProvider, err := dynamiccertificates.NewStaticSNICertKeyContent("self-signed loopback", certPem, keyPem, server.LoopbackClientServerNameOverride)
 	if err != nil {
 		return fmt.Errorf("failed to generate self-signed certificate for loopback connection: %v", err)
 	}
 
-	secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.NewRandom().String(), certPem)
+	secureLoopbackClientConfig, err := (*secureServingInfo).NewLoopbackClientConfig(uuid.New().String(), certPem)
 	switch {
 	// if we failed and there's no fallback loopback client config, we need to fail
 	case err != nil && *loopbackClientConfig == nil:
@@ -71,7 +71,8 @@ func (s *SecureServingOptionsWithLoopback) ApplyTo(secureServingInfo **server.Se
 
 	default:
 		*loopbackClientConfig = secureLoopbackClientConfig
-		(*secureServingInfo).SNICerts[server.LoopbackClientServerNameOverride] = &tlsCert
+		// Write to the front of SNICerts so that this overrides any other certs with the same name
+		(*secureServingInfo).SNICerts = append([]dynamiccertificates.SNICertKeyContentProvider{certProvider}, (*secureServingInfo).SNICerts...)
 	}
 
 	return nil

vendor/k8s.io/apiserver/pkg/server/resourceconfig/helpers.go

@@ -18,6 +18,7 @@ package resourceconfig
 
 import (
 	"fmt"
+	"regexp"
 	"strconv"
 	"strings"
 
@@ -51,6 +52,33 @@ func MergeResourceEncodingConfigs(
 	return resourceEncodingConfig
 }
 
+// Recognized values for the --runtime-config parameter to enable/disable groups of APIs
+const (
+	APIAll   = "api/all"
+	APIGA    = "api/ga"
+	APIBeta  = "api/beta"
+	APIAlpha = "api/alpha"
+)
+
+var (
+	gaPattern    = regexp.MustCompile(`^v\d+$`)
+	betaPattern  = regexp.MustCompile(`^v\d+beta\d+$`)
+	alphaPattern = regexp.MustCompile(`^v\d+alpha\d+$`)
+
+	matchers = map[string]func(gv schema.GroupVersion) bool{
+		// allows users to address all api versions
+		APIAll: func(gv schema.GroupVersion) bool { return true },
+		// allows users to address all api versions in the form v[0-9]+
+		APIGA: func(gv schema.GroupVersion) bool { return gaPattern.MatchString(gv.Version) },
+		// allows users to address all beta api versions
+		APIBeta: func(gv schema.GroupVersion) bool { return betaPattern.MatchString(gv.Version) },
+		// allows users to address all alpha api versions
+		APIAlpha: func(gv schema.GroupVersion) bool { return alphaPattern.MatchString(gv.Version) },
+	}
+
+	matcherOrder = []string{APIAll, APIGA, APIBeta, APIAlpha}
+)
+
 // MergeAPIResourceConfigs merges the given defaultAPIResourceConfig with the given resourceConfigOverrides.
 // Exclude the groups not registered in registry, and check if version is
 // not registered in group, then it will fail.
@@ -62,14 +90,15 @@ func MergeAPIResourceConfigs(
 	resourceConfig := defaultAPIResourceConfig
 	overrides := resourceConfigOverrides
 
-	// "api/all=false" allows users to selectively enable specific api versions.
-	allAPIFlagValue, ok := overrides["api/all"]
-	if ok {
-		if allAPIFlagValue == "false" {
-			// Disable all group versions.
-			resourceConfig.DisableAll()
-		} else if allAPIFlagValue == "true" {
-			resourceConfig.EnableAll()
+	for _, flag := range matcherOrder {
+		if value, ok := overrides[flag]; ok {
+			if value == "false" {
+				resourceConfig.DisableMatchingVersions(matchers[flag])
+			} else if value == "true" {
+				resourceConfig.EnableMatchingVersions(matchers[flag])
+			} else {
+				return nil, fmt.Errorf("invalid value %v=%v", flag, value)
+			}
 		}
 	}
 
@@ -78,7 +107,7 @@ func MergeAPIResourceConfigs(
 	// Iterate through all group/version overrides specified in runtimeConfig.
 	for key := range overrides {
 		// Have already handled them above. Can skip them here.
-		if key == "api/all" {
+		if _, ok := matchers[key]; ok {
 			continue
 		}
 
@@ -153,7 +182,7 @@ func getRuntimeConfigValue(overrides cliflag.ConfigurationMap, apiKey string, de
 func ParseGroups(resourceConfig cliflag.ConfigurationMap) ([]string, error) {
 	groups := []string{}
 	for key := range resourceConfig {
-		if key == "api/all" {
+		if _, ok := matchers[key]; ok {
 			continue
 		}
 		tokens := strings.Split(key, "/")

vendor/k8s.io/apiserver/pkg/server/routes/flags.go

@@ -121,6 +121,7 @@ func StringFlagPutHandler(setter StringFlagSetterFunc) http.HandlerFunc {
 // writePlainText renders a simple string response.
 func writePlainText(statusCode int, text string, w http.ResponseWriter) {
 	w.Header().Set("Content-Type", "text/plain")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
 	w.WriteHeader(statusCode)
 	fmt.Fprintln(w, text)
 }

vendor/k8s.io/apiserver/pkg/server/routes/metrics.go

@@ -22,9 +22,8 @@ import (
 
 	apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
 	"k8s.io/apiserver/pkg/server/mux"
-	etcdmetrics "k8s.io/apiserver/pkg/storage/etcd/metrics"
-
-	"github.com/prometheus/client_golang/prometheus"
+	etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
 )
 
 // DefaultMetrics installs the default prometheus metrics handler
@@ -33,7 +32,7 @@ type DefaultMetrics struct{}
 // Install adds the DefaultMetrics handler
 func (m DefaultMetrics) Install(c *mux.PathRecorderMux) {
 	register()
-	c.Handle("/metrics", prometheus.Handler())
+	c.Handle("/metrics", legacyregistry.Handler())
 }
 
 // MetricsWithReset install the prometheus metrics handler extended with support for the DELETE method
@@ -43,11 +42,11 @@ type MetricsWithReset struct{}
 // Install adds the MetricsWithReset handler
 func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
 	register()
-	defaultMetricsHandler := prometheus.Handler().ServeHTTP
+	defaultMetricsHandler := legacyregistry.Handler().ServeHTTP
 	c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
 		if req.Method == "DELETE" {
 			apimetrics.Reset()
-			etcdmetrics.Reset()
+			etcd3metrics.Reset()
 			io.WriteString(w, "metrics reset\n")
 			return
 		}
@@ -58,5 +57,5 @@ func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
 // register apiserver and etcd metrics
 func register() {
 	apimetrics.Register()
-	etcdmetrics.Register()
+	etcd3metrics.Register()
 }

vendor/k8s.io/apiserver/pkg/server/secure_serving.go

@@ -19,24 +19,114 @@ package server
 import (
 	"context"
 	"crypto/tls"
-	"crypto/x509"
 	"fmt"
 	"net"
 	"net/http"
-	"strings"
 	"time"
 
 	"golang.org/x/net/http2"
 	"k8s.io/klog"
 
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
-	"k8s.io/apimachinery/pkg/util/validation"
+	"k8s.io/apiserver/pkg/server/dynamiccertificates"
 )
 
 const (
 	defaultKeepAlivePeriod = 3 * time.Minute
 )
 
+// tlsConfig produces the tls.Config to serve with.
+func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, error) {
+	tlsConfig := &tls.Config{
+		// Can't use SSLv3 because of POODLE and BEAST
+		// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
+		// Can't use TLSv1.1 because of RC4 cipher usage
+		MinVersion: tls.VersionTLS12,
+		// enable HTTP2 for go's 1.7 HTTP Server
+		NextProtos: []string{"h2", "http/1.1"},
+	}
+
+	// these are static aspects of the tls.Config
+	if s.DisableHTTP2 {
+		klog.Info("Forcing use of http/1.1 only")
+		tlsConfig.NextProtos = []string{"http/1.1"}
+	}
+	if s.MinTLSVersion > 0 {
+		tlsConfig.MinVersion = s.MinTLSVersion
+	}
+	if len(s.CipherSuites) > 0 {
+		tlsConfig.CipherSuites = s.CipherSuites
+	}
+
+	if s.ClientCA != nil {
+		// Populate PeerCertificates in requests, but don't reject connections without certificates
+		// This allows certificates to be validated by authenticators, while still allowing other auth types
+		tlsConfig.ClientAuth = tls.RequestClientCert
+	}
+
+	if s.ClientCA != nil || s.Cert != nil || len(s.SNICerts) > 0 {
+		dynamicCertificateController := dynamiccertificates.NewDynamicServingCertificateController(
+			*tlsConfig,
+			s.ClientCA,
+			s.Cert,
+			s.SNICerts,
+			nil, // TODO see how to plumb an event recorder down in here. For now this results in simply klog messages.
+		)
+		// register if possible
+		if notifier, ok := s.ClientCA.(dynamiccertificates.Notifier); ok {
+			notifier.AddListener(dynamicCertificateController)
+		}
+		if notifier, ok := s.Cert.(dynamiccertificates.Notifier); ok {
+			notifier.AddListener(dynamicCertificateController)
+		}
+		// start controllers if possible
+		if controller, ok := s.ClientCA.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(); err != nil {
+				klog.Warningf("Initial population of client CA failed: %v", err)
+			}
+
+			go controller.Run(1, stopCh)
+		}
+		if controller, ok := s.Cert.(dynamiccertificates.ControllerRunner); ok {
+			// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+			// Files are required to be populated already, so this is for convenience.
+			if err := controller.RunOnce(); err != nil {
+				klog.Warningf("Initial population of default serving certificate failed: %v", err)
+			}
+
+			go controller.Run(1, stopCh)
+		}
+		for _, sniCert := range s.SNICerts {
+			if notifier, ok := sniCert.(dynamiccertificates.Notifier); ok {
+				notifier.AddListener(dynamicCertificateController)
+			}
+
+			if controller, ok := sniCert.(dynamiccertificates.ControllerRunner); ok {
+				// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+				// Files are required to be populated already, so this is for convenience.
+				if err := controller.RunOnce(); err != nil {
+					klog.Warningf("Initial population of SNI serving certificate failed: %v", err)
+				}
+
+				go controller.Run(1, stopCh)
+			}
+		}
+
+		// runonce to try to prime data.  If this fails, it's ok because we fail closed.
+		// Files are required to be populated already, so this is for convenience.
+		if err := dynamicCertificateController.RunOnce(); err != nil {
+			klog.Warningf("Initial population of dynamic certificates failed: %v", err)
+		}
+		go dynamicCertificateController.Run(1, stopCh)
+
+		tlsConfig.GetConfigForClient = dynamicCertificateController.GetConfigForClient
+	}
+
+	return tlsConfig, nil
+}
+
 // Serve runs the secure http server. It fails only if certificates cannot be loaded or the initial listen call fails.
 // The actual server loop (stoppable by closing stopCh) runs in a go routine, i.e. Serve does not block.
 // It returns a stoppedCh that is closed when all non-hijacked active requests have been processed.
@@ -45,46 +135,16 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
 		return nil, fmt.Errorf("listener must not be nil")
 	}
 
+	tlsConfig, err := s.tlsConfig(stopCh)
+	if err != nil {
+		return nil, err
+	}
+
 	secureServer := &http.Server{
 		Addr:           s.Listener.Addr().String(),
 		Handler:        handler,
 		MaxHeaderBytes: 1 << 20,
-		TLSConfig: &tls.Config{
-			NameToCertificate: s.SNICerts,
-			// Can't use SSLv3 because of POODLE and BEAST
-			// Can't use TLSv1.0 because of POODLE and BEAST using CBC cipher
-			// Can't use TLSv1.1 because of RC4 cipher usage
-			MinVersion: tls.VersionTLS12,
-			// enable HTTP2 for go's 1.7 HTTP Server
-			NextProtos: []string{"h2", "http/1.1"},
-		},
-	}
-
-	if s.MinTLSVersion > 0 {
-		secureServer.TLSConfig.MinVersion = s.MinTLSVersion
-	}
-	if len(s.CipherSuites) > 0 {
-		secureServer.TLSConfig.CipherSuites = s.CipherSuites
-	}
-
-	if s.Cert != nil {
-		secureServer.TLSConfig.Certificates = []tls.Certificate{*s.Cert}
-	}
-
-	// append all named certs. Otherwise, the go tls stack will think no SNI processing
-	// is necessary because there is only one cert anyway.
-	// Moreover, if ServerCert.CertFile/ServerCert.KeyFile are not set, the first SNI
-	// cert will become the default cert. That's what we expect anyway.
-	for _, c := range s.SNICerts {
-		secureServer.TLSConfig.Certificates = append(secureServer.TLSConfig.Certificates, *c)
-	}
-
-	if s.ClientCA != nil {
-		// Populate PeerCertificates in requests, but don't reject connections without certificates
-		// This allows certificates to be validated by authenticators, while still allowing other auth types
-		secureServer.TLSConfig.ClientAuth = tls.RequestClientCert
-		// Specify allowed CAs for client certificates
-		secureServer.TLSConfig.ClientCAs = s.ClientCA
+		TLSConfig:      tlsConfig,
 	}
 
 	// At least 99% of serialized resources in surveyed clusters were smaller than 256kb.
@@ -108,17 +168,19 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
 	// increase the connection buffer size from the 1MB default to handle the specified number of concurrent streams
 	http2Options.MaxUploadBufferPerConnection = http2Options.MaxUploadBufferPerStream * int32(http2Options.MaxConcurrentStreams)
 
-	// apply settings to the server
-	if err := http2.ConfigureServer(secureServer, http2Options); err != nil {
-		return nil, fmt.Errorf("error configuring http2: %v", err)
+	if !s.DisableHTTP2 {
+		// apply settings to the server
+		if err := http2.ConfigureServer(secureServer, http2Options); err != nil {
+			return nil, fmt.Errorf("error configuring http2: %v", err)
+		}
 	}
 
 	klog.Infof("Serving securely on %s", secureServer.Addr)
 	return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
 }
 
-// RunServer listens on the given port if listener is not given,
-// then spawns a go-routine continuously serving until the stopCh is closed.
+// RunServer spawns a go-routine continuously serving until the stopCh is
+// closed.
 // It returns a stoppedCh that is closed when all non-hijacked active requests
 // have been processed.
 // This function does not block
@@ -166,57 +228,6 @@ func RunServer(
 	return stoppedCh, nil
 }
 
-type NamedTLSCert struct {
-	TLSCert tls.Certificate
-
-	// Names is a list of domain patterns: fully qualified domain names, possibly prefixed with
-	// wildcard segments.
-	Names []string
-}
-
-// GetNamedCertificateMap returns a map of *tls.Certificate by name. It's
-// suitable for use in tls.Config#NamedCertificates. Returns an error if any of the certs
-// cannot be loaded. Returns nil if len(certs) == 0
-func GetNamedCertificateMap(certs []NamedTLSCert) (map[string]*tls.Certificate, error) {
-	// register certs with implicit names first, reverse order such that earlier trump over the later
-	byName := map[string]*tls.Certificate{}
-	for i := len(certs) - 1; i >= 0; i-- {
-		if len(certs[i].Names) > 0 {
-			continue
-		}
-		cert := &certs[i].TLSCert
-
-		// read names from certificate common names and DNS names
-		if len(cert.Certificate) == 0 {
-			return nil, fmt.Errorf("empty SNI certificate, skipping")
-		}
-		x509Cert, err := x509.ParseCertificate(cert.Certificate[0])
-		if err != nil {
-			return nil, fmt.Errorf("parse error for SNI certificate: %v", err)
-		}
-		cn := x509Cert.Subject.CommonName
-		if cn == "*" || len(validation.IsDNS1123Subdomain(strings.TrimPrefix(cn, "*."))) == 0 {
-			byName[cn] = cert
-		}
-		for _, san := range x509Cert.DNSNames {
-			byName[san] = cert
-		}
-		// intentionally all IPs in the cert are ignored as SNI forbids passing IPs
-		// to select a cert. Before go 1.6 the tls happily passed IPs as SNI values.
-	}
-
-	// register certs with explicit names last, overwriting every of the implicit ones,
-	// again in reverse order.
-	for i := len(certs) - 1; i >= 0; i-- {
-		namedCert := &certs[i]
-		for _, name := range namedCert.Names {
-			byName[name] = &certs[i].TLSCert
-		}
-	}
-
-	return byName, nil
-}
-
 // tcpKeepAliveListener sets TCP keep-alive timeouts on accepted
 // connections. It's used by ListenAndServe and ListenAndServeTLS so
 // dead TCP connections (e.g. closing laptop mid-download) eventually

vendor/k8s.io/apiserver/pkg/server/storage/resource_config.go

@@ -52,6 +52,24 @@ func (o *ResourceConfig) EnableAll() {
 	}
 }
 
+// DisableMatchingVersions disables all group/versions for which the matcher function returns true. It does not modify individual resource enablement/disablement.
+func (o *ResourceConfig) DisableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for k := range o.GroupVersionConfigs {
+		if matcher(k) {
+			o.GroupVersionConfigs[k] = false
+		}
+	}
+}
+
+// EnableMatchingVersions enables all group/versions for which the matcher function returns true. It does not modify individual resource enablement/disablement.
+func (o *ResourceConfig) EnableMatchingVersions(matcher func(gv schema.GroupVersion) bool) {
+	for k := range o.GroupVersionConfigs {
+		if matcher(k) {
+			o.GroupVersionConfigs[k] = true
+		}
+	}
+}
+
 // DisableVersions disables the versions entirely.
 func (o *ResourceConfig) DisableVersions(versions ...schema.GroupVersion) {
 	for _, version := range versions {

vendor/k8s.io/apiserver/pkg/server/storage/storage_factory.go

@@ -307,8 +307,8 @@ func (s *DefaultStorageFactory) Backends() []Backend {
 			tlsConfig.Certificates = []tls.Certificate{cert}
 		}
 	}
-	if len(s.StorageConfig.Transport.CAFile) > 0 {
-		if caCert, err := ioutil.ReadFile(s.StorageConfig.Transport.CAFile); err != nil {
+	if len(s.StorageConfig.Transport.TrustedCAFile) > 0 {
+		if caCert, err := ioutil.ReadFile(s.StorageConfig.Transport.TrustedCAFile); err != nil {
 			klog.Errorf("failed to read ca file while getting backends: %s", err)
 		} else {
 			caPool := x509.NewCertPool()