vendor: revendor metrics-server, custom-metrics-apiserver

This commit is contained in:
Sergiusz Urbaniak 2020-10-28 15:52:52 +01:00
parent 752ce84723
commit 523aa52367
1010 changed files with 91458 additions and 29107 deletions

View file

@ -53,6 +53,7 @@ import (
genericapifilters "k8s.io/apiserver/pkg/endpoints/filters"
apiopenapi "k8s.io/apiserver/pkg/endpoints/openapi"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
genericregistry "k8s.io/apiserver/pkg/registry/generic"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/apiserver/pkg/server/egressselector"
@ -60,12 +61,14 @@ import (
"k8s.io/apiserver/pkg/server/healthz"
"k8s.io/apiserver/pkg/server/routes"
serverstore "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/util/feature"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
"k8s.io/client-go/informers"
restclient "k8s.io/client-go/rest"
"k8s.io/component-base/logs"
"k8s.io/klog"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
utilsnet "k8s.io/utils/net"
// install apis
_ "k8s.io/apiserver/pkg/apis/apiserver/install"
@ -272,10 +275,6 @@ type AuthenticationInfo struct {
APIAudiences authenticator.Audiences
// Authenticator determines which subject is making the request
Authenticator authenticator.Request
// SupportsBasicAuth indicates that's at least one Authenticator supports basic auth
// If this is true, a basic auth challenge is returned on authentication failure
// TODO(roberthbailey): Remove once the server no longer supports http basic auth.
SupportsBasicAuth bool
}
type AuthorizationInfo struct {
@ -606,11 +605,18 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
}
genericApiServerHookName := "generic-apiserver-start-informers"
if c.SharedInformerFactory != nil && !s.isPostStartHookRegistered(genericApiServerHookName) {
err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
c.SharedInformerFactory.Start(context.StopCh)
return nil
})
if c.SharedInformerFactory != nil {
if !s.isPostStartHookRegistered(genericApiServerHookName) {
err := s.AddPostStartHook(genericApiServerHookName, func(context PostStartHookContext) error {
c.SharedInformerFactory.Start(context.StopCh)
return nil
})
if err != nil {
return nil, err
}
}
// TODO: Once we get rid of /healthz consider changing this to post-start-hook.
err := s.addReadyzChecks(healthz.NewInformerSyncHealthz(c.SharedInformerFactory))
if err != nil {
return nil, err
}
@ -620,6 +626,7 @@ func (c completedConfig) New(name string, delegationTarget DelegationTarget) (*G
if s.isPostStartHookRegistered(priorityAndFairnessConfigConsumerHookName) {
} else if c.FlowControl != nil {
err := s.AddPostStartHook(priorityAndFairnessConfigConsumerHookName, func(context PostStartHookContext) error {
go c.FlowControl.MaintainObservations(context.StopCh)
go c.FlowControl.Run(context.StopCh)
return nil
})
@ -670,7 +677,7 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
}
handler = genericapifilters.WithImpersonation(handler, c.Authorization.Authorizer, c.Serializer)
handler = genericapifilters.WithAudit(handler, c.AuditBackend, c.AuditPolicyChecker, c.LongRunningFunc)
failedHandler := genericapifilters.Unauthorized(c.Serializer, c.Authentication.SupportsBasicAuth)
failedHandler := genericapifilters.Unauthorized(c.Serializer)
failedHandler = genericapifilters.WithFailedAuthenticationAudit(failedHandler, c.AuditBackend, c.AuditPolicyChecker)
handler = genericapifilters.WithAuthentication(handler, c.Authentication.Authenticator, failedHandler, c.Authentication.APIAudiences)
handler = genericfilters.WithCORS(handler, c.CorsAllowedOriginList, nil, nil, nil, "true")
@ -680,6 +687,8 @@ func DefaultBuildHandlerChain(apiHandler http.Handler, c *Config) http.Handler {
if c.SecureServing != nil && !c.SecureServing.DisableHTTP2 && c.GoawayChance > 0 {
handler = genericfilters.WithProbabilisticGoaway(handler, c.GoawayChance)
}
handler = genericapifilters.WithAuditAnnotations(handler, c.AuditBackend, c.AuditPolicyChecker)
handler = genericapifilters.WithWarningRecorder(handler)
handler = genericapifilters.WithCacheControl(handler)
handler = genericfilters.WithPanicRecovery(handler)
return handler
@ -710,6 +719,9 @@ func installAPI(s *GenericAPIServer, c *Config) {
if c.EnableDiscovery {
s.Handler.GoRestfulContainer.Add(s.DiscoveryGroupManager.WebService())
}
if feature.DefaultFeatureGate.Enabled(features.APIPriorityAndFairness) {
c.FlowControl.Install(s.Handler.NonGoRestfulMux)
}
}
func NewRequestInfoResolver(c *Config) *apirequest.RequestInfoFactory {
@ -735,7 +747,7 @@ func (s *SecureServingInfo) HostPort() (string, int, error) {
if err != nil {
return "", 0, fmt.Errorf("failed to get port from listener address %q: %v", addr, err)
}
port, err := strconv.Atoi(portStr)
port, err := utilsnet.ParsePort(portStr, true)
if err != nil {
return "", 0, fmt.Errorf("invalid non-numeric port %q", portStr)
}

View file

@ -21,7 +21,7 @@ import (
"net/http"
"time"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apiserver/pkg/authentication/authenticator"
"k8s.io/apiserver/pkg/authentication/user"

View file

@ -33,7 +33,7 @@ import (
corev1listers "k8s.io/client-go/listers/core/v1"
"k8s.io/client-go/tools/cache"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// ConfigMapCAController provies a CAContentProvider that can dynamically react to configmap changes

View file

@ -29,7 +29,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// FileRefreshDuration is exposed so that integration tests can crank up the reload speed.

View file

@ -26,7 +26,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// DynamicCertKeyPairContent provides a CertKeyContentProvider that can dynamically react to new file content

View file

@ -25,7 +25,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// BuildNamedCertificates returns a map of *tls.Certificate by name. It's

View file

@ -31,7 +31,7 @@ import (
"k8s.io/client-go/tools/events"
"k8s.io/client-go/util/cert"
"k8s.io/client-go/util/workqueue"
"k8s.io/klog"
"k8s.io/klog/v2"
)
const workItemKey = "key"

View file

@ -34,7 +34,7 @@ import (
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/apis/apiserver"
egressmetrics "k8s.io/apiserver/pkg/server/egressselector/metrics"
"k8s.io/klog"
"k8s.io/klog/v2"
utiltrace "k8s.io/utils/trace"
client "sigs.k8s.io/apiserver-network-proxy/konnectivity-client/pkg/client"
)
@ -199,7 +199,7 @@ func (u *udsGRPCConnector) connect() (proxier, error) {
return c, err
})
tunnel, err := client.CreateGrpcTunnel(udsName, dialOption, grpc.WithInsecure())
tunnel, err := client.CreateSingleUseGrpcTunnel(udsName, dialOption, grpc.WithInsecure())
if err != nil {
return nil, err
}

View file

@ -21,7 +21,7 @@ import (
"regexp"
"strings"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// TODO: use restful.CrossOriginResourceSharing

View file

@ -27,8 +27,9 @@ import (
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
"k8s.io/klog"
"k8s.io/klog/v2"
)
const (
@ -50,13 +51,17 @@ func handleError(w http.ResponseWriter, r *http.Request, err error) {
klog.Errorf(err.Error())
}
// requestWatermark is used to trak maximal usage of inflight requests.
// requestWatermark is used to track maximal numbers of requests in a particular phase of handling
type requestWatermark struct {
phase string
readOnlyObserver, mutatingObserver fcmetrics.TimedObserver
lock sync.Mutex
readOnlyWatermark, mutatingWatermark int
}
func (w *requestWatermark) recordMutating(mutatingVal int) {
w.mutatingObserver.Set(float64(mutatingVal))
w.lock.Lock()
defer w.lock.Unlock()
@ -66,6 +71,8 @@ func (w *requestWatermark) recordMutating(mutatingVal int) {
}
func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
w.readOnlyObserver.Set(float64(readOnlyVal))
w.lock.Lock()
defer w.lock.Unlock()
@ -74,9 +81,14 @@ func (w *requestWatermark) recordReadOnly(readOnlyVal int) {
}
}
var watermark = &requestWatermark{}
// watermark tracks requests being executed (not waiting in a queue)
var watermark = &requestWatermark{
phase: metrics.ExecutingPhase,
readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{metrics.ReadOnlyKind}).RequestsExecuting,
mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{metrics.MutatingKind}).RequestsExecuting,
}
func startRecordingUsage() {
func startRecordingUsage(watermark *requestWatermark) {
go func() {
wait.Forever(func() {
watermark.lock.Lock()
@ -86,7 +98,7 @@ func startRecordingUsage() {
watermark.mutatingWatermark = 0
watermark.lock.Unlock()
metrics.UpdateInflightRequestMetrics(readOnlyWatermark, mutatingWatermark)
metrics.UpdateInflightRequestMetrics(watermark.phase, readOnlyWatermark, mutatingWatermark)
}, inflightUsageMetricUpdatePeriod)
}()
}
@ -100,7 +112,7 @@ func WithMaxInFlightLimit(
mutatingLimit int,
longRunningRequestCheck apirequest.LongRunningRequestCheck,
) http.Handler {
startOnce.Do(startRecordingUsage)
startOnce.Do(func() { startRecordingUsage(watermark) })
if nonMutatingLimit == 0 && mutatingLimit == 0 {
return handler
}
@ -108,9 +120,11 @@ func WithMaxInFlightLimit(
var mutatingChan chan bool
if nonMutatingLimit != 0 {
nonMutatingChan = make(chan bool, nonMutatingLimit)
watermark.readOnlyObserver.SetX1(float64(nonMutatingLimit))
}
if mutatingLimit != 0 {
mutatingChan = make(chan bool, mutatingLimit)
watermark.mutatingObserver.SetX1(float64(mutatingLimit))
}
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -141,21 +155,22 @@ func WithMaxInFlightLimit(
select {
case c <- true:
var mutatingLen, readOnlyLen int
// We note the concurrency level both while the
// request is being served and after it is done being
// served, because both states contribute to the
// sampled stats on concurrency.
if isMutatingRequest {
mutatingLen = len(mutatingChan)
watermark.recordMutating(len(c))
} else {
readOnlyLen = len(nonMutatingChan)
watermark.recordReadOnly(len(c))
}
defer func() {
<-c
if isMutatingRequest {
watermark.recordMutating(mutatingLen)
watermark.recordMutating(len(c))
} else {
watermark.recordReadOnly(readOnlyLen)
watermark.recordReadOnly(len(c))
}
}()
handler.ServeHTTP(w, r)

View file

@ -24,9 +24,11 @@ import (
fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
apitypes "k8s.io/apimachinery/pkg/types"
epmetrics "k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
utilflowcontrol "k8s.io/apiserver/pkg/util/flowcontrol"
"k8s.io/klog"
fcmetrics "k8s.io/apiserver/pkg/util/flowcontrol/metrics"
"k8s.io/klog/v2"
)
type priorityAndFairnessKeyType int
@ -53,7 +55,15 @@ func GetClassification(ctx context.Context) *PriorityAndFairnessClassification {
return ctx.Value(priorityAndFairnessKey).(*PriorityAndFairnessClassification)
}
var atomicMutatingLen, atomicNonMutatingLen int32
// waitingMark tracks requests waiting rather than being executed
var waitingMark = &requestWatermark{
phase: epmetrics.WaitingPhase,
readOnlyObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.ReadOnlyKind}).RequestsWaiting,
mutatingObserver: fcmetrics.ReadWriteConcurrencyObserverPairGenerator.Generate(1, 1, []string{epmetrics.MutatingKind}).RequestsWaiting,
}
var atomicMutatingExecuting, atomicReadOnlyExecuting int32
var atomicMutatingWaiting, atomicReadOnlyWaiting int32
// WithPriorityAndFairness limits the number of in-flight
// requests in a fine-grained way.
@ -66,7 +76,10 @@ func WithPriorityAndFairness(
klog.Warningf("priority and fairness support not found, skipping")
return handler
}
startOnce.Do(startRecordingUsage)
startOnce.Do(func() {
startRecordingUsage(watermark)
startRecordingUsage(waitingMark)
})
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
requestInfo, ok := apirequest.RequestInfoFrom(ctx)
@ -98,22 +111,23 @@ func WithPriorityAndFairness(
var served bool
isMutatingRequest := !nonMutatingRequestVerbs.Has(requestInfo.Verb)
execute := func() {
var mutatingLen, readOnlyLen int
noteExecutingDelta := func(delta int32) {
if isMutatingRequest {
mutatingLen = int(atomic.AddInt32(&atomicMutatingLen, 1))
watermark.recordMutating(int(atomic.AddInt32(&atomicMutatingExecuting, delta)))
} else {
readOnlyLen = int(atomic.AddInt32(&atomicNonMutatingLen, 1))
watermark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyExecuting, delta)))
}
defer func() {
if isMutatingRequest {
atomic.AddInt32(&atomicMutatingLen, -11)
watermark.recordMutating(mutatingLen)
} else {
atomic.AddInt32(&atomicNonMutatingLen, -1)
watermark.recordReadOnly(readOnlyLen)
}
}()
}
noteWaitingDelta := func(delta int32) {
if isMutatingRequest {
waitingMark.recordMutating(int(atomic.AddInt32(&atomicMutatingWaiting, delta)))
} else {
waitingMark.recordReadOnly(int(atomic.AddInt32(&atomicReadOnlyWaiting, delta)))
}
}
execute := func() {
noteExecutingDelta(1)
defer noteExecutingDelta(-1)
served = true
innerCtx := context.WithValue(ctx, priorityAndFairnessKey, classification)
innerReq := r.Clone(innerCtx)
@ -122,10 +136,16 @@ func WithPriorityAndFairness(
handler.ServeHTTP(w, innerReq)
}
digest := utilflowcontrol.RequestDigest{requestInfo, user}
fcIfc.Handle(ctx, digest, note, execute)
fcIfc.Handle(ctx, digest, note, func(inQueue bool) {
if inQueue {
noteWaitingDelta(1)
} else {
noteWaitingDelta(-1)
}
}, execute)
if !served {
epmetrics.RecordRequestTermination(r, requestInfo, epmetrics.APIServerComponent, http.StatusTooManyRequests)
tooManyRequests(r, w)
return
}
})

View file

@ -19,7 +19,7 @@ package filters
import (
"net/http"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server/httplog"

View file

@ -45,7 +45,7 @@ import (
"k8s.io/apiserver/pkg/server/routes"
utilopenapi "k8s.io/apiserver/pkg/util/openapi"
restclient "k8s.io/client-go/rest"
"k8s.io/klog"
"k8s.io/klog/v2"
openapibuilder "k8s.io/kube-openapi/pkg/builder"
openapicommon "k8s.io/kube-openapi/pkg/common"
"k8s.io/kube-openapi/pkg/handler"
@ -330,7 +330,7 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
}()
// close socket after delayed stopCh
err := s.NonBlockingRun(delayedStopCh)
stoppedCh, err := s.NonBlockingRun(delayedStopCh)
if err != nil {
return err
}
@ -345,6 +345,8 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
// wait for the delayed stopCh before closing the handler chain (it rejects everything after Wait has been called).
<-delayedStopCh
// wait for stoppedCh that is closed when the graceful termination (server.Shutdown) is finished.
<-stoppedCh
// Wait for all requests to finish, which are bounded by the RequestTimeout variable.
s.HandlerChainWaitGroup.Wait()
@ -354,7 +356,8 @@ func (s preparedGenericAPIServer) Run(stopCh <-chan struct{}) error {
// NonBlockingRun spawns the secure http server. An error is
// returned if the secure port cannot be listened on.
func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
// The returned channel is closed when the (asynchronous) termination is finished.
func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) (<-chan struct{}, error) {
// Use an stop channel to allow graceful shutdown without dropping audit events
// after http server shutdown.
auditStopCh := make(chan struct{})
@ -363,7 +366,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
// before http server start serving. Otherwise the Backend.ProcessEvents call might block.
if s.AuditBackend != nil {
if err := s.AuditBackend.Run(auditStopCh); err != nil {
return fmt.Errorf("failed to run the audit backend: %v", err)
return nil, fmt.Errorf("failed to run the audit backend: %v", err)
}
}
@ -376,7 +379,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
if err != nil {
close(internalStopCh)
close(auditStopCh)
return err
return nil, err
}
}
@ -399,7 +402,7 @@ func (s preparedGenericAPIServer) NonBlockingRun(stopCh <-chan struct{}) error {
klog.Errorf("Unable to send systemd daemon successful start message: %v\n", err)
}
return nil
return stoppedCh, nil
}
// installAPIResources is a private method for installing the REST storage backing each api groupversionresource

View file

@ -25,7 +25,7 @@ import (
"strings"
"github.com/emicklei/go-restful"
"k8s.io/klog"
"k8s.io/klog/v2"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/runtime"

View file

@ -20,6 +20,7 @@ import (
"bytes"
"fmt"
"net/http"
"reflect"
"strings"
"sync"
"sync/atomic"
@ -29,7 +30,7 @@ import (
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/httplog"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// HealthChecker is a named healthz checker.
@ -81,6 +82,43 @@ func (l *log) Check(_ *http.Request) error {
return fmt.Errorf("logging blocked")
}
type cacheSyncWaiter interface {
WaitForCacheSync(stopCh <-chan struct{}) map[reflect.Type]bool
}
type informerSync struct {
cacheSyncWaiter cacheSyncWaiter
}
var _ HealthChecker = &informerSync{}
// NewInformerSyncHealthz returns a new HealthChecker that will pass only if all informers in the given cacheSyncWaiter sync.
func NewInformerSyncHealthz(cacheSyncWaiter cacheSyncWaiter) HealthChecker {
return &informerSync{
cacheSyncWaiter: cacheSyncWaiter,
}
}
func (i *informerSync) Name() string {
return "informer-sync"
}
func (i *informerSync) Check(_ *http.Request) error {
stopCh := make(chan struct{})
// Close stopCh to force checking if informers are synced now.
close(stopCh)
informersByStarted := make(map[bool][]string)
for informerType, started := range i.cacheSyncWaiter.WaitForCacheSync(stopCh) {
informersByStarted[started] = append(informersByStarted[started], informerType.String())
}
if notStarted := informersByStarted[false]; len(notStarted) > 0 {
return fmt.Errorf("%d informers not started yet: %v", len(notStarted), notStarted)
}
return nil
}
// NamedCheck returns a healthz checker for the given name and function.
func NamedCheck(name string, check func(r *http.Request) error) HealthChecker {
return &healthzCheck{name, check}
@ -131,6 +169,8 @@ func InstallPathHandler(mux mux, path string, checks ...HealthChecker) {
/* subresource = */ path,
/* scope = */ "",
/* component = */ "",
/* deprecated */ false,
/* removedRelease */ "",
handleRootHealthz(checks...)))
for _, check := range checks {
mux.Handle(fmt.Sprintf("%s/%v", path, check.Name()), adaptCheckToHandler(check.Check))
@ -170,35 +210,38 @@ func getExcludedChecks(r *http.Request) sets.String {
// handleRootHealthz returns an http.HandlerFunc that serves the provided checks.
func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
failed := false
excluded := getExcludedChecks(r)
var verboseOut bytes.Buffer
// failedVerboseLogOutput is for output to the log. It indicates detailed failed output information for the log.
var failedVerboseLogOutput bytes.Buffer
var failedChecks []string
var individualCheckOutput bytes.Buffer
for _, check := range checks {
// no-op the check if we've specified we want to exclude the check
if excluded.Has(check.Name()) {
excluded.Delete(check.Name())
fmt.Fprintf(&verboseOut, "[+]%v excluded: ok\n", check.Name())
fmt.Fprintf(&individualCheckOutput, "[+]%s excluded: ok\n", check.Name())
continue
}
if err := check.Check(r); err != nil {
// don't include the error since this endpoint is public. If someone wants more detail
// they should have explicit permission to the detailed checks.
klog.V(4).Infof("healthz check %v failed: %v", check.Name(), err)
fmt.Fprintf(&verboseOut, "[-]%v failed: reason withheld\n", check.Name())
failed = true
fmt.Fprintf(&individualCheckOutput, "[-]%s failed: reason withheld\n", check.Name())
// but we do want detailed information for our log
fmt.Fprintf(&failedVerboseLogOutput, "[-]%s failed: %v\n", check.Name(), err)
failedChecks = append(failedChecks, check.Name())
} else {
fmt.Fprintf(&verboseOut, "[+]%v ok\n", check.Name())
fmt.Fprintf(&individualCheckOutput, "[+]%s ok\n", check.Name())
}
}
if excluded.Len() > 0 {
fmt.Fprintf(&verboseOut, "warn: some health checks cannot be excluded: no matches for %v\n", formatQuoted(excluded.List()...))
klog.Warningf("cannot exclude some health checks, no health checks are installed matching %v",
fmt.Fprintf(&individualCheckOutput, "warn: some health checks cannot be excluded: no matches for %s\n", formatQuoted(excluded.List()...))
klog.Warningf("cannot exclude some health checks, no health checks are installed matching %s",
formatQuoted(excluded.List()...))
}
// always be verbose on failure
if failed {
klog.V(2).Infof("%vhealthz check failed", verboseOut.String())
http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%vhealthz check failed", verboseOut.String()), http.StatusInternalServerError)
if len(failedChecks) > 0 {
klog.V(2).Infof("healthz check failed: %s\n%v", strings.Join(failedChecks, ","), failedVerboseLogOutput.String())
http.Error(httplog.Unlogged(r, w), fmt.Sprintf("%shealthz check failed", individualCheckOutput.String()), http.StatusInternalServerError)
return
}
@ -209,7 +252,7 @@ func handleRootHealthz(checks ...HealthChecker) http.HandlerFunc {
return
}
verboseOut.WriteTo(w)
individualCheckOutput.WriteTo(w)
fmt.Fprint(w, "healthz check passed\n")
})
}

View file

@ -26,7 +26,7 @@ import (
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/server/healthz"
restclient "k8s.io/client-go/rest"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// PostStartHookFunc is a function that is called after the server has started.

View file

@ -25,7 +25,7 @@ import (
"runtime"
"time"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// StacktracePred returns true if a stacktrace should be logged for this status.
@ -85,7 +85,9 @@ func WithLogging(handler http.Handler, pred StacktracePred) http.Handler {
rl := newLogged(req, w).StacktraceWhen(pred)
req = req.WithContext(context.WithValue(ctx, respLoggerContextKey, rl))
defer rl.Log()
if klog.V(3).Enabled() {
defer func() { klog.InfoS("HTTP", rl.LogArgs()...) }()
}
handler.ServeHTTP(rl, req)
})
}
@ -153,24 +155,34 @@ func (rl *respLogger) Addf(format string, data ...interface{}) {
rl.addedInfo += "\n" + fmt.Sprintf(format, data...)
}
// Log is intended to be called once at the end of your request handler, via defer
func (rl *respLogger) Log() {
func (rl *respLogger) LogArgs() []interface{} {
latency := time.Since(rl.startTime)
if klog.V(3) {
if !rl.hijacked {
klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v resp=%v UserAgent=%q srcIP=%q: %v%v",
rl.req.Method, rl.req.RequestURI,
latency, rl.status,
rl.req.UserAgent(), rl.req.RemoteAddr,
rl.statusStack, rl.addedInfo,
))
} else {
klog.InfoDepth(1, fmt.Sprintf("verb=%q URI=%q latency=%v UserAgent=%q srcIP=%q: hijacked",
rl.req.Method, rl.req.RequestURI,
latency, rl.req.UserAgent(), rl.req.RemoteAddr,
))
if rl.hijacked {
return []interface{}{
"verb", rl.req.Method,
"URI", rl.req.RequestURI,
"latency", latency,
"userAgent", rl.req.UserAgent(),
"srcIP", rl.req.RemoteAddr,
"hijacked", true,
}
}
args := []interface{}{
"verb", rl.req.Method,
"URI", rl.req.RequestURI,
"latency", latency,
"userAgent", rl.req.UserAgent(),
"srcIP", rl.req.RemoteAddr,
"resp", rl.status,
}
if len(rl.statusStack) > 0 {
args = append(args, "statusStack", rl.statusStack)
}
if len(rl.addedInfo) > 0 {
args = append(args, "addedInfo", rl.addedInfo)
}
return args
}
// Header implements http.ResponseWriter.
@ -194,7 +206,7 @@ func (rl *respLogger) Write(b []byte) (int, error) {
func (rl *respLogger) Flush() {
if flusher, ok := rl.w.(http.Flusher); ok {
flusher.Flush()
} else if klog.V(2) {
} else if klog.V(2).Enabled() {
klog.InfoDepth(1, fmt.Sprintf("Unable to convert %+v into http.Flusher", rl.w))
}
}

View file

@ -25,7 +25,7 @@ import (
"sync"
"sync/atomic"
"k8s.io/klog"
"k8s.io/klog/v2"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apimachinery/pkg/util/sets"

View file

@ -25,9 +25,8 @@ import (
"github.com/spf13/pflag"
"gopkg.in/natefinch/lumberjack.v2"
"k8s.io/klog"
"k8s.io/klog/v2"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
utilnet "k8s.io/apimachinery/pkg/util/net"
auditinternal "k8s.io/apiserver/pkg/apis/audit"
@ -36,20 +35,12 @@ import (
auditv1beta1 "k8s.io/apiserver/pkg/apis/audit/v1beta1"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/audit/policy"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/server"
"k8s.io/apiserver/pkg/server/egressselector"
utilfeature "k8s.io/apiserver/pkg/util/feature"
pluginbuffered "k8s.io/apiserver/plugin/pkg/audit/buffered"
plugindynamic "k8s.io/apiserver/plugin/pkg/audit/dynamic"
pluginenforced "k8s.io/apiserver/plugin/pkg/audit/dynamic/enforced"
pluginlog "k8s.io/apiserver/plugin/pkg/audit/log"
plugintruncate "k8s.io/apiserver/plugin/pkg/audit/truncate"
pluginwebhook "k8s.io/apiserver/plugin/pkg/audit/webhook"
"k8s.io/client-go/informers"
"k8s.io/client-go/kubernetes"
v1core "k8s.io/client-go/kubernetes/typed/core/v1"
restclient "k8s.io/client-go/rest"
)
const (
@ -80,7 +71,6 @@ type AuditOptions struct {
// Plugin options
LogOptions AuditLogOptions
WebhookOptions AuditWebhookOptions
DynamicOptions AuditDynamicOptions
}
const (
@ -180,10 +170,6 @@ func NewAuditOptions() *AuditOptions {
TruncateOptions: NewAuditTruncateOptions(),
GroupVersionString: "audit.k8s.io/v1",
},
DynamicOptions: AuditDynamicOptions{
Enabled: false,
BatchConfig: plugindynamic.NewDefaultWebhookBatchConfig(),
},
}
}
@ -206,7 +192,6 @@ func (o *AuditOptions) Validate() []error {
var allErrors []error
allErrors = append(allErrors, o.LogOptions.Validate()...)
allErrors = append(allErrors, o.WebhookOptions.Validate()...)
allErrors = append(allErrors, o.DynamicOptions.Validate()...)
return allErrors
}
@ -286,15 +271,10 @@ func (o *AuditOptions) AddFlags(fs *pflag.FlagSet) {
o.WebhookOptions.AddFlags(fs)
o.WebhookOptions.BatchOptions.AddFlags(pluginwebhook.PluginName, fs)
o.WebhookOptions.TruncateOptions.AddFlags(pluginwebhook.PluginName, fs)
o.DynamicOptions.AddFlags(fs)
}
func (o *AuditOptions) ApplyTo(
c *server.Config,
kubeClientConfig *restclient.Config,
informers informers.SharedInformerFactory,
processInfo *ProcessInfo,
webhookOptions *WebhookOptions,
) error {
if o == nil {
return nil
@ -347,23 +327,7 @@ func (o *AuditOptions) ApplyTo(
// 4. Apply dynamic options.
var dynamicBackend audit.Backend
if o.DynamicOptions.enabled() {
// if dynamic is enabled the webhook and log backends need to be wrapped in an enforced backend with the static policy
if webhookBackend != nil {
webhookBackend = pluginenforced.NewBackend(webhookBackend, checker)
}
if logBackend != nil {
logBackend = pluginenforced.NewBackend(logBackend, checker)
}
// build dynamic backend
dynamicBackend, checker, err = o.DynamicOptions.newBackend(c.ExternalAddress, kubeClientConfig, informers, processInfo, webhookOptions)
if err != nil {
return err
}
// union dynamic and webhook backends so that truncate options can be applied to both
dynamicBackend = appendBackend(webhookBackend, dynamicBackend)
dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(dynamicBackend, groupVersion)
} else if webhookBackend != nil {
if webhookBackend != nil {
// if only webhook is enabled wrap it in the truncate options
dynamicBackend = o.WebhookOptions.TruncateOptions.wrapBackend(webhookBackend, groupVersion)
}
@ -610,66 +574,6 @@ func (o *AuditWebhookOptions) newUntruncatedBackend(customDial utilnet.DialFunc)
return webhook, nil
}
func (o *AuditDynamicOptions) AddFlags(fs *pflag.FlagSet) {
fs.BoolVar(&o.Enabled, "audit-dynamic-configuration", o.Enabled,
"Enables dynamic audit configuration. This feature also requires the DynamicAuditing feature flag")
}
func (o *AuditDynamicOptions) enabled() bool {
return o.Enabled && utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing)
}
func (o *AuditDynamicOptions) Validate() []error {
var allErrors []error
if o.Enabled && !utilfeature.DefaultFeatureGate.Enabled(features.DynamicAuditing) {
allErrors = append(allErrors, fmt.Errorf("--audit-dynamic-configuration set, but DynamicAuditing feature gate is not enabled"))
}
return allErrors
}
func (o *AuditDynamicOptions) newBackend(
hostname string,
kubeClientConfig *restclient.Config,
informers informers.SharedInformerFactory,
processInfo *ProcessInfo,
webhookOptions *WebhookOptions,
) (audit.Backend, policy.Checker, error) {
if err := validateProcessInfo(processInfo); err != nil {
return nil, nil, err
}
clientset, err := kubernetes.NewForConfig(kubeClientConfig)
if err != nil {
return nil, nil, err
}
if webhookOptions == nil {
webhookOptions = NewWebhookOptions()
}
checker := policy.NewDynamicChecker()
informer := informers.Auditregistration().V1alpha1().AuditSinks()
eventSink := &v1core.EventSinkImpl{Interface: clientset.CoreV1().Events(processInfo.Namespace)}
dc := &plugindynamic.Config{
Informer: informer,
BufferedConfig: o.BatchConfig,
EventConfig: plugindynamic.EventConfig{
Sink: eventSink,
Source: corev1.EventSource{
Component: processInfo.Name,
Host: hostname,
},
},
WebhookConfig: plugindynamic.WebhookConfig{
AuthInfoResolverWrapper: webhookOptions.AuthInfoResolverWrapper,
ServiceResolver: webhookOptions.ServiceResolver,
},
}
backend, err := plugindynamic.NewBackend(dc)
if err != nil {
return nil, nil, fmt.Errorf("could not create dynamic audit backend: %v", err)
}
return backend, checker, nil
}
// defaultWebhookBatchConfig returns the default BatchConfig used by the Webhook backend.
func defaultWebhookBatchConfig() pluginbuffered.BatchConfig {
return pluginbuffered.BatchConfig{

View file

@ -17,8 +17,6 @@ limitations under the License.
package options
import (
"context"
"encoding/json"
"fmt"
"strings"
"time"
@ -27,7 +25,6 @@ import (
"github.com/spf13/pflag"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apiserver/pkg/authentication/authenticatorfactory"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
@ -35,7 +32,7 @@ import (
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
"k8s.io/client-go/tools/clientcmd"
"k8s.io/klog"
"k8s.io/klog/v2"
openapicommon "k8s.io/kube-openapi/pkg/common"
)
@ -319,7 +316,6 @@ func (s *DelegatingAuthenticationOptions) ApplyTo(authenticationInfo *server.Aut
if openAPIConfig != nil {
openAPIConfig.SecurityDefinitions = securityDefinitions
}
authenticationInfo.SupportsBasicAuth = false
return nil
}
@ -331,66 +327,28 @@ const (
// by --requestheader-username-headers. This is created in the cluster by the kube-apiserver.
// "WARNING: generally do not depend on authorization being already done for incoming requests.")
authenticationConfigMapName = "extension-apiserver-authentication"
authenticationRoleName = "extension-apiserver-authentication-reader"
)
func (s *DelegatingAuthenticationOptions) createRequestHeaderConfig(client kubernetes.Interface) (*authenticatorfactory.RequestHeaderConfig, error) {
requestHeaderCAProvider, err := dynamiccertificates.NewDynamicCAFromConfigMapController("client-ca", authenticationConfigMapNamespace, authenticationConfigMapName, "requestheader-client-ca-file", client)
dynamicRequestHeaderProvider, err := newDynamicRequestHeaderController(client)
if err != nil {
return nil, fmt.Errorf("unable to create request header authentication config: %v", err)
}
authConfigMap, err := client.CoreV1().ConfigMaps(authenticationConfigMapNamespace).Get(context.TODO(), authenticationConfigMapName, metav1.GetOptions{})
switch {
case errors.IsNotFound(err):
// ignore, authConfigMap is nil now
return nil, nil
case errors.IsForbidden(err):
klog.Warningf("Unable to get configmap/%s in %s. Usually fixed by "+
"'kubectl create rolebinding -n %s ROLEBINDING_NAME --role=%s --serviceaccount=YOUR_NS:YOUR_SA'",
authenticationConfigMapName, authenticationConfigMapNamespace, authenticationConfigMapNamespace, authenticationRoleName)
return nil, err
case err != nil:
return nil, err
}
usernameHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-username-headers"])
if err != nil {
return nil, err
}
groupHeaders, err := deserializeStrings(authConfigMap.Data["requestheader-group-headers"])
if err != nil {
return nil, err
}
extraHeaderPrefixes, err := deserializeStrings(authConfigMap.Data["requestheader-extra-headers-prefix"])
if err != nil {
return nil, err
}
allowedNames, err := deserializeStrings(authConfigMap.Data["requestheader-allowed-names"])
if err != nil {
// look up authentication configuration in the cluster and in case of an err defer to authentication-tolerate-lookup-failure flag
if err := dynamicRequestHeaderProvider.RunOnce(); err != nil {
return nil, err
}
return &authenticatorfactory.RequestHeaderConfig{
CAContentProvider: requestHeaderCAProvider,
UsernameHeaders: headerrequest.StaticStringSlice(usernameHeaders),
GroupHeaders: headerrequest.StaticStringSlice(groupHeaders),
ExtraHeaderPrefixes: headerrequest.StaticStringSlice(extraHeaderPrefixes),
AllowedClientNames: headerrequest.StaticStringSlice(allowedNames),
CAContentProvider: dynamicRequestHeaderProvider,
UsernameHeaders: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.UsernameHeaders)),
GroupHeaders: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.GroupHeaders)),
ExtraHeaderPrefixes: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.ExtraHeaderPrefixes)),
AllowedClientNames: headerrequest.StringSliceProvider(headerrequest.StringSliceProviderFunc(dynamicRequestHeaderProvider.AllowedClientNames)),
}, nil
}
func deserializeStrings(in string) ([]string, error) {
if len(in) == 0 {
return nil, nil
}
var ret []string
if err := json.Unmarshal([]byte(in), &ret); err != nil {
return nil, err
}
return ret, nil
}
// getClient returns a Kubernetes clientset. If s.RemoteKubeConfigFileOptional is true, nil will be returned
// if no kubeconfig is specified by the user and the in-cluster config is not found.
func (s *DelegatingAuthenticationOptions) getClient() (kubernetes.Interface, error) {

View file

@ -0,0 +1,79 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"k8s.io/apimachinery/pkg/util/errors"
"k8s.io/apiserver/pkg/authentication/request/headerrequest"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
"k8s.io/client-go/kubernetes"
)
var _ dynamiccertificates.ControllerRunner = &DynamicRequestHeaderController{}
var _ dynamiccertificates.Notifier = &DynamicRequestHeaderController{}
var _ dynamiccertificates.CAContentProvider = &DynamicRequestHeaderController{}
var _ headerrequest.RequestHeaderAuthRequestProvider = &DynamicRequestHeaderController{}
// DynamicRequestHeaderController combines DynamicCAFromConfigMapController and RequestHeaderAuthRequestController
// into one controller for dynamically filling RequestHeaderConfig struct
type DynamicRequestHeaderController struct {
*dynamiccertificates.ConfigMapCAController
*headerrequest.RequestHeaderAuthRequestController
}
// newDynamicRequestHeaderController creates a new controller that implements DynamicRequestHeaderController
func newDynamicRequestHeaderController(client kubernetes.Interface) (*DynamicRequestHeaderController, error) {
requestHeaderCAController, err := dynamiccertificates.NewDynamicCAFromConfigMapController(
"client-ca",
authenticationConfigMapNamespace,
authenticationConfigMapName,
"requestheader-client-ca-file",
client)
if err != nil {
return nil, fmt.Errorf("unable to create DynamicCAFromConfigMap controller: %v", err)
}
requestHeaderAuthRequestController := headerrequest.NewRequestHeaderAuthRequestController(
authenticationConfigMapName,
authenticationConfigMapNamespace,
client,
"requestheader-username-headers",
"requestheader-group-headers",
"requestheader-extra-headers-prefix",
"requestheader-allowed-names",
)
return &DynamicRequestHeaderController{
ConfigMapCAController: requestHeaderCAController,
RequestHeaderAuthRequestController: requestHeaderAuthRequestController,
}, nil
}
func (c *DynamicRequestHeaderController) RunOnce() error {
errs := []error{}
errs = append(errs, c.ConfigMapCAController.RunOnce())
errs = append(errs, c.RequestHeaderAuthRequestController.RunOnce())
return errors.NewAggregate(errs)
}
func (c *DynamicRequestHeaderController) Run(workers int, stopCh <-chan struct{}) {
go c.ConfigMapCAController.Run(workers, stopCh)
go c.RequestHeaderAuthRequestController.Run(workers, stopCh)
<-stopCh
}

View file

@ -21,7 +21,7 @@ import (
"time"
"github.com/spf13/pflag"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/authorization/authorizerfactory"

View file

@ -43,7 +43,7 @@ type DeprecatedInsecureServingOptions struct {
// ListenFunc can be overridden to create a custom listener, e.g. for mocking in tests.
// It defaults to options.CreateListener.
ListenFunc func(network, addr string) (net.Listener, int, error)
ListenFunc func(network, addr string, config net.ListenConfig) (net.Listener, int, error)
}
// Validate ensures that the insecure port values within the range of the port.
@ -113,7 +113,7 @@ func (s *DeprecatedInsecureServingOptions) ApplyTo(c **server.DeprecatedInsecure
listen = s.ListenFunc
}
addr := net.JoinHostPort(s.BindAddress.String(), fmt.Sprintf("%d", s.BindPort))
s.Listener, s.BindPort, err = listen(s.BindNetwork, addr)
s.Listener, s.BindPort, err = listen(s.BindNetwork, addr, net.ListenConfig{})
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}

View file

@ -161,15 +161,14 @@ func GetTransformerOverrides(filepath string) (map[schema.GroupResource]value.Tr
}
defer f.Close()
result, err := ParseEncryptionConfiguration(f)
result, err := parseEncryptionConfiguration(f)
if err != nil {
return nil, fmt.Errorf("error while parsing encryption provider configuration file %q: %v", filepath, err)
}
return result, nil
}
// ParseEncryptionConfiguration parses configuration data and returns the transformer overrides
func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
func parseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.Transformer, error) {
configFileContents, err := ioutil.ReadAll(f)
if err != nil {
return nil, fmt.Errorf("could not read contents: %v", err)
@ -184,7 +183,7 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T
// For each entry in the configuration
for _, resourceConfig := range config.Resources {
transformers, err := GetPrefixTransformers(&resourceConfig)
transformers, err := prefixTransformers(&resourceConfig)
if err != nil {
return nil, err
}
@ -205,7 +204,6 @@ func ParseEncryptionConfiguration(f io.Reader) (map[schema.GroupResource]value.T
}
// loadConfig decodes data as a EncryptionConfiguration object.
func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
scheme := runtime.NewScheme()
codecs := serializer.NewCodecFactory(scheme)
@ -227,8 +225,7 @@ func loadConfig(data []byte) (*apiserverconfig.EncryptionConfiguration, error) {
// The factory to create kms service. This is to make writing test easier.
var envelopeServiceFactory = envelope.NewGRPCService
// GetPrefixTransformers constructs and returns the appropriate prefix transformers for the passed resource using its configuration.
func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
func prefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]value.PrefixTransformer, error) {
var result []value.PrefixTransformer
for _, provider := range config.Providers {
var (
@ -238,18 +235,18 @@ func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]val
switch {
case provider.AESGCM != nil:
transformer, err = GetAESPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
transformer, err = aesPrefixTransformer(provider.AESGCM, aestransformer.NewGCMTransformer, aesGCMTransformerPrefixV1)
case provider.AESCBC != nil:
transformer, err = GetAESPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
transformer, err = aesPrefixTransformer(provider.AESCBC, aestransformer.NewCBCTransformer, aesCBCTransformerPrefixV1)
case provider.Secretbox != nil:
transformer, err = GetSecretboxPrefixTransformer(provider.Secretbox)
transformer, err = secretboxPrefixTransformer(provider.Secretbox)
case provider.KMS != nil:
envelopeService, err := envelopeServiceFactory(provider.KMS.Endpoint, provider.KMS.Timeout.Duration)
if err != nil {
return nil, fmt.Errorf("could not configure KMS plugin %q, error: %v", provider.KMS.Name, err)
}
transformer, err = getEnvelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
transformer, err = envelopePrefixTransformer(provider.KMS, envelopeService, kmsTransformerPrefixV1)
case provider.Identity != nil:
transformer = value.PrefixTransformer{
Transformer: identity.NewEncryptCheckTransformer(),
@ -267,12 +264,9 @@ func GetPrefixTransformers(config *apiserverconfig.ResourceConfiguration) ([]val
return result, nil
}
// BlockTransformerFunc takes an AES cipher block and returns a value transformer.
type BlockTransformerFunc func(cipher.Block) value.Transformer
type blockTransformerFunc func(cipher.Block) value.Transformer
// GetAESPrefixTransformer returns a prefix transformer from the provided configuration.
// Returns an AES transformer based on the provided prefix and block transformer.
func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
func aesPrefixTransformer(config *apiserverconfig.AESConfiguration, fn blockTransformerFunc, prefix string) (value.PrefixTransformer, error) {
var result value.PrefixTransformer
if len(config.Keys) == 0 {
@ -319,8 +313,7 @@ func GetAESPrefixTransformer(config *apiserverconfig.AESConfiguration, fn BlockT
return result, nil
}
// GetSecretboxPrefixTransformer returns a prefix transformer from the provided configuration
func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
func secretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguration) (value.PrefixTransformer, error) {
var result value.PrefixTransformer
if len(config.Keys) == 0 {
@ -370,9 +363,7 @@ func GetSecretboxPrefixTransformer(config *apiserverconfig.SecretboxConfiguratio
return result, nil
}
// getEnvelopePrefixTransformer returns a prefix transformer from the provided config.
// envelopeService is used as the root of trust.
func getEnvelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
func envelopePrefixTransformer(config *apiserverconfig.KMSConfiguration, envelopeService envelope.Service, prefix string) (value.PrefixTransformer, error) {
envelopeTransformer, err := envelope.NewEnvelopeTransformer(envelopeService, int(*config.CacheSize), aestransformer.NewCBCTransformer)
if err != nil {
return value.PrefixTransformer{}, err

View file

@ -35,6 +35,7 @@ import (
serverstorage "k8s.io/apiserver/pkg/server/storage"
"k8s.io/apiserver/pkg/storage/storagebackend"
storagefactory "k8s.io/apiserver/pkg/storage/storagebackend/factory"
"k8s.io/klog/v2"
)
type EtcdOptions struct {
@ -176,6 +177,9 @@ func (s *EtcdOptions) AddFlags(fs *pflag.FlagSet) {
fs.DurationVar(&s.StorageConfig.CountMetricPollPeriod, "etcd-count-metric-poll-period", s.StorageConfig.CountMetricPollPeriod, ""+
"Frequency of polling etcd for number of resources per type. 0 disables the metric collection.")
fs.DurationVar(&s.StorageConfig.DBMetricPollInterval, "etcd-db-metric-poll-interval", s.StorageConfig.DBMetricPollInterval,
"The interval of requests to poll etcd and update metric. 0 disables the metric collection")
}
func (s *EtcdOptions) ApplyTo(c *server.Config) error {
@ -235,12 +239,15 @@ func (f *SimpleRestOptionsFactory) GetRESTOptions(resource schema.GroupResource)
if err != nil {
return generic.RESTOptions{}, err
}
cacheSize, ok := sizes[resource]
if !ok {
cacheSize = f.Options.DefaultWatchCacheSize
size, ok := sizes[resource]
if ok && size > 0 {
klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
}
if ok && size <= 0 {
ret.Decorator = generic.UndecoratedStorage
} else {
ret.Decorator = genericregistry.StorageWithCacher()
}
// depending on cache size this might return an undecorated storage
ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
}
return ret, nil
}
@ -269,12 +276,15 @@ func (f *StorageFactoryRestOptionsFactory) GetRESTOptions(resource schema.GroupR
if err != nil {
return generic.RESTOptions{}, err
}
cacheSize, ok := sizes[resource]
if !ok {
cacheSize = f.Options.DefaultWatchCacheSize
size, ok := sizes[resource]
if ok && size > 0 {
klog.Warningf("Dropping watch-cache-size for %v - watchCache size is now dynamic", resource)
}
if ok && size <= 0 {
ret.Decorator = generic.UndecoratedStorage
} else {
ret.Decorator = genericregistry.StorageWithCacher()
}
// depending on cache size this might return an undecorated storage
ret.Decorator = genericregistry.StorageWithCacher(cacheSize)
}
return ret, nil

View file

@ -1,56 +0,0 @@
/*
Copyright 2018 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"os"
)
// ProcessInfo holds the apiserver process information used to send events
type ProcessInfo struct {
// Name of the api process to identify events
Name string
// Namespace of the api process to send events
Namespace string
}
// NewProcessInfo returns a new process info with the hostname concatenated to the name given
func NewProcessInfo(name, namespace string) *ProcessInfo {
// try to concat the hostname if available
host, _ := os.Hostname()
if host != "" {
name = fmt.Sprintf("%s-%s", name, host)
}
return &ProcessInfo{
Name: name,
Namespace: namespace,
}
}
// validateProcessInfo checks for a complete process info
func validateProcessInfo(p *ProcessInfo) error {
if p == nil {
return fmt.Errorf("ProcessInfo must be set")
} else if p.Name == "" {
return fmt.Errorf("ProcessInfo name must be set")
} else if p.Namespace == "" {
return fmt.Errorf("ProcessInfo namespace must be set")
}
return nil
}

View file

@ -48,14 +48,11 @@ type RecommendedOptions struct {
// admission plugin initializers to Admission.ApplyTo.
ExtraAdmissionInitializers func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error)
Admission *AdmissionOptions
// ProcessInfo is used to identify events created by the server.
ProcessInfo *ProcessInfo
Webhook *WebhookOptions
// API Server Egress Selector is used to control outbound traffic from the API Server
EgressSelector *EgressSelectorOptions
}
func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *ProcessInfo) *RecommendedOptions {
func NewRecommendedOptions(prefix string, codec runtime.Codec) *RecommendedOptions {
sso := NewSecureServingOptions()
// We are composing recommended options for an aggregated api-server,
@ -78,8 +75,6 @@ func NewRecommendedOptions(prefix string, codec runtime.Codec, processInfo *Proc
FeatureGate: feature.DefaultFeatureGate,
ExtraAdmissionInitializers: func(c *server.RecommendedConfig) ([]admission.PluginInitializer, error) { return nil, nil },
Admission: NewAdmissionOptions(),
ProcessInfo: processInfo,
Webhook: NewWebhookOptions(),
EgressSelector: NewEgressSelectorOptions(),
}
}
@ -111,7 +106,7 @@ func (o *RecommendedOptions) ApplyTo(config *server.RecommendedConfig) error {
if err := o.Authorization.ApplyTo(&config.Config.Authorization); err != nil {
return err
}
if err := o.Audit.ApplyTo(&config.Config, config.ClientConfig, config.SharedInformerFactory, o.ProcessInfo, o.Webhook); err != nil {
if err := o.Audit.ApplyTo(&config.Config); err != nil {
return err
}
if err := o.Features.ApplyTo(&config.Config); err != nil {

View file

@ -50,7 +50,6 @@ type ServerRunOptions struct {
// We intentionally did not add a flag for this option. Users of the
// apiserver library can wire it to a flag.
MaxRequestBodyBytes int64
TargetRAMMB int
EnablePriorityAndFairness bool
}
@ -69,7 +68,7 @@ func NewServerRunOptions() *ServerRunOptions {
}
}
// ApplyOptions applies the run options to the method receiver and returns self
// ApplyTo applies the run options to the method receiver and returns self
func (s *ServerRunOptions) ApplyTo(c *server.Config) error {
c.CorsAllowedOriginList = s.CorsAllowedOriginList
c.ExternalAddress = s.ExternalHost
@ -108,9 +107,6 @@ func (s *ServerRunOptions) DefaultAdvertiseAddress(secure *SecureServingOptions)
// Validate checks validation of ServerRunOptions
func (s *ServerRunOptions) Validate() []error {
errors := []error{}
if s.TargetRAMMB < 0 {
errors = append(errors, fmt.Errorf("--target-ram-mb can not be negative value"))
}
if s.LivezGracePeriod < 0 {
errors = append(errors, fmt.Errorf("--livez-grace-period can not be a negative value"))
@ -165,8 +161,10 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"List of allowed origins for CORS, comma separated. An allowed origin can be a regular "+
"expression to support subdomain matching. If this list is empty CORS will not be enabled.")
fs.IntVar(&s.TargetRAMMB, "target-ram-mb", s.TargetRAMMB,
"Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
deprecatedTargetRAMMB := 0
fs.IntVar(&deprecatedTargetRAMMB, "target-ram-mb", deprecatedTargetRAMMB,
"DEPRECATED: Memory limit for apiserver in MB (used to configure sizes of caches, etc.)")
fs.MarkDeprecated("target-ram-mb", "This flag will be removed in v1.23")
fs.StringVar(&s.ExternalHost, "external-hostname", s.ExternalHost,
"The hostname to use when generating externalized URLs for this master (e.g. Swagger API Docs or OpenID Discovery).")
@ -209,8 +207,8 @@ func (s *ServerRunOptions) AddUniversalFlags(fs *pflag.FlagSet) {
"If true and the APIPriorityAndFairness feature gate is enabled, replace the max-in-flight handler with an enhanced one that queues and dispatches with priority and fairness")
fs.DurationVar(&s.ShutdownDelayDuration, "shutdown-delay-duration", s.ShutdownDelayDuration, ""+
"Time to delay the termination. During that time the server keeps serving requests normally and /healthz "+
"returns success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
"Time to delay the termination. During that time the server keeps serving requests normally. The endpoints /healthz and /livez "+
"will return success, but /readyz immediately returns failure. Graceful termination starts after this delay "+
"has elapsed. This can be used to allow load balancer to stop sending traffic to this server.")
utilfeature.DefaultMutableFeatureGate.AddFlag(fs)

View file

@ -17,6 +17,7 @@ limitations under the License.
package options
import (
"context"
"fmt"
"net"
"path"
@ -24,7 +25,7 @@ import (
"strings"
"github.com/spf13/pflag"
"k8s.io/klog"
"k8s.io/klog/v2"
utilnet "k8s.io/apimachinery/pkg/util/net"
"k8s.io/apiserver/pkg/server"
@ -66,6 +67,10 @@ type SecureServingOptions struct {
// HTTP2MaxStreamsPerConnection is the limit that the api server imposes on each client.
// A value of zero means to use the default provided by golang's HTTP/2 support.
HTTP2MaxStreamsPerConnection int
// PermitPortSharing controls if SO_REUSEPORT is used when binding the port, which allows
// more than one instance to bind on the same address and port.
PermitPortSharing bool
}
type CertKey struct {
@ -166,11 +171,13 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
fs.StringVar(&s.ServerCert.CertKey.KeyFile, "tls-private-key-file", s.ServerCert.CertKey.KeyFile,
"File containing the default x509 private key matching --tls-cert-file.")
tlsCipherPossibleValues := cliflag.TLSCipherPossibleValues()
tlsCipherPreferredValues := cliflag.PreferredTLSCipherNames()
tlsCipherInsecureValues := cliflag.InsecureTLSCipherNames()
fs.StringSliceVar(&s.CipherSuites, "tls-cipher-suites", s.CipherSuites,
"Comma-separated list of cipher suites for the server. "+
"If omitted, the default Go cipher suites will be use. "+
"Possible values: "+strings.Join(tlsCipherPossibleValues, ","))
"If omitted, the default Go cipher suites will be used. \n"+
"Preferred values: "+strings.Join(tlsCipherPreferredValues, ", ")+". \n"+
"Insecure values: "+strings.Join(tlsCipherInsecureValues, ", ")+".")
tlsPossibleVersions := cliflag.TLSPossibleVersions()
fs.StringVar(&s.MinTLSVersion, "tls-min-version", s.MinTLSVersion,
@ -192,6 +199,10 @@ func (s *SecureServingOptions) AddFlags(fs *pflag.FlagSet) {
"The limit that the server gives to clients for "+
"the maximum number of streams in an HTTP/2 connection. "+
"Zero means to use golang's default.")
fs.BoolVar(&s.PermitPortSharing, "permit-port-sharing", s.PermitPortSharing,
"If true, SO_REUSEPORT will be used when binding the port, which allows "+
"more than one instance to bind on the same address and port. [default=false]")
}
// ApplyTo fills up serving information in the server configuration.
@ -206,7 +217,14 @@ func (s *SecureServingOptions) ApplyTo(config **server.SecureServingInfo) error
if s.Listener == nil {
var err error
addr := net.JoinHostPort(s.BindAddress.String(), strconv.Itoa(s.BindPort))
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr)
c := net.ListenConfig{}
if s.PermitPortSharing {
c.Control = permitPortReuse
}
s.Listener, s.BindPort, err = CreateListener(s.BindNetwork, addr, c)
if err != nil {
return fmt.Errorf("failed to create listener: %v", err)
}
@ -317,11 +335,12 @@ func (s *SecureServingOptions) MaybeDefaultWithSelfSignedCerts(publicAddress str
return nil
}
func CreateListener(network, addr string) (net.Listener, int, error) {
func CreateListener(network, addr string, config net.ListenConfig) (net.Listener, int, error) {
if len(network) == 0 {
network = "tcp"
}
ln, err := net.Listen(network, addr)
ln, err := config.Listen(context.TODO(), network, addr)
if err != nil {
return nil, 0, fmt.Errorf("failed to listen on %v: %v", addr, err)
}

View file

@ -1,5 +1,7 @@
// +build !windows
/*
Copyright 2018 The Kubernetes Authors.
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
@ -17,18 +19,13 @@ limitations under the License.
package options
import (
utilwebhook "k8s.io/apiserver/pkg/util/webhook"
"syscall"
"golang.org/x/sys/unix"
)
// WebhookOptions holds the outgoing webhook options
type WebhookOptions struct {
ServiceResolver utilwebhook.ServiceResolver
AuthInfoResolverWrapper utilwebhook.AuthenticationInfoResolverWrapper
}
// NewWebhookOptions returns the default options for outgoing webhooks
func NewWebhookOptions() *WebhookOptions {
return &WebhookOptions{
ServiceResolver: utilwebhook.NewDefaultServiceResolver(),
}
func permitPortReuse(network, addr string, conn syscall.RawConn) error {
return conn.Control(func(fd uintptr) {
syscall.SetsockoptInt(int(fd), syscall.SOL_SOCKET, unix.SO_REUSEPORT, 1)
})
}

View file

@ -0,0 +1,30 @@
// +build windows
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package options
import (
"fmt"
"syscall"
)
// Windows only supports SO_REUSEADDR, which may cause undefined behavior, as
// there is no protection against port hijacking.
func permitPortReuse(network, address string, c syscall.RawConn) error {
return fmt.Errorf("port reuse is not supported on Windows")
}

View file

@ -26,7 +26,7 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
serverstore "k8s.io/apiserver/pkg/server/storage"
cliflag "k8s.io/component-base/cli/flag"
"k8s.io/klog"
"k8s.io/klog/v2"
)
// GroupVersionRegistry provides access to registered group versions.

View file

@ -24,7 +24,7 @@ import (
"path"
"sync"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apiserver/pkg/server/mux"
)

View file

@ -17,9 +17,6 @@ limitations under the License.
package routes
import (
"io"
"net/http"
apimetrics "k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/mux"
etcd3metrics "k8s.io/apiserver/pkg/storage/etcd3/metrics"
@ -43,16 +40,7 @@ type MetricsWithReset struct{}
// Install adds the MetricsWithReset handler
func (m MetricsWithReset) Install(c *mux.PathRecorderMux) {
register()
defaultMetricsHandler := legacyregistry.Handler().ServeHTTP
c.HandleFunc("/metrics", func(w http.ResponseWriter, req *http.Request) {
if req.Method == "DELETE" {
apimetrics.Reset()
etcd3metrics.Reset()
io.WriteString(w, "metrics reset\n")
return
}
defaultMetricsHandler(w, req)
})
c.Handle("/metrics", legacyregistry.HandlerWithReset())
}
// register apiserver and etcd metrics

View file

@ -19,7 +19,7 @@ package routes
import (
restful "github.com/emicklei/go-restful"
"github.com/go-openapi/spec"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apiserver/pkg/server/mux"
"k8s.io/kube-openapi/pkg/builder"
@ -38,9 +38,16 @@ func (oa OpenAPI) Install(c *restful.Container, mux *mux.PathRecorderMux) (*hand
if err != nil {
klog.Fatalf("Failed to build open api spec for root: %v", err)
}
openAPIVersionedService, err := handler.RegisterOpenAPIVersionedService(spec, "/openapi/v2", mux)
openAPIVersionedService, err := handler.NewOpenAPIService(spec)
if err != nil {
klog.Fatalf("Failed to create OpenAPIService: %v", err)
}
err = openAPIVersionedService.RegisterOpenAPIVersionedService("/openapi/v2", mux)
if err != nil {
klog.Fatalf("Failed to register versioned open api spec for root: %v", err)
}
return openAPIVersionedService, spec
}

View file

@ -20,14 +20,20 @@ import (
"context"
"crypto/tls"
"fmt"
"io"
"log"
"net"
"net/http"
"os"
"strings"
"time"
"golang.org/x/net/http2"
"k8s.io/klog"
"k8s.io/component-base/cli/flag"
"k8s.io/klog/v2"
utilruntime "k8s.io/apimachinery/pkg/util/runtime"
"k8s.io/apiserver/pkg/endpoints/metrics"
"k8s.io/apiserver/pkg/server/dynamiccertificates"
)
@ -56,6 +62,14 @@ func (s *SecureServingInfo) tlsConfig(stopCh <-chan struct{}) (*tls.Config, erro
}
if len(s.CipherSuites) > 0 {
tlsConfig.CipherSuites = s.CipherSuites
insecureCiphers := flag.InsecureTLSCiphers()
for i := 0; i < len(s.CipherSuites); i++ {
for cipherName, cipherID := range insecureCiphers {
if s.CipherSuites[i] == cipherID {
klog.Warningf("Use of insecure cipher '%s' detected.", cipherName)
}
}
}
}
if s.ClientCA != nil {
@ -175,6 +189,11 @@ func (s *SecureServingInfo) Serve(handler http.Handler, shutdownTimeout time.Dur
}
}
// use tlsHandshakeErrorWriter to handle messages of tls handshake error
tlsErrorWriter := &tlsHandshakeErrorWriter{os.Stderr}
tlsErrorLogger := log.New(tlsErrorWriter, "", 0)
secureServer.ErrorLog = tlsErrorLogger
klog.Infof("Serving securely on %s", secureServer.Addr)
return RunServer(secureServer, s.Listener, shutdownTimeout, stopCh)
}
@ -209,7 +228,7 @@ func RunServer(
defer utilruntime.HandleCrash()
var listener net.Listener
listener = tcpKeepAliveListener{ln.(*net.TCPListener)}
listener = tcpKeepAliveListener{ln}
if server.TLSConfig != nil {
listener = tls.NewListener(listener, server.TLSConfig)
}
@ -235,15 +254,36 @@ func RunServer(
//
// Copied from Go 1.7.2 net/http/server.go
type tcpKeepAliveListener struct {
*net.TCPListener
net.Listener
}
func (ln tcpKeepAliveListener) Accept() (net.Conn, error) {
tc, err := ln.AcceptTCP()
c, err := ln.Listener.Accept()
if err != nil {
return nil, err
}
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
return tc, nil
if tc, ok := c.(*net.TCPConn); ok {
tc.SetKeepAlive(true)
tc.SetKeepAlivePeriod(defaultKeepAlivePeriod)
}
return c, nil
}
// tlsHandshakeErrorWriter writes TLS handshake errors to klog with
// trace level - V(5), to avoid flooding of tls handshake errors.
type tlsHandshakeErrorWriter struct {
out io.Writer
}
const tlsHandshakeErrorPrefix = "http: TLS handshake error"
func (w *tlsHandshakeErrorWriter) Write(p []byte) (int, error) {
if strings.Contains(string(p), tlsHandshakeErrorPrefix) {
klog.V(5).Info(string(p))
metrics.TLSHandshakeErrors.Inc()
return len(p), nil
}
// for non tls handshake error, log it as usual
return w.out.Write(p)
}

View file

@ -17,6 +17,7 @@ limitations under the License.
package server
import (
"context"
"os"
"os/signal"
)
@ -27,21 +28,30 @@ var shutdownHandler chan os.Signal
// SetupSignalHandler registered for SIGTERM and SIGINT. A stop channel is returned
// which is closed on one of these signals. If a second signal is caught, the program
// is terminated with exit code 1.
// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
// be called once.
func SetupSignalHandler() <-chan struct{} {
return SetupSignalContext().Done()
}
// SetupSignalContext is same as SetupSignalHandler, but a context.Context is returned.
// Only one of SetupSignalContext and SetupSignalHandler should be called, and only can
// be called once.
func SetupSignalContext() context.Context {
close(onlyOneSignalHandler) // panics when called twice
shutdownHandler = make(chan os.Signal, 2)
stop := make(chan struct{})
ctx, cancel := context.WithCancel(context.Background())
signal.Notify(shutdownHandler, shutdownSignals...)
go func() {
<-shutdownHandler
close(stop)
cancel()
<-shutdownHandler
os.Exit(1) // second signal. Exit directly.
}()
return stop
return ctx
}
// RequestShutdown emulates a received event that is considered as shutdown signal (SIGTERM/SIGINT)

View file

@ -22,7 +22,7 @@ import (
"io/ioutil"
"strings"
"k8s.io/klog"
"k8s.io/klog/v2"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"