Update custom-metrics-apiserver and metrics-server

vendor/k8s.io/apiserver/pkg/authentication/token/cache/cached_token_authenticator.go

@@ -22,16 +22,26 @@ import (
 	"crypto/rand"
 	"crypto/sha256"
 	"encoding/binary"
+	"errors"
 	"hash"
 	"io"
+	"runtime"
 	"sync"
 	"time"
 	"unsafe"
 
+	"golang.org/x/sync/singleflight"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	utilclock "k8s.io/apimachinery/pkg/util/clock"
 	"k8s.io/apiserver/pkg/authentication/authenticator"
+	"k8s.io/klog"
 )
 
+var errAuthnCrash = apierrors.NewInternalError(errors.New("authentication failed unexpectedly"))
+
+const sharedLookupTimeout = 30 * time.Second
+
 // cacheRecord holds the three return values of the authenticator.Token AuthenticateToken method
 type cacheRecord struct {
 	resp *authenticator.Response
@@ -47,6 +57,7 @@ type cachedTokenAuthenticator struct {
 	failureTTL time.Duration
 
 	cache cache
+	group singleflight.Group
 
 	// hashPool is a per authenticator pool of hash.Hash (to avoid allocations from building the Hash)
 	// HMAC with SHA-256 and a random key is used to prevent precomputation and length extension attacks
@@ -98,26 +109,82 @@ func newWithClock(authenticator authenticator.Token, cacheErrs bool, successTTL,
 
 // AuthenticateToken implements authenticator.Token
 func (a *cachedTokenAuthenticator) AuthenticateToken(ctx context.Context, token string) (*authenticator.Response, bool, error) {
-	auds, _ := authenticator.AudiencesFrom(ctx)
+	doneAuthenticating := stats.authenticating()
+
+	auds, audsOk := authenticator.AudiencesFrom(ctx)
 
 	key := keyFunc(a.hashPool, auds, token)
 	if record, ok := a.cache.get(key); ok {
+		// Record cache hit
+		doneAuthenticating(true)
 		return record.resp, record.ok, record.err
 	}
 
-	resp, ok, err := a.authenticator.AuthenticateToken(ctx, token)
-	if !a.cacheErrs && err != nil {
-		return resp, ok, err
+	// Record cache miss
+	doneBlocking := stats.blocking()
+	defer doneBlocking()
+	defer doneAuthenticating(false)
+
+	type lookup struct {
+		resp *authenticator.Response
+		ok   bool
 	}
 
-	switch {
-	case ok && a.successTTL > 0:
-		a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.successTTL)
-	case !ok && a.failureTTL > 0:
-		a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.failureTTL)
-	}
+	c := a.group.DoChan(key, func() (val interface{}, err error) {
+		doneFetching := stats.fetching()
+		// We're leaving the request handling stack so we need to handle crashes
+		// ourselves. Log a stack trace and return a 500 if something panics.
+		defer func() {
+			if r := recover(); r != nil {
+				err = errAuthnCrash
+				// Same as stdlib http server code. Manually allocate stack
+				// trace buffer size to prevent excessively large logs
+				const size = 64 << 10
+				buf := make([]byte, size)
+				buf = buf[:runtime.Stack(buf, false)]
+				klog.Errorf("%v\n%s", r, buf)
+			}
+			doneFetching(err == nil)
+		}()
 
-	return resp, ok, err
+		// Check again for a cached record. We may have raced with a fetch.
+		if record, ok := a.cache.get(key); ok {
+			return lookup{record.resp, record.ok}, record.err
+		}
+
+		// Detach the context because the lookup may be shared by multiple callers,
+		// however propagate the audience.
+		ctx, cancel := context.WithTimeout(context.Background(), sharedLookupTimeout)
+		defer cancel()
+
+		if audsOk {
+			ctx = authenticator.WithAudiences(ctx, auds)
+		}
+
+		resp, ok, err := a.authenticator.AuthenticateToken(ctx, token)
+		if !a.cacheErrs && err != nil {
+			return nil, err
+		}
+
+		switch {
+		case ok && a.successTTL > 0:
+			a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.successTTL)
+		case !ok && a.failureTTL > 0:
+			a.cache.set(key, &cacheRecord{resp: resp, ok: ok, err: err}, a.failureTTL)
+		}
+		return lookup{resp, ok}, err
+	})
+
+	select {
+	case result := <-c:
+		if result.Err != nil {
+			return nil, false, result.Err
+		}
+		lookup := result.Val.(lookup)
+		return lookup.resp, lookup.ok, nil
+	case <-ctx.Done():
+		return nil, false, ctx.Err()
+	}
 }
 
 // keyFunc generates a string key by hashing the inputs.

vendor/k8s.io/apiserver/pkg/authentication/token/cache/stats.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package cache
+
+import (
+	"time"
+
+	"k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+)
+
+var (
+	requestLatency = metrics.NewHistogramVec(
+		&metrics.HistogramOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "request_duration_seconds",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	requestCount = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "request_total",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	fetchCount = metrics.NewCounterVec(
+		&metrics.CounterOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "fetch_total",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+	activeFetchCount = metrics.NewGaugeVec(
+		&metrics.GaugeOpts{
+			Namespace:      "authentication",
+			Subsystem:      "token_cache",
+			Name:           "active_fetch_count",
+			StabilityLevel: metrics.ALPHA,
+		},
+		[]string{"status"},
+	)
+)
+
+func init() {
+	legacyregistry.MustRegister(
+		requestLatency,
+		requestCount,
+		fetchCount,
+		activeFetchCount,
+	)
+}
+
+const (
+	hitTag  = "hit"
+	missTag = "miss"
+
+	fetchFailedTag = "error"
+	fetchOkTag     = "ok"
+
+	fetchInFlightTag = "in_flight"
+	fetchBlockedTag  = "blocked"
+)
+
+type statsCollector struct{}
+
+var stats = statsCollector{}
+
+func (statsCollector) authenticating() func(hit bool) {
+	start := time.Now()
+	return func(hit bool) {
+		var tag string
+		if hit {
+			tag = hitTag
+		} else {
+			tag = missTag
+		}
+
+		latency := time.Since(start)
+
+		requestCount.WithLabelValues(tag).Inc()
+		requestLatency.WithLabelValues(tag).Observe(float64(latency.Milliseconds()) / 1000)
+	}
+}
+
+func (statsCollector) blocking() func() {
+	activeFetchCount.WithLabelValues(fetchBlockedTag).Inc()
+	return activeFetchCount.WithLabelValues(fetchBlockedTag).Dec
+}
+
+func (statsCollector) fetching() func(ok bool) {
+	activeFetchCount.WithLabelValues(fetchInFlightTag).Inc()
+	return func(ok bool) {
+		var tag string
+		if ok {
+			tag = fetchOkTag
+		} else {
+			tag = fetchFailedTag
+		}
+
+		fetchCount.WithLabelValues(tag).Inc()
+
+		activeFetchCount.WithLabelValues(fetchInFlightTag).Dec()
+	}
+}