Update custom-metrics-apiserver and metrics-server

vendor/k8s.io/apiserver/pkg/util/apihelpers/helpers.go

@@ -0,0 +1,100 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apihelpers
+
+import (
+	"sort"
+
+	flowcontrol "k8s.io/api/flowcontrol/v1alpha1"
+)
+
+// SetFlowSchemaCondition sets conditions.
+func SetFlowSchemaCondition(flowSchema *flowcontrol.FlowSchema, newCondition flowcontrol.FlowSchemaCondition) {
+	existingCondition := GetFlowSchemaConditionByType(flowSchema, newCondition.Type)
+	if existingCondition == nil {
+		flowSchema.Status.Conditions = append(flowSchema.Status.Conditions, newCondition)
+		return
+	}
+
+	if existingCondition.Status != newCondition.Status {
+		existingCondition.Status = newCondition.Status
+		existingCondition.LastTransitionTime = newCondition.LastTransitionTime
+	}
+
+	existingCondition.Reason = newCondition.Reason
+	existingCondition.Message = newCondition.Message
+}
+
+// GetFlowSchemaConditionByType gets conditions.
+func GetFlowSchemaConditionByType(flowSchema *flowcontrol.FlowSchema, conditionType flowcontrol.FlowSchemaConditionType) *flowcontrol.FlowSchemaCondition {
+	for i := range flowSchema.Status.Conditions {
+		if flowSchema.Status.Conditions[i].Type == conditionType {
+			return &flowSchema.Status.Conditions[i]
+		}
+	}
+	return nil
+}
+
+// SetPriorityLevelConfigurationCondition sets conditions.
+func SetPriorityLevelConfigurationCondition(priorityLevel *flowcontrol.PriorityLevelConfiguration, newCondition flowcontrol.PriorityLevelConfigurationCondition) {
+	existingCondition := GetPriorityLevelConfigurationConditionByType(priorityLevel, newCondition.Type)
+	if existingCondition == nil {
+		priorityLevel.Status.Conditions = append(priorityLevel.Status.Conditions, newCondition)
+		return
+	}
+
+	if existingCondition.Status != newCondition.Status {
+		existingCondition.Status = newCondition.Status
+		existingCondition.LastTransitionTime = newCondition.LastTransitionTime
+	}
+
+	existingCondition.Reason = newCondition.Reason
+	existingCondition.Message = newCondition.Message
+}
+
+// GetPriorityLevelConfigurationConditionByType gets conditions.
+func GetPriorityLevelConfigurationConditionByType(priorityLevel *flowcontrol.PriorityLevelConfiguration, conditionType flowcontrol.PriorityLevelConfigurationConditionType) *flowcontrol.PriorityLevelConfigurationCondition {
+	for i := range priorityLevel.Status.Conditions {
+		if priorityLevel.Status.Conditions[i].Type == conditionType {
+			return &priorityLevel.Status.Conditions[i]
+		}
+	}
+	return nil
+}
+
+var _ sort.Interface = FlowSchemaSequence{}
+
+// FlowSchemaSequence holds sorted set of pointers to FlowSchema objects.
+// FlowSchemaSequence implements `sort.Interface`
+type FlowSchemaSequence []*flowcontrol.FlowSchema
+
+func (s FlowSchemaSequence) Len() int {
+	return len(s)
+}
+
+func (s FlowSchemaSequence) Less(i, j int) bool {
+	// the flow-schema w/ lower matching-precedence is prior
+	if ip, jp := s[i].Spec.MatchingPrecedence, s[j].Spec.MatchingPrecedence; ip != jp {
+		return ip < jp
+	}
+	// sort alphabetically
+	return s[i].Name < s[j].Name
+}
+
+func (s FlowSchemaSequence) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_controller.go

@@ -0,0 +1,725 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flowcontrol
+
+import (
+	"context"
+	"crypto/sha256"
+	"encoding/binary"
+	"encoding/json"
+	"fmt"
+	"math"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	apiequality "k8s.io/apimachinery/pkg/api/equality"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	apitypes "k8s.io/apimachinery/pkg/types"
+	apierrors "k8s.io/apimachinery/pkg/util/errors"
+	"k8s.io/apimachinery/pkg/util/wait"
+	fcboot "k8s.io/apiserver/pkg/apis/flowcontrol/bootstrap"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+	"k8s.io/apiserver/pkg/util/apihelpers"
+	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
+	fcfmt "k8s.io/apiserver/pkg/util/flowcontrol/format"
+	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/client-go/tools/cache"
+	"k8s.io/client-go/util/workqueue"
+	"k8s.io/klog"
+
+	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	fcclientv1a1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1"
+	fclistersv1a1 "k8s.io/client-go/listers/flowcontrol/v1alpha1"
+)
+
+// This file contains a simple local (to the apiserver) controller
+// that digests API Priority and Fairness config objects (FlowSchema
+// and PriorityLevelConfiguration) into the data structure that the
+// filter uses.  At this first level of development this controller
+// takes the simplest possible approach: whenever notified of any
+// change to any config object, or when any priority level that is
+// undesired becomes completely unused, all the config objects are
+// read and processed as a whole.
+
+// StartFunction begins the process of handlig a request.  If the
+// request gets queued then this function uses the given hashValue as
+// the source of entropy as it shuffle-shards the request into a
+// queue.  The descr1 and descr2 values play no role in the logic but
+// appear in log messages.  This method does not return until the
+// queuing, if any, for this request is done.  If `execute` is false
+// then `afterExecution` is irrelevant and the request should be
+// rejected.  Otherwise the request should be executed and
+// `afterExecution` must be called exactly once.
+type StartFunction func(ctx context.Context, hashValue uint64) (execute bool, afterExecution func())
+
+// RequestDigest holds necessary info from request for flow-control
+type RequestDigest struct {
+	RequestInfo *request.RequestInfo
+	User        user.Info
+}
+
+// `*configController` maintains eventual consistency with the API
+// objects that configure API Priority and Fairness, and provides a
+// procedural interface to the configured behavior.  The methods of
+// this type and cfgMeal follow the convention that the suffix
+// "Locked" means that the caller must hold the configController lock.
+type configController struct {
+	queueSetFactory fq.QueueSetFactory
+
+	// configQueue holds `(interface{})(0)` when the configuration
+	// objects need to be reprocessed.
+	configQueue workqueue.RateLimitingInterface
+
+	plLister         fclistersv1a1.PriorityLevelConfigurationLister
+	plInformerSynced cache.InformerSynced
+
+	fsLister         fclistersv1a1.FlowSchemaLister
+	fsInformerSynced cache.InformerSynced
+
+	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface
+
+	// serverConcurrencyLimit is the limit on the server's total
+	// number of non-exempt requests being served at once.  This comes
+	// from server configuration.
+	serverConcurrencyLimit int
+
+	// requestWaitLimit comes from server configuration.
+	requestWaitLimit time.Duration
+
+	// This must be locked while accessing flowSchemas or
+	// priorityLevelStates.  It is the lock involved in
+	// LockingWriteMultiple.
+	lock sync.Mutex
+
+	// flowSchemas holds the flow schema objects, sorted by increasing
+	// numerical (decreasing logical) matching precedence.  Every
+	// FlowSchema in this slice is immutable.
+	flowSchemas apihelpers.FlowSchemaSequence
+
+	// priorityLevelStates maps the PriorityLevelConfiguration object
+	// name to the state for that level.  Every name referenced from a
+	// member of `flowSchemas` has an entry here.
+	priorityLevelStates map[string]*priorityLevelState
+}
+
+// priorityLevelState holds the state specific to a priority level.
+type priorityLevelState struct {
+	// the API object or prototype prescribing this level.  Nothing
+	// reached through this pointer is mutable.
+	pl *fctypesv1a1.PriorityLevelConfiguration
+
+	// qsCompleter holds the QueueSetCompleter derived from `config`
+	// and `queues` if config is not exempt, nil otherwise.
+	qsCompleter fq.QueueSetCompleter
+
+	// The QueueSet for this priority level.  This is nil if and only
+	// if the priority level is exempt.
+	queues fq.QueueSet
+
+	// quiescing==true indicates that this priority level should be
+	// removed when its queues have all drained.  May be true only if
+	// queues is non-nil.
+	quiescing bool
+
+	// number of goroutines between Controller::Match and calling the
+	// returned StartFunction
+	numPending int
+}
+
+// NewTestableController is extra flexible to facilitate testing
+func newTestableController(
+	informerFactory kubeinformers.SharedInformerFactory,
+	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	serverConcurrencyLimit int,
+	requestWaitLimit time.Duration,
+	queueSetFactory fq.QueueSetFactory,
+) *configController {
+	cfgCtl := &configController{
+		queueSetFactory:        queueSetFactory,
+		serverConcurrencyLimit: serverConcurrencyLimit,
+		requestWaitLimit:       requestWaitLimit,
+		flowcontrolClient:      flowcontrolClient,
+		priorityLevelStates:    make(map[string]*priorityLevelState),
+	}
+	klog.V(2).Infof("NewTestableController with serverConcurrencyLimit=%d, requestWaitLimit=%s", serverConcurrencyLimit, requestWaitLimit)
+	cfgCtl.initializeConfigController(informerFactory)
+	// ensure the data structure reflects the mandatory config
+	cfgCtl.lockAndDigestConfigObjects(nil, nil)
+	return cfgCtl
+}
+
+// initializeConfigController sets up the controller that processes
+// config API objects.
+func (cfgCtl *configController) initializeConfigController(informerFactory kubeinformers.SharedInformerFactory) {
+	cfgCtl.configQueue = workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(200*time.Millisecond, 8*time.Hour), "priority_and_fairness_config_queue")
+	fci := informerFactory.Flowcontrol().V1alpha1()
+	pli := fci.PriorityLevelConfigurations()
+	fsi := fci.FlowSchemas()
+	cfgCtl.plLister = pli.Lister()
+	cfgCtl.plInformerSynced = pli.Informer().HasSynced
+	cfgCtl.fsLister = fsi.Lister()
+	cfgCtl.fsInformerSynced = fsi.Informer().HasSynced
+	pli.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			pl := obj.(*fctypesv1a1.PriorityLevelConfiguration)
+			klog.V(7).Infof("Triggered API priority and fairness config reloading due to creation of PLC %s", pl.Name)
+			cfgCtl.configQueue.Add(0)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			newPL := newObj.(*fctypesv1a1.PriorityLevelConfiguration)
+			oldPL := oldObj.(*fctypesv1a1.PriorityLevelConfiguration)
+			if !apiequality.Semantic.DeepEqual(oldPL.Spec, newPL.Spec) {
+				klog.V(7).Infof("Triggered API priority and fairness config reloading due to spec update of PLC %s", newPL.Name)
+				cfgCtl.configQueue.Add(0)
+			}
+		},
+		DeleteFunc: func(obj interface{}) {
+			name, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+			klog.V(7).Infof("Triggered API priority and fairness config reloading due to deletion of PLC %s", name)
+			cfgCtl.configQueue.Add(0)
+
+		}})
+	fsi.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj interface{}) {
+			fs := obj.(*fctypesv1a1.FlowSchema)
+			klog.V(7).Infof("Triggered API priority and fairness config reloading due to creation of FS %s", fs.Name)
+			cfgCtl.configQueue.Add(0)
+		},
+		UpdateFunc: func(oldObj, newObj interface{}) {
+			newFS := newObj.(*fctypesv1a1.FlowSchema)
+			oldFS := oldObj.(*fctypesv1a1.FlowSchema)
+			if !apiequality.Semantic.DeepEqual(oldFS.Spec, newFS.Spec) {
+				klog.V(7).Infof("Triggered API priority and fairness config reloading due to spec update of FS %s", newFS.Name)
+				cfgCtl.configQueue.Add(0)
+			}
+		},
+		DeleteFunc: func(obj interface{}) {
+			name, _ := cache.DeletionHandlingMetaNamespaceKeyFunc(obj)
+			klog.V(7).Infof("Triggered API priority and fairness config reloading due to deletion of FS %s", name)
+			cfgCtl.configQueue.Add(0)
+
+		}})
+}
+
+func (cfgCtl *configController) Run(stopCh <-chan struct{}) error {
+	defer cfgCtl.configQueue.ShutDown()
+	klog.Info("Starting API Priority and Fairness config controller")
+	if ok := cache.WaitForCacheSync(stopCh, cfgCtl.plInformerSynced, cfgCtl.fsInformerSynced); !ok {
+		return fmt.Errorf("Never achieved initial sync")
+	}
+	klog.Info("Running API Priority and Fairness config worker")
+	wait.Until(cfgCtl.runWorker, time.Second, stopCh)
+	klog.Info("Shutting down API Priority and Fairness config worker")
+	return nil
+}
+
+func (cfgCtl *configController) runWorker() {
+	for cfgCtl.processNextWorkItem() {
+	}
+}
+
+func (cfgCtl *configController) processNextWorkItem() bool {
+	obj, shutdown := cfgCtl.configQueue.Get()
+	if shutdown {
+		return false
+	}
+
+	func(obj interface{}) {
+		defer cfgCtl.configQueue.Done(obj)
+		if !cfgCtl.syncOne() {
+			cfgCtl.configQueue.AddRateLimited(obj)
+		} else {
+			cfgCtl.configQueue.Forget(obj)
+		}
+	}(obj)
+
+	return true
+}
+
+// syncOne attempts to sync all the API Priority and Fairness config
+// objects.  It either succeeds and returns `true` or logs an error
+// and returns `false`.
+func (cfgCtl *configController) syncOne() bool {
+	all := labels.Everything()
+	newPLs, err := cfgCtl.plLister.List(all)
+	if err != nil {
+		klog.Errorf("Unable to list PriorityLevelConfiguration objects: %s", err.Error())
+		return false
+	}
+	newFSs, err := cfgCtl.fsLister.List(all)
+	if err != nil {
+		klog.Errorf("Unable to list FlowSchema objects: %s", err.Error())
+		return false
+	}
+	err = cfgCtl.digestConfigObjects(newPLs, newFSs)
+	if err == nil {
+		return true
+	}
+	klog.Error(err)
+	return false
+}
+
+// cfgMeal is the data involved in the process of digesting the API
+// objects that configure API Priority and Fairness.  All the config
+// objects are digested together, because this is the simplest way to
+// cope with the various dependencies between objects.  The process of
+// digestion is done in four passes over config objects --- three
+// passes over PriorityLevelConfigurations and one pass over the
+// FlowSchemas --- with the work dvided among the passes according to
+// those dependencies.
+type cfgMeal struct {
+	cfgCtl *configController
+
+	newPLStates map[string]*priorityLevelState
+
+	// The sum of the concurrency shares of the priority levels in the
+	// new configuration
+	shareSum float64
+
+	// These keep track of which mandatory priority level config
+	// objects have been digested
+	haveExemptPL, haveCatchAllPL bool
+
+	// Buffered FlowSchema status updates to do.  Do them when the
+	// lock is not held, to avoid a deadlock due to such a request
+	// provoking a call into this controller while the lock held
+	// waiting on that request to complete.
+	fsStatusUpdates []fsStatusUpdate
+}
+
+// A buffered set of status updates for a FlowSchema
+type fsStatusUpdate struct {
+	flowSchema *fctypesv1a1.FlowSchema
+	condition  fctypesv1a1.FlowSchemaCondition
+	oldValue   fctypesv1a1.FlowSchemaCondition
+}
+
+// digestConfigObjects is given all the API objects that configure
+// cfgCtl and writes its consequent new configState.
+func (cfgCtl *configController) digestConfigObjects(newPLs []*fctypesv1a1.PriorityLevelConfiguration, newFSs []*fctypesv1a1.FlowSchema) error {
+	fsStatusUpdates := cfgCtl.lockAndDigestConfigObjects(newPLs, newFSs)
+	var errs []error
+	for _, fsu := range fsStatusUpdates {
+		enc, err := json.Marshal(fsu.condition)
+		if err != nil {
+			// should never happen because these conditions are created here and well formed
+			panic(fmt.Sprintf("Failed to json.Marshall(%#+v): %s", fsu.condition, err.Error()))
+		}
+		klog.V(4).Infof("Writing Condition %s to FlowSchema %s because its previous value was %s", string(enc), fsu.flowSchema.Name, fcfmt.Fmt(fsu.oldValue))
+		_, err = cfgCtl.flowcontrolClient.FlowSchemas().Patch(context.TODO(), fsu.flowSchema.Name, apitypes.StrategicMergePatchType, []byte(fmt.Sprintf(`{"status": {"conditions": [ %s ] } }`, string(enc))), metav1.PatchOptions{FieldManager: "api-priority-and-fairness-config-consumer-v1"}, "status")
+		if err != nil {
+			errs = append(errs, errors.Wrap(err, fmt.Sprintf("failed to set a status.condition for FlowSchema %s", fsu.flowSchema.Name)))
+		}
+	}
+	if len(errs) == 0 {
+		return nil
+	}
+	return apierrors.NewAggregate(errs)
+}
+
+func (cfgCtl *configController) lockAndDigestConfigObjects(newPLs []*fctypesv1a1.PriorityLevelConfiguration, newFSs []*fctypesv1a1.FlowSchema) []fsStatusUpdate {
+	cfgCtl.lock.Lock()
+	defer cfgCtl.lock.Unlock()
+	meal := cfgMeal{
+		cfgCtl:      cfgCtl,
+		newPLStates: make(map[string]*priorityLevelState),
+	}
+
+	meal.digestNewPLsLocked(newPLs)
+	meal.digestFlowSchemasLocked(newFSs)
+	meal.processOldPLsLocked()
+
+	// Supply missing mandatory PriorityLevelConfiguration objects
+	if !meal.haveExemptPL {
+		meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationExempt, cfgCtl.requestWaitLimit)
+	}
+	if !meal.haveCatchAllPL {
+		meal.imaginePL(fcboot.MandatoryPriorityLevelConfigurationCatchAll, cfgCtl.requestWaitLimit)
+	}
+
+	meal.finishQueueSetReconfigsLocked()
+
+	// The new config has been constructed
+	cfgCtl.priorityLevelStates = meal.newPLStates
+	klog.V(5).Infof("Switched to new API Priority and Fairness configuration")
+	return meal.fsStatusUpdates
+}
+
+// Digest the new set of PriorityLevelConfiguration objects.
+// Pretend broken ones do not exist.
+func (meal *cfgMeal) digestNewPLsLocked(newPLs []*fctypesv1a1.PriorityLevelConfiguration) {
+	for _, pl := range newPLs {
+		state := meal.cfgCtl.priorityLevelStates[pl.Name]
+		if state == nil {
+			state = &priorityLevelState{}
+		}
+		qsCompleter, err := qscOfPL(meal.cfgCtl.queueSetFactory, state.queues, pl, meal.cfgCtl.requestWaitLimit)
+		if err != nil {
+			klog.Warningf("Ignoring PriorityLevelConfiguration object %s because its spec (%s) is broken: %s", pl.Name, fcfmt.Fmt(pl.Spec), err)
+			continue
+		}
+		meal.newPLStates[pl.Name] = state
+		state.pl = pl
+		state.qsCompleter = qsCompleter
+		if state.quiescing { // it was undesired, but no longer
+			klog.V(3).Infof("Priority level %q was undesired and has become desired again", pl.Name)
+			state.quiescing = false
+		}
+		if state.pl.Spec.Limited != nil {
+			meal.shareSum += float64(state.pl.Spec.Limited.AssuredConcurrencyShares)
+		}
+		meal.haveExemptPL = meal.haveExemptPL || pl.Name == fctypesv1a1.PriorityLevelConfigurationNameExempt
+		meal.haveCatchAllPL = meal.haveCatchAllPL || pl.Name == fctypesv1a1.PriorityLevelConfigurationNameCatchAll
+	}
+}
+
+// Digest the given FlowSchema objects.  Ones that reference a missing
+// or broken priority level are not to be passed on to the filter for
+// use.  We do this before holding over old priority levels so that
+// requests stop going to those levels and FlowSchemaStatus values
+// reflect this.  This function also adds any missing mandatory
+// FlowSchema objects.  The given objects must all have distinct
+// names.
+func (meal *cfgMeal) digestFlowSchemasLocked(newFSs []*fctypesv1a1.FlowSchema) {
+	fsSeq := make(apihelpers.FlowSchemaSequence, 0, len(newFSs))
+	fsMap := make(map[string]*fctypesv1a1.FlowSchema, len(newFSs))
+	var haveExemptFS, haveCatchAllFS bool
+	for i, fs := range newFSs {
+		otherFS := fsMap[fs.Name]
+		if otherFS != nil {
+			// This client is forbidden to do this.
+			panic(fmt.Sprintf("Given two FlowSchema objects with the same name: %s and %s", fcfmt.Fmt(otherFS), fcfmt.Fmt(fs)))
+		}
+		fsMap[fs.Name] = fs
+		_, goodPriorityRef := meal.newPLStates[fs.Spec.PriorityLevelConfiguration.Name]
+
+		// Ensure the object's status reflects whether its priority
+		// level reference is broken.
+		//
+		// TODO: consider not even trying if server is not handling
+		// requests yet.
+		meal.presyncFlowSchemaStatus(fs, !goodPriorityRef, fs.Spec.PriorityLevelConfiguration.Name)
+
+		if !goodPriorityRef {
+			klog.V(6).Infof("Ignoring FlowSchema %s because of bad priority level reference %q", fs.Name, fs.Spec.PriorityLevelConfiguration.Name)
+			continue
+		}
+		fsSeq = append(fsSeq, newFSs[i])
+		haveExemptFS = haveExemptFS || fs.Name == fctypesv1a1.FlowSchemaNameExempt
+		haveCatchAllFS = haveCatchAllFS || fs.Name == fctypesv1a1.FlowSchemaNameCatchAll
+	}
+	// sort into the order to be used for matching
+	sort.Sort(fsSeq)
+
+	// Supply missing mandatory FlowSchemas, in correct position
+	if !haveExemptFS {
+		fsSeq = append(apihelpers.FlowSchemaSequence{fcboot.MandatoryFlowSchemaExempt}, fsSeq...)
+	}
+	if !haveCatchAllFS {
+		fsSeq = append(fsSeq, fcboot.MandatoryFlowSchemaCatchAll)
+	}
+
+	meal.cfgCtl.flowSchemas = fsSeq
+	if klog.V(5) {
+		for _, fs := range fsSeq {
+			klog.Infof("Using FlowSchema %s", fcfmt.Fmt(fs))
+		}
+	}
+}
+
+// Consider all the priority levels in the previous configuration.
+// Keep the ones that are in the new config, supply mandatory
+// behavior, or are still busy; for the rest: drop it if it has no
+// queues, otherwise start the quiescing process if that has not
+// already been started.
+func (meal *cfgMeal) processOldPLsLocked() {
+	for plName, plState := range meal.cfgCtl.priorityLevelStates {
+		if meal.newPLStates[plName] != nil {
+			// Still desired and already updated
+			continue
+		}
+		if plName == fctypesv1a1.PriorityLevelConfigurationNameExempt && !meal.haveExemptPL || plName == fctypesv1a1.PriorityLevelConfigurationNameCatchAll && !meal.haveCatchAllPL {
+			// BTW, we know the Spec has not changed because the
+			// mandatory objects have immutable Specs
+			klog.V(3).Infof("Retaining mandatory priority level %q despite lack of API object", plName)
+		} else {
+			if plState.queues == nil || plState.numPending == 0 && plState.queues.IsIdle() {
+				// Either there are no queues or they are done
+				// draining and no use is coming from another
+				// goroutine
+				klog.V(3).Infof("Removing undesired priority level %q (nilQueues=%v), Type=%v", plName, plState.queues == nil, plState.pl.Spec.Type)
+				continue
+			}
+			if !plState.quiescing {
+				klog.V(3).Infof("Priority level %q became undesired", plName)
+				plState.quiescing = true
+			}
+		}
+		var err error
+		plState.qsCompleter, err = qscOfPL(meal.cfgCtl.queueSetFactory, plState.queues, plState.pl, meal.cfgCtl.requestWaitLimit)
+		if err != nil {
+			// This can not happen because qscOfPL already approved this config
+			panic(fmt.Sprintf("%s from name=%q spec=%s", err, plName, fcfmt.Fmt(plState.pl.Spec)))
+		}
+		if plState.pl.Spec.Limited != nil {
+			// We deliberately include the lingering priority levels
+			// here so that their queues get some concurrency and they
+			// continue to drain.  During this interim a lingering
+			// priority level continues to get a concurrency
+			// allocation determined by all the share values in the
+			// regular way.
+			meal.shareSum += float64(plState.pl.Spec.Limited.AssuredConcurrencyShares)
+		}
+		meal.haveExemptPL = meal.haveExemptPL || plName == fctypesv1a1.PriorityLevelConfigurationNameExempt
+		meal.haveCatchAllPL = meal.haveCatchAllPL || plName == fctypesv1a1.PriorityLevelConfigurationNameCatchAll
+		meal.newPLStates[plName] = plState
+	}
+}
+
+// For all the priority levels of the new config, divide up the
+// server's total concurrency limit among them and create/update their
+// QueueSets.
+func (meal *cfgMeal) finishQueueSetReconfigsLocked() {
+	for plName, plState := range meal.newPLStates {
+		if plState.pl.Spec.Limited == nil {
+			klog.V(5).Infof("Using exempt priority level %q: quiescing=%v", plName, plState.quiescing)
+			continue
+		}
+
+		// The use of math.Ceil here means that the results might sum
+		// to a little more than serverConcurrencyLimit but the
+		// difference will be negligible.
+		concurrencyLimit := int(math.Ceil(float64(meal.cfgCtl.serverConcurrencyLimit) * float64(plState.pl.Spec.Limited.AssuredConcurrencyShares) / meal.shareSum))
+		metrics.UpdateSharedConcurrencyLimit(plName, concurrencyLimit)
+
+		if plState.queues == nil {
+			klog.V(5).Infof("Introducing queues for priority level %q: config=%s, concurrencyLimit=%d, quiescing=%v (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, plState.quiescing, plState.pl.Spec.Limited.AssuredConcurrencyShares, meal.shareSum)
+		} else {
+			klog.V(5).Infof("Retaining queues for priority level %q: config=%s, concurrencyLimit=%d, quiescing=%v, numPending=%d (shares=%v, shareSum=%v)", plName, fcfmt.Fmt(plState.pl.Spec), concurrencyLimit, plState.quiescing, plState.numPending, plState.pl.Spec.Limited.AssuredConcurrencyShares, meal.shareSum)
+		}
+		plState.queues = plState.qsCompleter.Complete(fq.DispatchingConfig{ConcurrencyLimit: concurrencyLimit})
+	}
+}
+
+// qscOfPL returns a pointer to an appropriate QueuingConfig or nil
+// if no limiting is called for.  Returns nil and an error if the given
+// object is malformed in a way that is a problem for this package.
+func qscOfPL(qsf fq.QueueSetFactory, queues fq.QueueSet, pl *fctypesv1a1.PriorityLevelConfiguration, requestWaitLimit time.Duration) (fq.QueueSetCompleter, error) {
+	if (pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt) != (pl.Spec.Limited == nil) {
+		return nil, errors.New("broken union structure at the top")
+	}
+	if (pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt) != (pl.Name == fctypesv1a1.PriorityLevelConfigurationNameExempt) {
+		// This package does not attempt to cope with a priority level dynamically switching between exempt and not.
+		return nil, errors.New("non-alignment between name and type")
+	}
+	if pl.Spec.Limited == nil {
+		return nil, nil
+	}
+	if (pl.Spec.Limited.LimitResponse.Type == fctypesv1a1.LimitResponseTypeReject) != (pl.Spec.Limited.LimitResponse.Queuing == nil) {
+		return nil, errors.New("broken union structure for limit response")
+	}
+	qcAPI := pl.Spec.Limited.LimitResponse.Queuing
+	qcQS := fq.QueuingConfig{Name: pl.Name}
+	if qcAPI != nil {
+		qcQS = fq.QueuingConfig{Name: pl.Name,
+			DesiredNumQueues: int(qcAPI.Queues),
+			QueueLengthLimit: int(qcAPI.QueueLengthLimit),
+			HandSize:         int(qcAPI.HandSize),
+			RequestWaitLimit: requestWaitLimit,
+		}
+	}
+	var qsc fq.QueueSetCompleter
+	var err error
+	if queues != nil {
+		qsc, err = queues.BeginConfigChange(qcQS)
+	} else {
+		qsc, err = qsf.BeginConstruction(qcQS)
+	}
+	if err != nil {
+		err = errors.Wrap(err, fmt.Sprintf("priority level %q has QueuingConfiguration %#+v, which is invalid", pl.Name, qcAPI))
+	}
+	return qsc, err
+}
+
+func (meal *cfgMeal) presyncFlowSchemaStatus(fs *fctypesv1a1.FlowSchema, isDangling bool, plName string) {
+	danglingCondition := apihelpers.GetFlowSchemaConditionByType(fs, fctypesv1a1.FlowSchemaConditionDangling)
+	if danglingCondition == nil {
+		danglingCondition = &fctypesv1a1.FlowSchemaCondition{
+			Type: fctypesv1a1.FlowSchemaConditionDangling,
+		}
+	}
+	desiredStatus := fctypesv1a1.ConditionFalse
+	var desiredReason, desiredMessage string
+	if isDangling {
+		desiredStatus = fctypesv1a1.ConditionTrue
+		desiredReason = "NotFound"
+		desiredMessage = fmt.Sprintf("This FlowSchema references the PriorityLevelConfiguration object named %q but there is no such object", plName)
+	} else {
+		desiredReason = "Found"
+		desiredMessage = fmt.Sprintf("This FlowSchema references the PriorityLevelConfiguration object named %q and it exists", plName)
+	}
+	if danglingCondition.Status == desiredStatus && danglingCondition.Reason == desiredReason && danglingCondition.Message == desiredMessage {
+		return
+	}
+	meal.fsStatusUpdates = append(meal.fsStatusUpdates, fsStatusUpdate{
+		flowSchema: fs,
+		condition: fctypesv1a1.FlowSchemaCondition{
+			Type:               fctypesv1a1.FlowSchemaConditionDangling,
+			Status:             desiredStatus,
+			LastTransitionTime: metav1.Now(),
+			Reason:             desiredReason,
+			Message:            desiredMessage,
+		},
+		oldValue: *danglingCondition})
+}
+
+// imaginePL adds a priority level based on one of the mandatory ones
+func (meal *cfgMeal) imaginePL(proto *fctypesv1a1.PriorityLevelConfiguration, requestWaitLimit time.Duration) {
+	klog.V(3).Infof("No %s PriorityLevelConfiguration found, imagining one", proto.Name)
+	qsCompleter, err := qscOfPL(meal.cfgCtl.queueSetFactory, nil, proto, requestWaitLimit)
+	if err != nil {
+		// This can not happen because proto is one of the mandatory
+		// objects and these are not erroneous
+		panic(err)
+	}
+	meal.newPLStates[proto.Name] = &priorityLevelState{
+		pl:          proto,
+		qsCompleter: qsCompleter,
+	}
+	if proto.Spec.Limited != nil {
+		meal.shareSum += float64(proto.Spec.Limited.AssuredConcurrencyShares)
+	}
+	return
+}
+
+type immediateRequest struct{}
+
+func (immediateRequest) Finish(execute func()) bool {
+	execute()
+	return false
+}
+
+// startRequest classifies and, if appropriate, enqueues the request.
+// Returns a nil Request if and only if the request is to be rejected.
+// The returned bool indicates whether the request is exempt from
+// limitation.  The startWaitingTime is when the request started
+// waiting in its queue, or `Time{}` if this did not happen.
+func (cfgCtl *configController) startRequest(ctx context.Context, rd RequestDigest) (fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration, isExempt bool, req fq.Request, startWaitingTime time.Time) {
+	klog.V(7).Infof("startRequest(%#+v)", rd)
+	cfgCtl.lock.Lock()
+	defer cfgCtl.lock.Unlock()
+	for _, fs := range cfgCtl.flowSchemas {
+		if matchesFlowSchema(rd, fs) {
+			plName := fs.Spec.PriorityLevelConfiguration.Name
+			plState := cfgCtl.priorityLevelStates[plName]
+			if plState.pl.Spec.Type == fctypesv1a1.PriorityLevelEnablementExempt {
+				klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, immediate", rd, fs.Name, fs.Spec.DistinguisherMethod, plName)
+				return fs, plState.pl, true, immediateRequest{}, time.Time{}
+			}
+			var numQueues int32
+			if plState.pl.Spec.Limited.LimitResponse.Type == fctypesv1a1.LimitResponseTypeQueue {
+				numQueues = plState.pl.Spec.Limited.LimitResponse.Queuing.Queues
+
+			}
+			var hashValue uint64
+			if numQueues > 1 {
+				flowDistinguisher := computeFlowDistinguisher(rd, fs.Spec.DistinguisherMethod)
+				hashValue = hashFlowID(fs.Name, flowDistinguisher)
+			}
+			startWaitingTime = time.Now()
+			klog.V(7).Infof("startRequest(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, numQueues=%d", rd, fs.Name, fs.Spec.DistinguisherMethod, plName, numQueues)
+			req, idle := plState.queues.StartRequest(ctx, hashValue, fs.Name, rd.RequestInfo, rd.User)
+			if idle {
+				cfgCtl.maybeReapLocked(plName, plState)
+			}
+			return fs, plState.pl, false, req, startWaitingTime
+		}
+	}
+	// This can never happen because every configState has a
+	// FlowSchema that matches everything.  If somehow control reaches
+	// here, panic with some relevant information.
+	var catchAll *fctypesv1a1.FlowSchema
+	for _, fs := range cfgCtl.flowSchemas {
+		if fs.Name == fctypesv1a1.FlowSchemaNameCatchAll {
+			catchAll = fs
+		}
+	}
+	panic(fmt.Sprintf("No match; rd=%#+v, catchAll=%s", rd, fcfmt.Fmt(catchAll)))
+}
+
+// Call this after getting a clue that the given priority level is undesired and idle
+func (cfgCtl *configController) maybeReap(plName string) {
+	cfgCtl.lock.Lock()
+	defer cfgCtl.lock.Unlock()
+	plState := cfgCtl.priorityLevelStates[plName]
+	if plState == nil {
+		klog.V(7).Infof("plName=%s, plState==nil", plName)
+		return
+	}
+	if plState.queues != nil {
+		useless := plState.quiescing && plState.numPending == 0 && plState.queues.IsIdle()
+		klog.V(7).Infof("plState.quiescing=%v, plState.numPending=%d, useless=%v", plState.quiescing, plState.numPending, useless)
+		if !useless {
+			return
+		}
+	}
+	klog.V(3).Infof("Triggered API priority and fairness config reloading because priority level %s is undesired and idle", plName)
+	cfgCtl.configQueue.Add(0)
+}
+
+// Call this if both (1) plState.queues is non-nil and reported being
+// idle, and (2) cfgCtl's lock has not been released since then.
+func (cfgCtl *configController) maybeReapLocked(plName string, plState *priorityLevelState) {
+	if !(plState.quiescing && plState.numPending == 0) {
+		return
+	}
+	klog.V(3).Infof("Triggered API priority and fairness config reloading because priority level %s is undesired and idle", plName)
+	cfgCtl.configQueue.Add(0)
+}
+
+// computeFlowDistinguisher extracts the flow distinguisher according to the given method
+func computeFlowDistinguisher(rd RequestDigest, method *fctypesv1a1.FlowDistinguisherMethod) string {
+	if method == nil {
+		return ""
+	}
+	switch method.Type {
+	case fctypesv1a1.FlowDistinguisherMethodByUserType:
+		return rd.User.GetName()
+	case fctypesv1a1.FlowDistinguisherMethodByNamespaceType:
+		return rd.RequestInfo.Namespace
+	default:
+		// this line shall never reach
+		panic("invalid flow-distinguisher method")
+	}
+}
+
+func hashFlowID(fsName, fDistinguisher string) uint64 {
+	hash := sha256.New()
+	var sep = [1]byte{0}
+	hash.Write([]byte(fsName))
+	hash.Write(sep[:])
+	hash.Write([]byte(fDistinguisher))
+	var sum [32]byte
+	hash.Sum(sum[:0])
+	return binary.LittleEndian.Uint64(sum[:8])
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/apf_filter.go

@@ -0,0 +1,118 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flowcontrol
+
+import (
+	"context"
+	"strconv"
+	"time"
+
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apiserver/pkg/util/flowcontrol/counter"
+	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
+	fqs "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset"
+	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	kubeinformers "k8s.io/client-go/informers"
+	"k8s.io/klog"
+
+	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	fcclientv1a1 "k8s.io/client-go/kubernetes/typed/flowcontrol/v1alpha1"
+)
+
+// Interface defines how the API Priority and Fairness filter interacts with the underlying system.
+type Interface interface {
+	// Handle takes care of queuing and dispatching a request
+	// characterized by the given digest.  The given `noteFn` will be
+	// invoked with the results of request classification.  If Handle
+	// decides that the request should be executed then `execute()`
+	// will be invoked once to execute the request; otherwise
+	// `execute()` will not be invoked.
+	Handle(ctx context.Context,
+		requestDigest RequestDigest,
+		noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
+		execFn func(),
+	)
+
+	// Run monitors config objects from the main apiservers and causes
+	// any needed changes to local behavior.  This method ceases
+	// activity and returns after the given channel is closed.
+	Run(stopCh <-chan struct{}) error
+}
+
+// This request filter implements https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md
+
+// New creates a new instance to implement API priority and fairness
+func New(
+	informerFactory kubeinformers.SharedInformerFactory,
+	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	serverConcurrencyLimit int,
+	requestWaitLimit time.Duration,
+) Interface {
+	grc := counter.NoOp{}
+	return NewTestable(
+		informerFactory,
+		flowcontrolClient,
+		serverConcurrencyLimit,
+		requestWaitLimit,
+		fqs.NewQueueSetFactory(&clock.RealClock{}, grc),
+	)
+}
+
+// NewTestable is extra flexible to facilitate testing
+func NewTestable(
+	informerFactory kubeinformers.SharedInformerFactory,
+	flowcontrolClient fcclientv1a1.FlowcontrolV1alpha1Interface,
+	serverConcurrencyLimit int,
+	requestWaitLimit time.Duration,
+	queueSetFactory fq.QueueSetFactory,
+) Interface {
+	return newTestableController(informerFactory, flowcontrolClient, serverConcurrencyLimit, requestWaitLimit, queueSetFactory)
+}
+
+func (cfgCtl *configController) Handle(ctx context.Context, requestDigest RequestDigest,
+	noteFn func(fs *fctypesv1a1.FlowSchema, pl *fctypesv1a1.PriorityLevelConfiguration),
+	execFn func()) {
+	fs, pl, isExempt, req, startWaitingTime := cfgCtl.startRequest(ctx, requestDigest)
+	queued := startWaitingTime != time.Time{}
+	noteFn(fs, pl)
+	if req == nil {
+		if queued {
+			metrics.ObserveWaitingDuration(pl.Name, fs.Name, strconv.FormatBool(req != nil), time.Since(startWaitingTime))
+		}
+		klog.V(7).Infof("Handle(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, isExempt=%v, reject", requestDigest, fs.Name, fs.Spec.DistinguisherMethod, pl.Name, isExempt)
+		return
+	}
+	klog.V(7).Infof("Handle(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, isExempt=%v, queued=%v", requestDigest, fs.Name, fs.Spec.DistinguisherMethod, pl.Name, isExempt, queued)
+	var executed bool
+	idle := req.Finish(func() {
+		if queued {
+			metrics.ObserveWaitingDuration(pl.Name, fs.Name, strconv.FormatBool(req != nil), time.Since(startWaitingTime))
+		}
+		metrics.AddDispatch(pl.Name, fs.Name)
+		executed = true
+		startExecutionTime := time.Now()
+		execFn()
+		metrics.ObserveExecutionDuration(pl.Name, fs.Name, time.Since(startExecutionTime))
+	})
+	if queued && !executed {
+		metrics.ObserveWaitingDuration(pl.Name, fs.Name, strconv.FormatBool(req != nil), time.Since(startWaitingTime))
+	}
+	klog.V(7).Infof("Handle(%#+v) => fsName=%q, distMethod=%#+v, plName=%q, isExempt=%v, queued=%v, Finish() => idle=%v", requestDigest, fs.Name, fs.Spec.DistinguisherMethod, pl.Name, isExempt, queued, idle)
+	if idle {
+		cfgCtl.maybeReap(pl.Name)
+	}
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/counter/interface.go

@@ -0,0 +1,33 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package counter
+
+// GoRoutineCounter keeps track of the number of active goroutines
+// working on/for something.  This is a utility that makes such code more
+// testable.  The code uses this utility to report the number of active
+// goroutines to the test code, so that the test code can advance a fake
+// clock when and only when the code being tested has finished all
+// the work that is ready to do at the present time.
+type GoRoutineCounter interface {
+	// Add adds the given delta to the count of active goroutines.
+	// Call Add(1) before forking a goroutine, Add(-1) at the end of that goroutine.
+	// Call Add(-1) just before waiting on something from another goroutine (e.g.,
+	// just before a `select`).
+	// Call Add(1) just before doing something that unblocks a goroutine that is
+	// waiting on that something.
+	Add(delta int)
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/counter/noop.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package counter
+
+// NoOp is a GoRoutineCounter that does not actually count
+type NoOp struct{}
+
+var _ GoRoutineCounter = NoOp{}
+
+// Add would adjust the count, if a count were being kept
+func (NoOp) Add(int) {}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/interface.go

@@ -0,0 +1,125 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package fairqueuing
+
+import (
+	"context"
+	"time"
+)
+
+// QueueSetFactory is used to create QueueSet objects.  Creation, like
+// config update, is done in two phases: the first phase consumes the
+// QueuingConfig and the second consumes the DispatchingConfig.  They
+// are separated so that errors from the first phase can be found
+// before committing to a concurrency allotment for the second.
+type QueueSetFactory interface {
+	// BeginConstruction does the first phase of creating a QueueSet
+	BeginConstruction(QueuingConfig) (QueueSetCompleter, error)
+}
+
+// QueueSetCompleter finishes the two-step process of creating or
+// reconfiguring a QueueSet
+type QueueSetCompleter interface {
+	// Complete returns a QueueSet configured by the given
+	// dispatching configuration.
+	Complete(DispatchingConfig) QueueSet
+}
+
+// QueueSet is the abstraction for the queuing and dispatching
+// functionality of one non-exempt priority level.  It covers the
+// functionality described in the "Assignment to a Queue", "Queuing",
+// and "Dispatching" sections of
+// https://github.com/kubernetes/enhancements/blob/master/keps/sig-api-machinery/20190228-priority-and-fairness.md
+// .  Some day we may have connections between priority levels, but
+// today is not that day.
+type QueueSet interface {
+	// BeginConfigChange starts the two-step process of updating the
+	// configuration.  No change is made until Complete is called.  If
+	// `C := X.BeginConstruction(q)` then `C.Complete(d)` returns the
+	// same value `X`.  If the QueuingConfig's DesiredNumQueues field
+	// is zero then the other queuing-specific config parameters are
+	// not changed, so that the queues continue draining as before.
+	// In any case, reconfiguration does not discard any queue unless
+	// and until it is undesired and empty.
+	BeginConfigChange(QueuingConfig) (QueueSetCompleter, error)
+
+	// IsIdle returns a bool indicating whether the QueueSet was idle
+	// at the moment of the return.  Idle means the QueueSet has zero
+	// requests queued and zero executing.  This bit can change only
+	// (1) during a call to StartRequest and (2) during a call to
+	// Request::Finish.  In the latter case idleness can only change
+	// from false to true.
+	IsIdle() bool
+
+	// StartRequest begins the process of handling a request.  If the
+	// request gets queued and the number of queues is greater than
+	// 1 then Wait uses the given hashValue as the source of entropy
+	// as it shuffle-shards the request into a queue.  The descr1 and
+	// descr2 values play no role in the logic but appear in log
+	// messages.  This method always returns quickly (without waiting
+	// for the request to be dequeued).  If this method returns a nil
+	// Request value then caller should reject the request and the
+	// returned bool indicates whether the QueueSet was idle at the
+	// moment of the return.  Otherwise idle==false and the client
+	// must call the Wait method of the Request exactly once.
+	StartRequest(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) (req Request, idle bool)
+}
+
+// Request represents the remainder of the handling of one request
+type Request interface {
+	// Finish determines whether to execute or reject the request and
+	// invokes `execute` if the decision is to execute the request.
+	// The returned `idle bool` value indicates whether the QueueSet
+	// was idle when the value was calculated, but might no longer be
+	// accurate by the time the client examines that value.
+	Finish(execute func()) (idle bool)
+}
+
+// QueuingConfig defines the configuration of the queuing aspect of a QueueSet.
+type QueuingConfig struct {
+	// Name is used to identify a queue set, allowing for descriptive information about its intended use
+	Name string
+
+	// DesiredNumQueues is the number of queues that the API says
+	// should exist now.  This may be zero, in which case
+	// QueueLengthLimit, HandSize, and RequestWaitLimit are ignored.
+	DesiredNumQueues int
+
+	// QueueLengthLimit is the maximum number of requests that may be waiting in a given queue at a time
+	QueueLengthLimit int
+
+	// HandSize is a parameter of shuffle sharding.  Upon arrival of a request, a queue is chosen by randomly
+	// dealing a "hand" of this many queues and then picking one of minimum length.
+	HandSize int
+
+	// RequestWaitLimit is the maximum amount of time that a request may wait in a queue.
+	// If, by the end of that time, the request has not been dispatched then it is rejected.
+	RequestWaitLimit time.Duration
+}
+
+// DispatchingConfig defines the configuration of the dispatching aspect of a QueueSet.
+type DispatchingConfig struct {
+	// ConcurrencyLimit is the maximum number of requests of this QueueSet that may be executing at a time
+	ConcurrencyLimit int
+}
+
+// EmptyHandler is used to notify the callee when all the queues
+// of a QueueSet have been drained.
+type EmptyHandler interface {
+	// HandleEmpty is called to deliver the notification
+	HandleEmpty()
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise/interface.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package promise
+
+// This file defines interfaces for promises and futures and related
+// things.  These are about coordination among multiple goroutines and
+// so are safe for concurrent calls --- although moderated in some
+// cases by a requirement that the caller hold a certain lock.
+
+// Readable represents a variable that is initially not set and later
+// becomes set.  Some instances may be set to multiple values in
+// series.  A Readable for a variable that can only get one value is
+// commonly known as a "future".
+type Readable interface {
+	// Get reads the current value of this variable.  If this variable
+	// is not set yet then this call blocks until this variable gets a
+	// value.
+	Get() interface{}
+
+	// IsSet returns immediately with an indication of whether this
+	// variable has been set.
+	IsSet() bool
+}
+
+// LockingReadable is a Readable whose implementation is protected by
+// a lock
+type LockingReadable interface {
+	Readable
+
+	// GetLocked is like Get but the caller must already hold the
+	// lock.  GetLocked may release, and later re-acquire, the lock
+	// any number of times.  Get may acquire, and later release, the
+	// lock any number of times.
+	GetLocked() interface{}
+
+	// IsSetLocked is like IsSet but the caller must already hold the
+	// lock.  IsSetLocked may release, and later re-acquire, the lock
+	// any number of times.  IsSet may acquire, and later release, the
+	// lock any number of times.
+	IsSetLocked() bool
+}
+
+// WriteOnceOnly represents a variable that is initially not set and
+// can be set once.
+type WriteOnceOnly interface {
+	// Set normally writes a value into this variable, unblocks every
+	// goroutine waiting for this variable to have a value, and
+	// returns true.  In the unhappy case that this variable is
+	// already set, this method returns false without modifying the
+	// variable's value.
+	Set(interface{}) bool
+}
+
+// WriteOnce represents a variable that is initially not set and can
+// be set once and is readable.  This is the common meaning for
+// "promise".
+type WriteOnce interface {
+	Readable
+	WriteOnceOnly
+}
+
+// LockingWriteOnceOnly is a WriteOnceOnly whose implementation is
+// protected by a lock.
+type LockingWriteOnceOnly interface {
+	WriteOnceOnly
+
+	// SetLocked is like Set but the caller must already hold the
+	// lock.  SetLocked may release, and later re-acquire, the lock
+	// any number of times.  Set may acquire, and later release, the
+	// lock any number of times
+	SetLocked(interface{}) bool
+}
+
+// LockingWriteOnce is a WriteOnce whose implementation is protected
+// by a lock.
+type LockingWriteOnce interface {
+	LockingReadable
+	LockingWriteOnceOnly
+}
+
+// WriteMultipleOnly represents a variable that is initially not set
+// and can be set one or more times (unlike a traditional "promise",
+// which can be written only once).
+type WriteMultipleOnly interface {
+	// Set writes a value into this variable and unblocks every
+	// goroutine waiting for this variable to have a value
+	Set(interface{})
+}
+
+// WriteMultiple represents a variable that is initially not set and
+// can be set one or more times (unlike a traditional "promise", which
+// can be written only once) and is readable.
+type WriteMultiple interface {
+	Readable
+	WriteMultipleOnly
+}
+
+// LockingWriteMultipleOnly is a WriteMultipleOnly whose
+// implementation is protected by a lock.
+type LockingWriteMultipleOnly interface {
+	WriteMultipleOnly
+
+	// SetLocked is like Set but the caller must already hold the
+	// lock.  SetLocked may release, and later re-acquire, the lock
+	// any number of times.  Set may acquire, and later release, the
+	// lock any number of times
+	SetLocked(interface{})
+}
+
+// LockingWriteMultiple is a WriteMultiple whose implementation is
+// protected by a lock.
+type LockingWriteMultiple interface {
+	LockingReadable
+	LockingWriteMultipleOnly
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise/lockingpromise/lockingpromise.go

@@ -0,0 +1,124 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package lockingpromise
+
+import (
+	"sync"
+
+	"k8s.io/apiserver/pkg/util/flowcontrol/counter"
+	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise"
+)
+
+// promisoid is the data and behavior common to all the promise-like
+// abstractions implemented here.  This implementation is based on a
+// condition variable.  This implementation tracks active goroutines:
+// the given counter is decremented for a goroutine waiting for this
+// varible to be set and incremented when such a goroutine is
+// unblocked.
+type promisoid struct {
+	lock          sync.Locker
+	cond          sync.Cond
+	activeCounter counter.GoRoutineCounter // counter of active goroutines
+	waitingCount  int                      // number of goroutines idle due to this being unset
+	isSet         bool
+	value         interface{}
+}
+
+func (pr *promisoid) Get() interface{} {
+	pr.lock.Lock()
+	defer pr.lock.Unlock()
+	return pr.GetLocked()
+}
+
+func (pr *promisoid) GetLocked() interface{} {
+	if !pr.isSet {
+		pr.waitingCount++
+		pr.activeCounter.Add(-1)
+		pr.cond.Wait()
+	}
+	return pr.value
+}
+
+func (pr *promisoid) IsSet() bool {
+	pr.lock.Lock()
+	defer pr.lock.Unlock()
+	return pr.IsSetLocked()
+}
+
+func (pr *promisoid) IsSetLocked() bool {
+	return pr.isSet
+}
+
+func (pr *promisoid) SetLocked(value interface{}) {
+	pr.isSet = true
+	pr.value = value
+	if pr.waitingCount > 0 {
+		pr.activeCounter.Add(pr.waitingCount)
+		pr.waitingCount = 0
+		pr.cond.Broadcast()
+	}
+}
+
+type writeOnce struct {
+	promisoid
+}
+
+var _ promise.LockingWriteOnce = &writeOnce{}
+
+// NewWriteOnce makes a new promise.LockingWriteOnce
+func NewWriteOnce(lock sync.Locker, activeCounter counter.GoRoutineCounter) promise.LockingWriteOnce {
+	return &writeOnce{promisoid{
+		lock:          lock,
+		cond:          *sync.NewCond(lock),
+		activeCounter: activeCounter,
+	}}
+}
+
+func (wr *writeOnce) Set(value interface{}) bool {
+	wr.lock.Lock()
+	defer wr.lock.Unlock()
+	return wr.SetLocked(value)
+}
+
+func (wr *writeOnce) SetLocked(value interface{}) bool {
+	if wr.isSet {
+		return false
+	}
+	wr.promisoid.SetLocked(value)
+	return true
+}
+
+type writeMultiple struct {
+	promisoid
+}
+
+var _ promise.LockingWriteMultiple = &writeMultiple{}
+
+// NewWriteMultiple makes a new promise.LockingWriteMultiple
+func NewWriteMultiple(lock sync.Locker, activeCounter counter.GoRoutineCounter) promise.LockingWriteMultiple {
+	return &writeMultiple{promisoid{
+		lock:          lock,
+		cond:          *sync.NewCond(lock),
+		activeCounter: activeCounter,
+	}}
+}
+
+func (wr *writeMultiple) Set(value interface{}) {
+	wr.lock.Lock()
+	defer wr.lock.Unlock()
+	wr.SetLocked(value)
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/doc.go

@@ -0,0 +1,120 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package queueset implements a technique called "fair queuing for
+// server requests".  One QueueSet is a set of queues operating
+// according to this technique.
+//
+// Fair queuing for server requests is inspired by the fair queuing
+// technique from the world of networking.  You can find a good paper
+// on that at https://dl.acm.org/citation.cfm?doid=75247.75248 or
+// http://people.csail.mit.edu/imcgraw/links/research/pubs/networks/WFQ.pdf
+// and there is an implementation outline in the Wikipedia article at
+// https://en.wikipedia.org/wiki/Fair_queuing .
+//
+// Fair queuing for server requests differs from traditional fair
+// queuing in three ways: (1) we are dispatching application layer
+// requests to a server rather than transmitting packets on a network
+// link, (2) multiple requests can be executing at once, and (3) the
+// service time (execution duration) is not known until the execution
+// completes.
+//
+// The first two differences can easily be handled by straightforward
+// adaptation of the concept called "R(t)" in the original paper and
+// "virtual time" in the implementation outline. In that
+// implementation outline, the notation now() is used to mean reading
+// the virtual clock. In the original paper’s terms, "R(t)" is the
+// number of "rounds" that have been completed at real time t ---
+// where a round consists of virtually transmitting one bit from every
+// non-empty queue in the router (regardless of which queue holds the
+// packet that is really being transmitted at the moment); in this
+// conception, a packet is considered to be "in" its queue until the
+// packet’s transmission is finished. For our problem, we can define a
+// round to be giving one nanosecond of CPU to every non-empty queue
+// in the apiserver (where emptiness is judged based on both queued
+// and executing requests from that queue), and define R(t) = (server
+// start time) + (1 ns) * (number of rounds since server start). Let
+// us write NEQ(t) for that number of non-empty queues in the
+// apiserver at time t.  Let us also write C for the concurrency
+// limit.  In the original paper, the partial derivative of R(t) with
+// respect to t is
+//
+//                 1 / NEQ(t) .
+//
+// To generalize from transmitting one packet at a time to executing C
+// requests at a time, that derivative becomes
+//
+//                C / NEQ(t) .
+//
+// However, sometimes there are fewer than C requests available to
+// execute.  For a given queue "q", let us also write "reqs(q, t)" for
+// the number of requests of that queue that are executing at that
+// time.  The total number of requests executing is sum[over q]
+// reqs(q, t) and if that is less than C then virtual time is not
+// advancing as fast as it would if all C seats were occupied; in this
+// case the numerator of the quotient in that derivative should be
+// adjusted proportionally.  Putting it all together for fair queing
+// for server requests: at a particular time t, the partial derivative
+// of R(t) with respect to t is
+//
+//      min( C, sum[over q] reqs(q, t) ) / NEQ(t) .
+//
+// In terms of the implementation outline, this is the rate at which
+// virtual time is advancing at time t (in virtual nanoseconds per
+// real nanosecond). Where the networking implementation outline adds
+// packet size to a virtual time, in our version this corresponds to
+// adding a service time (i.e., duration) to virtual time.
+//
+// The third difference is handled by modifying the algorithm to
+// dispatch based on an initial guess at the request’s service time
+// (duration) and then make the corresponding adjustments once the
+// request’s actual service time is known. This is similar, although
+// not exactly isomorphic, to the original paper’s adjustment by
+// `$\delta$` for the sake of promptness.
+//
+// For implementation simplicity (see below), let us use the same
+// initial service time guess for every request; call that duration
+// G. A good choice might be the service time limit (1
+// minute). Different guesses will give slightly different dynamics,
+// but any positive number can be used for G without ruining the
+// long-term behavior.
+//
+// As in ordinary fair queuing, there is a bound on divergence from
+// the ideal. In plain fair queuing the bound is one packet; in our
+// version it is C requests.
+//
+// To support efficiently making the necessary adjustments once a
+// request’s actual service time is known, the virtual finish time of
+// a request and the last virtual finish time of a queue are not
+// represented directly but instead computed from queue length,
+// request position in the queue, and an alternate state variable that
+// holds the queue’s virtual start time. While the queue is empty and
+// has no requests executing: the value of its virtual start time
+// variable is ignored and its last virtual finish time is considered
+// to be in the virtual past. When a request arrives to an empty queue
+// with no requests executing, the queue’s virtual start time is set
+// to the current virtual time. The virtual finish time of request
+// number J in the queue (counting from J=1 for the head) is J * G +
+// (queue's virtual start time). While the queue is non-empty: the
+// last virtual finish time of the queue is the virtual finish time of
+// the last request in the queue. While the queue is empty and has a
+// request executing: the last virtual finish time is the queue’s
+// virtual start time. When a request is dequeued for service the
+// queue’s virtual start time is advanced by G. When a request
+// finishes being served, and the actual service time was S, the
+// queue’s virtual start time is decremented by G - S.
+//
+package queueset

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/queueset.go

@@ -0,0 +1,711 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package queueset
+
+import (
+	"context"
+	"fmt"
+	"math"
+	"sync"
+	"time"
+
+	"github.com/pkg/errors"
+
+	"k8s.io/apimachinery/pkg/util/clock"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apiserver/pkg/util/flowcontrol/counter"
+	fq "k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing"
+	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise/lockingpromise"
+	"k8s.io/apiserver/pkg/util/flowcontrol/metrics"
+	"k8s.io/apiserver/pkg/util/shufflesharding"
+	"k8s.io/klog"
+)
+
+const nsTimeFmt = "2006-01-02 15:04:05.000000000"
+
+// queueSetFactory implements the QueueSetFactory interface
+// queueSetFactory makes QueueSet objects.
+type queueSetFactory struct {
+	counter counter.GoRoutineCounter
+	clock   clock.PassiveClock
+}
+
+// `*queueSetCompleter` implements QueueSetCompleter.  Exactly one of
+// the fields `factory` and `theSet` is non-nil.
+type queueSetCompleter struct {
+	factory *queueSetFactory
+	theSet  *queueSet
+	qCfg    fq.QueuingConfig
+	dealer  *shufflesharding.Dealer
+}
+
+// queueSet implements the Fair Queuing for Server Requests technique
+// described in this package's doc, and a pointer to one implements
+// the QueueSet interface.  The clock, GoRoutineCounter, and estimated
+// service time should not be changed; the fields listed after the
+// lock must be accessed only while holding the lock.  The methods of
+// this type follow the naming convention that the suffix "Locked"
+// means the caller must hold the lock; for a method whose name does
+// not end in "Locked" either acquires the lock or does not care about
+// locking.
+type queueSet struct {
+	clock                clock.PassiveClock
+	counter              counter.GoRoutineCounter
+	estimatedServiceTime float64
+
+	lock sync.Mutex
+
+	// qCfg holds the current queuing configuration.  Its
+	// DesiredNumQueues may be less than the current number of queues.
+	// If its DesiredNumQueues is zero then its other queuing
+	// parameters retain the settings they had when DesiredNumQueues
+	// was last non-zero (if ever).
+	qCfg fq.QueuingConfig
+
+	// the current dispatching configuration.
+	dCfg fq.DispatchingConfig
+
+	// If `config.DesiredNumQueues` is non-zero then dealer is not nil
+	// and is good for `config`.
+	dealer *shufflesharding.Dealer
+
+	// queues may be longer than the desired number, while the excess
+	// queues are still draining.
+	queues []*queue
+
+	// virtualTime is the number of virtual seconds since process startup
+	virtualTime float64
+
+	// lastRealTime is what `clock.Now()` yielded when `virtualTime` was last updated
+	lastRealTime time.Time
+
+	// robinIndex is the index of the last queue dispatched
+	robinIndex int
+
+	// totRequestsWaiting is the sum, over all the queues, of the
+	// number of requests waiting in that queue
+	totRequestsWaiting int
+
+	// totRequestsExecuting is the total number of requests of this
+	// queueSet that are currently executing.  That is the same as the
+	// sum, over all the queues, of the number of requests executing
+	// from that queue.
+	totRequestsExecuting int
+}
+
+// NewQueueSetFactory creates a new QueueSetFactory object
+func NewQueueSetFactory(c clock.PassiveClock, counter counter.GoRoutineCounter) fq.QueueSetFactory {
+	return &queueSetFactory{
+		counter: counter,
+		clock:   c,
+	}
+}
+
+func (qsf *queueSetFactory) BeginConstruction(qCfg fq.QueuingConfig) (fq.QueueSetCompleter, error) {
+	dealer, err := checkConfig(qCfg)
+	if err != nil {
+		return nil, err
+	}
+	return &queueSetCompleter{
+		factory: qsf,
+		qCfg:    qCfg,
+		dealer:  dealer}, nil
+}
+
+// checkConfig returns a non-nil Dealer if the config is valid and
+// calls for one, and returns a non-nil error if the given config is
+// invalid.
+func checkConfig(qCfg fq.QueuingConfig) (*shufflesharding.Dealer, error) {
+	if qCfg.DesiredNumQueues == 0 {
+		return nil, nil
+	}
+	dealer, err := shufflesharding.NewDealer(qCfg.DesiredNumQueues, qCfg.HandSize)
+	if err != nil {
+		err = errors.Wrap(err, "the QueueSetConfig implies an invalid shuffle sharding config (DesiredNumQueues is deckSize)")
+	}
+	return dealer, err
+}
+
+func (qsc *queueSetCompleter) Complete(dCfg fq.DispatchingConfig) fq.QueueSet {
+	qs := qsc.theSet
+	if qs == nil {
+		qs = &queueSet{
+			clock:                qsc.factory.clock,
+			counter:              qsc.factory.counter,
+			estimatedServiceTime: 60,
+			qCfg:                 qsc.qCfg,
+			virtualTime:          0,
+			lastRealTime:         qsc.factory.clock.Now(),
+		}
+	}
+	qs.setConfiguration(qsc.qCfg, qsc.dealer, dCfg)
+	return qs
+}
+
+// createQueues is a helper method for initializing an array of n queues
+func createQueues(n, baseIndex int) []*queue {
+	fqqueues := make([]*queue, n)
+	for i := 0; i < n; i++ {
+		fqqueues[i] = &queue{index: baseIndex + i, requests: make([]*request, 0)}
+	}
+	return fqqueues
+}
+
+func (qs *queueSet) BeginConfigChange(qCfg fq.QueuingConfig) (fq.QueueSetCompleter, error) {
+	dealer, err := checkConfig(qCfg)
+	if err != nil {
+		return nil, err
+	}
+	return &queueSetCompleter{
+		theSet: qs,
+		qCfg:   qCfg,
+		dealer: dealer}, nil
+}
+
+// SetConfiguration is used to set the configuration for a queueSet.
+// Update handling for when fields are updated is handled here as well -
+// eg: if DesiredNum is increased, SetConfiguration reconciles by
+// adding more queues.
+func (qs *queueSet) setConfiguration(qCfg fq.QueuingConfig, dealer *shufflesharding.Dealer, dCfg fq.DispatchingConfig) {
+	qs.lockAndSyncTime()
+	defer qs.lock.Unlock()
+
+	if qCfg.DesiredNumQueues > 0 {
+		// Adding queues is the only thing that requires immediate action
+		// Removing queues is handled by omitting indexes >DesiredNum from
+		// chooseQueueIndexLocked
+		numQueues := len(qs.queues)
+		if qCfg.DesiredNumQueues > numQueues {
+			qs.queues = append(qs.queues,
+				createQueues(qCfg.DesiredNumQueues-numQueues, len(qs.queues))...)
+		}
+	} else {
+		qCfg.QueueLengthLimit = qs.qCfg.QueueLengthLimit
+		qCfg.HandSize = qs.qCfg.HandSize
+		qCfg.RequestWaitLimit = qs.qCfg.RequestWaitLimit
+	}
+
+	qs.qCfg = qCfg
+	qs.dCfg = dCfg
+	qs.dealer = dealer
+
+	qs.dispatchAsMuchAsPossibleLocked()
+}
+
+// A decision about a request
+type requestDecision int
+
+// Values passed through a request's decision
+const (
+	decisionExecute requestDecision = iota
+	decisionReject
+	decisionCancel
+)
+
+// StartRequest begins the process of handling a request.  We take the
+// approach of updating the metrics about total requests queued and
+// executing at each point where there is a change in that quantity,
+// because the metrics --- and only the metrics --- track that
+// quantity per FlowSchema.
+func (qs *queueSet) StartRequest(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) (fq.Request, bool) {
+	qs.lockAndSyncTime()
+	defer qs.lock.Unlock()
+	var req *request
+
+	// ========================================================================
+	// Step 0:
+	// Apply only concurrency limit, if zero queues desired
+	if qs.qCfg.DesiredNumQueues < 1 {
+		if qs.totRequestsExecuting >= qs.dCfg.ConcurrencyLimit {
+			klog.V(5).Infof("QS(%s): rejecting request %q %#+v %#+v because %d are executing and the limit is %d", qs.qCfg.Name, fsName, descr1, descr2, qs.totRequestsExecuting, qs.dCfg.ConcurrencyLimit)
+			metrics.AddReject(qs.qCfg.Name, fsName, "concurrency-limit")
+			return nil, qs.isIdleLocked()
+		}
+		req = qs.dispatchSansQueueLocked(ctx, fsName, descr1, descr2)
+		return req, false
+	}
+
+	// ========================================================================
+	// Step 1:
+	// 1) Start with shuffle sharding, to pick a queue.
+	// 2) Reject old requests that have been waiting too long
+	// 3) Reject current request if there is not enough concurrency shares and
+	// we are at max queue length
+	// 4) If not rejected, create a request and enqueue
+	req = qs.timeoutOldRequestsAndRejectOrEnqueueLocked(ctx, hashValue, fsName, descr1, descr2)
+	// req == nil means that the request was rejected - no remaining
+	// concurrency shares and at max queue length already
+	if req == nil {
+		klog.V(5).Infof("QS(%s): rejecting request %q %#+v %#+v due to queue full", qs.qCfg.Name, fsName, descr1, descr2)
+		metrics.AddReject(qs.qCfg.Name, fsName, "queue-full")
+		return nil, qs.isIdleLocked()
+	}
+
+	// ========================================================================
+	// Step 2:
+	// The next step is to invoke the method that dequeues as much
+	// as possible.
+	// This method runs a loop, as long as there are non-empty
+	// queues and the number currently executing is less than the
+	// assured concurrency value.  The body of the loop uses the
+	// fair queuing technique to pick a queue and dispatch a
+	// request from that queue.
+	qs.dispatchAsMuchAsPossibleLocked()
+
+	// ========================================================================
+	// Step 3:
+
+	// Set up a relay from the context's Done channel to the world
+	// of well-counted goroutines. We Are Told that every
+	// request's context's Done channel gets closed by the time
+	// the request is done being processed.
+	doneCh := ctx.Done()
+	if doneCh != nil {
+		qs.preCreateOrUnblockGoroutine()
+		go func() {
+			defer runtime.HandleCrash()
+			qs.goroutineDoneOrBlocked()
+			_ = <-doneCh
+			// Whatever goroutine unblocked the preceding receive MUST
+			// have already either (a) incremented qs.counter or (b)
+			// known that said counter is not actually counting or (c)
+			// known that the count does not need to be accurate.
+			// BTW, the count only needs to be accurate in a test that
+			// uses FakeEventClock::Run().
+			klog.V(6).Infof("QS(%s): Context of request %q %#+v %#+v is Done", qs.qCfg.Name, fsName, descr1, descr2)
+			qs.cancelWait(req)
+			qs.goroutineDoneOrBlocked()
+		}()
+	}
+	return req, false
+}
+
+func (req *request) Finish(execFn func()) bool {
+	exec, idle := req.wait()
+	if !exec {
+		return idle
+	}
+	execFn()
+	return req.qs.finishRequestAndDispatchAsMuchAsPossible(req)
+}
+
+func (req *request) wait() (bool, bool) {
+	qs := req.qs
+	qs.lock.Lock()
+	defer qs.lock.Unlock()
+	if req.waitStarted {
+		// This can not happen, because the client is forbidden to
+		// call Wait twice on the same request
+		panic(fmt.Sprintf("Multiple calls to the Wait method, QueueSet=%s, startTime=%s, descr1=%#+v, descr2=%#+v", req.qs.qCfg.Name, req.startTime, req.descr1, req.descr2))
+	}
+	req.waitStarted = true
+
+	// ========================================================================
+	// Step 4:
+	// The final step is to wait on a decision from
+	// somewhere and then act on it.
+	decisionAny := req.decision.GetLocked()
+	qs.syncTimeLocked()
+	decision, isDecision := decisionAny.(requestDecision)
+	if !isDecision {
+		panic(fmt.Sprintf("QS(%s): Impossible decision %#+v (of type %T) for request %#+v %#+v", qs.qCfg.Name, decisionAny, decisionAny, req.descr1, req.descr2))
+	}
+	switch decision {
+	case decisionReject:
+		klog.V(5).Infof("QS(%s): request %#+v %#+v timed out after being enqueued\n", qs.qCfg.Name, req.descr1, req.descr2)
+		metrics.AddReject(qs.qCfg.Name, req.fsName, "time-out")
+		return false, qs.isIdleLocked()
+	case decisionCancel:
+		// TODO(aaron-prindle) add metrics for this case
+		klog.V(5).Infof("QS(%s): Ejecting request %#+v %#+v from its queue", qs.qCfg.Name, req.descr1, req.descr2)
+		return false, qs.isIdleLocked()
+	case decisionExecute:
+		klog.V(5).Infof("QS(%s): Dispatching request %#+v %#+v from its queue", qs.qCfg.Name, req.descr1, req.descr2)
+		return true, false
+	default:
+		// This can not happen, all possible values are handled above
+		panic(decision)
+	}
+}
+
+func (qs *queueSet) IsIdle() bool {
+	qs.lock.Lock()
+	defer qs.lock.Unlock()
+	return qs.isIdleLocked()
+}
+
+func (qs *queueSet) isIdleLocked() bool {
+	return qs.totRequestsWaiting == 0 && qs.totRequestsExecuting == 0
+}
+
+// lockAndSyncTime acquires the lock and updates the virtual time.
+// Doing them together avoids the mistake of modify some queue state
+// before calling syncTimeLocked.
+func (qs *queueSet) lockAndSyncTime() {
+	qs.lock.Lock()
+	qs.syncTimeLocked()
+}
+
+// syncTimeLocked updates the virtual time based on the assumption
+// that the current state of the queues has been in effect since
+// `qs.lastRealTime`.  Thus, it should be invoked after acquiring the
+// lock and before modifying the state of any queue.
+func (qs *queueSet) syncTimeLocked() {
+	realNow := qs.clock.Now()
+	timeSinceLast := realNow.Sub(qs.lastRealTime).Seconds()
+	qs.lastRealTime = realNow
+	qs.virtualTime += timeSinceLast * qs.getVirtualTimeRatioLocked()
+}
+
+// getVirtualTimeRatio calculates the rate at which virtual time has
+// been advancing, according to the logic in `doc.go`.
+func (qs *queueSet) getVirtualTimeRatioLocked() float64 {
+	activeQueues := 0
+	reqs := 0
+	for _, queue := range qs.queues {
+		reqs += queue.requestsExecuting
+		if len(queue.requests) > 0 || queue.requestsExecuting > 0 {
+			activeQueues++
+		}
+	}
+	if activeQueues == 0 {
+		return 0
+	}
+	return math.Min(float64(reqs), float64(qs.dCfg.ConcurrencyLimit)) / float64(activeQueues)
+}
+
+// timeoutOldRequestsAndRejectOrEnqueueLocked encapsulates the logic required
+// to validate and enqueue a request for the queueSet/QueueSet:
+// 1) Start with shuffle sharding, to pick a queue.
+// 2) Reject old requests that have been waiting too long
+// 3) Reject current request if there is not enough concurrency shares and
+// we are at max queue length
+// 4) If not rejected, create a request and enqueue
+// returns the enqueud request on a successful enqueue
+// returns nil in the case that there is no available concurrency or
+// the queuelengthlimit has been reached
+func (qs *queueSet) timeoutOldRequestsAndRejectOrEnqueueLocked(ctx context.Context, hashValue uint64, fsName string, descr1, descr2 interface{}) *request {
+	//	Start with the shuffle sharding, to pick a queue.
+	queueIdx := qs.chooseQueueIndexLocked(hashValue, descr1, descr2)
+	queue := qs.queues[queueIdx]
+	// The next step is the logic to reject requests that have been waiting too long
+	qs.removeTimedOutRequestsFromQueueLocked(queue, fsName)
+	// NOTE: currently timeout is only checked for each new request.  This means that there can be
+	// requests that are in the queue longer than the timeout if there are no new requests
+	// We prefer the simplicity over the promptness, at least for now.
+
+	// Create a request and enqueue
+	req := &request{
+		qs:          qs,
+		fsName:      fsName,
+		ctx:         ctx,
+		decision:    lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
+		arrivalTime: qs.clock.Now(),
+		queue:       queue,
+		descr1:      descr1,
+		descr2:      descr2,
+	}
+	if ok := qs.rejectOrEnqueueLocked(req); !ok {
+		return nil
+	}
+	metrics.ObserveQueueLength(qs.qCfg.Name, fsName, len(queue.requests))
+	return req
+}
+
+// chooseQueueIndexLocked uses shuffle sharding to select a queue index
+// using the given hashValue and the shuffle sharding parameters of the queueSet.
+func (qs *queueSet) chooseQueueIndexLocked(hashValue uint64, descr1, descr2 interface{}) int {
+	bestQueueIdx := -1
+	bestQueueLen := int(math.MaxInt32)
+	// the dealer uses the current desired number of queues, which is no larger than the number in `qs.queues`.
+	qs.dealer.Deal(hashValue, func(queueIdx int) {
+		thisLen := len(qs.queues[queueIdx].requests)
+		klog.V(7).Infof("QS(%s): For request %#+v %#+v considering queue %d of length %d", qs.qCfg.Name, descr1, descr2, queueIdx, thisLen)
+		if thisLen < bestQueueLen {
+			bestQueueIdx, bestQueueLen = queueIdx, thisLen
+		}
+	})
+	klog.V(6).Infof("QS(%s): For request %#+v %#+v chose queue %d, had %d waiting & %d executing", qs.qCfg.Name, descr1, descr2, bestQueueIdx, bestQueueLen, qs.queues[bestQueueIdx].requestsExecuting)
+	return bestQueueIdx
+}
+
+// removeTimedOutRequestsFromQueueLocked rejects old requests that have been enqueued
+// past the requestWaitLimit
+func (qs *queueSet) removeTimedOutRequestsFromQueueLocked(queue *queue, fsName string) {
+	timeoutIdx := -1
+	now := qs.clock.Now()
+	reqs := queue.requests
+	// reqs are sorted oldest -> newest
+	// can short circuit loop (break) if oldest requests are not timing out
+	// as newer requests also will not have timed out
+
+	// now - requestWaitLimit = waitLimit
+	waitLimit := now.Add(-qs.qCfg.RequestWaitLimit)
+	for i, req := range reqs {
+		if waitLimit.After(req.arrivalTime) {
+			req.decision.SetLocked(decisionReject)
+			// get index for timed out requests
+			timeoutIdx = i
+			metrics.AddRequestsInQueues(qs.qCfg.Name, req.fsName, -1)
+		} else {
+			break
+		}
+	}
+	// remove timed out requests from queue
+	if timeoutIdx != -1 {
+		// timeoutIdx + 1 to remove the last timeout req
+		removeIdx := timeoutIdx + 1
+		// remove all the timeout requests
+		queue.requests = reqs[removeIdx:]
+		// decrement the # of requestsEnqueued
+		qs.totRequestsWaiting -= removeIdx
+	}
+}
+
+// rejectOrEnqueueLocked rejects or enqueues the newly arrived
+// request, which has been assigned to a queue.  If up against the
+// queue length limit and the concurrency limit then returns false.
+// Otherwise enqueues and returns true.
+func (qs *queueSet) rejectOrEnqueueLocked(request *request) bool {
+	queue := request.queue
+	curQueueLength := len(queue.requests)
+	// rejects the newly arrived request if resource criteria not met
+	if qs.totRequestsExecuting >= qs.dCfg.ConcurrencyLimit &&
+		curQueueLength >= qs.qCfg.QueueLengthLimit {
+		return false
+	}
+
+	qs.enqueueLocked(request)
+	return true
+}
+
+// enqueues a request into its queue.
+func (qs *queueSet) enqueueLocked(request *request) {
+	queue := request.queue
+	if len(queue.requests) == 0 && queue.requestsExecuting == 0 {
+		// the queue’s virtual start time is set to the virtual time.
+		queue.virtualStart = qs.virtualTime
+		if klog.V(6) {
+			klog.Infof("QS(%s) at r=%s v=%.9fs: initialized queue %d virtual start time due to request %#+v %#+v", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), queue.virtualStart, queue.index, request.descr1, request.descr2)
+		}
+	}
+	queue.Enqueue(request)
+	qs.totRequestsWaiting++
+	metrics.AddRequestsInQueues(qs.qCfg.Name, request.fsName, 1)
+}
+
+// dispatchAsMuchAsPossibleLocked runs a loop, as long as there
+// are non-empty queues and the number currently executing is less than the
+// assured concurrency value.  The body of the loop uses the fair queuing
+// technique to pick a queue, dequeue the request at the head of that
+// queue, increment the count of the number executing, and send true
+// to the request's channel.
+func (qs *queueSet) dispatchAsMuchAsPossibleLocked() {
+	for qs.totRequestsWaiting != 0 && qs.totRequestsExecuting < qs.dCfg.ConcurrencyLimit {
+		ok := qs.dispatchLocked()
+		if !ok {
+			break
+		}
+	}
+}
+
+func (qs *queueSet) dispatchSansQueueLocked(ctx context.Context, fsName string, descr1, descr2 interface{}) *request {
+	now := qs.clock.Now()
+	req := &request{
+		qs:          qs,
+		fsName:      fsName,
+		ctx:         ctx,
+		startTime:   now,
+		decision:    lockingpromise.NewWriteOnce(&qs.lock, qs.counter),
+		arrivalTime: now,
+		descr1:      descr1,
+		descr2:      descr2,
+	}
+	req.decision.SetLocked(decisionExecute)
+	qs.totRequestsExecuting++
+	metrics.AddRequestsExecuting(qs.qCfg.Name, fsName, 1)
+	if klog.V(5) {
+		klog.Infof("QS(%s) at r=%s v=%.9fs: immediate dispatch of request %q %#+v %#+v, qs will have %d executing", qs.qCfg.Name, now.Format(nsTimeFmt), qs.virtualTime, fsName, descr1, descr2, qs.totRequestsExecuting)
+	}
+	return req
+}
+
+// dispatchLocked uses the Fair Queuing for Server Requests method to
+// select a queue and dispatch the oldest request in that queue.  The
+// return value indicates whether a request was dispatched; this will
+// be false when there are no requests waiting in any queue.
+func (qs *queueSet) dispatchLocked() bool {
+	queue := qs.selectQueueLocked()
+	if queue == nil {
+		return false
+	}
+	request, ok := queue.Dequeue()
+	if !ok { // This should never happen.  But if it does...
+		return false
+	}
+	request.startTime = qs.clock.Now()
+	// At this moment the request leaves its queue and starts
+	// executing.  We do not recognize any interim state between
+	// "queued" and "executing".  While that means "executing"
+	// includes a little overhead from this package, this is not a
+	// problem because other overhead is also included.
+	qs.totRequestsWaiting--
+	qs.totRequestsExecuting++
+	queue.requestsExecuting++
+	metrics.AddRequestsInQueues(qs.qCfg.Name, request.fsName, -1)
+	metrics.AddRequestsExecuting(qs.qCfg.Name, request.fsName, 1)
+	if klog.V(6) {
+		klog.Infof("QS(%s) at r=%s v=%.9fs: dispatching request %#+v %#+v from queue %d with virtual start time %.9fs, queue will have %d waiting & %d executing", qs.qCfg.Name, request.startTime.Format(nsTimeFmt), qs.virtualTime, request.descr1, request.descr2, queue.index, queue.virtualStart, len(queue.requests), queue.requestsExecuting)
+	}
+	// When a request is dequeued for service -> qs.virtualStart += G
+	queue.virtualStart += qs.estimatedServiceTime
+	request.decision.SetLocked(decisionExecute)
+	return ok
+}
+
+// cancelWait ensures the request is not waiting.  This is only
+// applicable to a request that has been assigned to a queue.
+func (qs *queueSet) cancelWait(req *request) {
+	qs.lock.Lock()
+	defer qs.lock.Unlock()
+	if req.decision.IsSetLocked() {
+		// The request has already been removed from the queue
+		// and so we consider its wait to be over.
+		return
+	}
+	req.decision.SetLocked(decisionCancel)
+	queue := req.queue
+	// remove the request from the queue as it has timed out
+	for i := range queue.requests {
+		if req == queue.requests[i] {
+			// remove the request
+			queue.requests = append(queue.requests[:i], queue.requests[i+1:]...)
+			qs.totRequestsWaiting--
+			metrics.AddRequestsInQueues(qs.qCfg.Name, req.fsName, -1)
+			break
+		}
+	}
+	return
+}
+
+// selectQueueLocked examines the queues in round robin order and
+// returns the first one of those for which the virtual finish time of
+// the oldest waiting request is minimal.
+func (qs *queueSet) selectQueueLocked() *queue {
+	minVirtualFinish := math.Inf(1)
+	var minQueue *queue
+	var minIndex int
+	nq := len(qs.queues)
+	for range qs.queues {
+		qs.robinIndex = (qs.robinIndex + 1) % nq
+		queue := qs.queues[qs.robinIndex]
+		if len(queue.requests) != 0 {
+			currentVirtualFinish := queue.GetVirtualFinish(0, qs.estimatedServiceTime)
+			if currentVirtualFinish < minVirtualFinish {
+				minVirtualFinish = currentVirtualFinish
+				minQueue = queue
+				minIndex = qs.robinIndex
+			}
+		}
+	}
+	// we set the round robin indexing to start at the chose queue
+	// for the next round.  This way the non-selected queues
+	// win in the case that the virtual finish times are the same
+	qs.robinIndex = minIndex
+	return minQueue
+}
+
+// finishRequestAndDispatchAsMuchAsPossible is a convenience method
+// which calls finishRequest for a given request and then dispatches
+// as many requests as possible.  This is all of what needs to be done
+// once a request finishes execution or is canceled.  This returns a bool
+// indicating whether the QueueSet is now idle.
+func (qs *queueSet) finishRequestAndDispatchAsMuchAsPossible(req *request) bool {
+	qs.lockAndSyncTime()
+	defer qs.lock.Unlock()
+
+	qs.finishRequestLocked(req)
+	qs.dispatchAsMuchAsPossibleLocked()
+	return qs.isIdleLocked()
+}
+
+// finishRequestLocked is a callback that should be used when a
+// previously dispatched request has completed it's service.  This
+// callback updates important state in the queueSet
+func (qs *queueSet) finishRequestLocked(r *request) {
+	qs.totRequestsExecuting--
+	metrics.AddRequestsExecuting(qs.qCfg.Name, r.fsName, -1)
+
+	if r.queue == nil {
+		if klog.V(6) {
+			klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, qs will have %d executing", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, qs.totRequestsExecuting)
+		}
+		return
+	}
+
+	S := qs.clock.Since(r.startTime).Seconds()
+
+	// When a request finishes being served, and the actual service time was S,
+	// the queue’s virtual start time is decremented by G - S.
+	r.queue.virtualStart -= qs.estimatedServiceTime - S
+
+	// request has finished, remove from requests executing
+	r.queue.requestsExecuting--
+
+	if klog.V(6) {
+		klog.Infof("QS(%s) at r=%s v=%.9fs: request %#+v %#+v finished, adjusted queue %d virtual start time to %.9fs due to service time %.9fs, queue will have %d waiting & %d executing", qs.qCfg.Name, qs.clock.Now().Format(nsTimeFmt), qs.virtualTime, r.descr1, r.descr2, r.queue.index, r.queue.virtualStart, S, len(r.queue.requests), r.queue.requestsExecuting)
+	}
+
+	// If there are more queues than desired and this one has no
+	// requests then remove it
+	if len(qs.queues) > qs.qCfg.DesiredNumQueues &&
+		len(r.queue.requests) == 0 &&
+		r.queue.requestsExecuting == 0 {
+		qs.queues = removeQueueAndUpdateIndexes(qs.queues, r.queue.index)
+
+		// decrement here to maintain the invariant that (qs.robinIndex+1) % numQueues
+		// is the index of the next queue after the one last dispatched from
+		if qs.robinIndex >= r.queue.index {
+			qs.robinIndex--
+		}
+	}
+}
+
+// removeQueueAndUpdateIndexes uses reslicing to remove an index from a slice
+// and then updates the 'index' field of the queues to be correct
+func removeQueueAndUpdateIndexes(queues []*queue, index int) []*queue {
+	keptQueues := append(queues[:index], queues[index+1:]...)
+	for i := index; i < len(keptQueues); i++ {
+		keptQueues[i].index--
+	}
+	return keptQueues
+}
+
+// preCreateOrUnblockGoroutine needs to be called before creating a
+// goroutine associated with this queueSet or unblocking a blocked
+// one, to properly update the accounting used in testing.
+func (qs *queueSet) preCreateOrUnblockGoroutine() {
+	qs.counter.Add(1)
+}
+
+// goroutineDoneOrBlocked needs to be called at the end of every
+// goroutine associated with this queueSet or when such a goroutine is
+// about to wait on some other goroutine to do something; this is to
+// properly update the accounting used in testing.
+func (qs *queueSet) goroutineDoneOrBlocked() {
+	qs.counter.Add(-1)
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/queueset/types.go

@@ -0,0 +1,96 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package queueset
+
+import (
+	"context"
+	"time"
+
+	"k8s.io/apiserver/pkg/util/flowcontrol/fairqueuing/promise"
+)
+
+// request is a temporary container for "requests" with additional
+// tracking fields required for the functionality FQScheduler
+type request struct {
+	qs     *queueSet
+	fsName string
+	ctx    context.Context
+
+	// The relevant queue.  Is nil if this request did not go through
+	// a queue.
+	queue *queue
+
+	// startTime is the real time when the request began executing
+	startTime time.Time
+
+	// decision gets set to a `requestDecision` indicating what to do
+	// with this request.  It gets set exactly once, when the request
+	// is removed from its queue.  The value will be decisionReject,
+	// decisionCancel, or decisionExecute; decisionTryAnother never
+	// appears here.
+	decision promise.LockingWriteOnce
+
+	// arrivalTime is the real time when the request entered this system
+	arrivalTime time.Time
+
+	// descr1 and descr2 are not used in any logic but they appear in
+	// log messages
+	descr1, descr2 interface{}
+
+	// Indicates whether client has called Request::Wait()
+	waitStarted bool
+}
+
+// queue is an array of requests with additional metadata required for
+// the FQScheduler
+type queue struct {
+	requests []*request
+
+	// virtualStart is the virtual time (virtual seconds since process
+	// startup) when the oldest request in the queue (if there is any)
+	// started virtually executing
+	virtualStart float64
+
+	requestsExecuting int
+	index             int
+}
+
+// Enqueue enqueues a request into the queue
+func (q *queue) Enqueue(request *request) {
+	q.requests = append(q.requests, request)
+}
+
+// Dequeue dequeues a request from the queue
+func (q *queue) Dequeue() (*request, bool) {
+	if len(q.requests) == 0 {
+		return nil, false
+	}
+	request := q.requests[0]
+	q.requests = q.requests[1:]
+	return request, true
+}
+
+// GetVirtualFinish returns the expected virtual finish time of the request at
+// index J in the queue with estimated finish time G
+func (q *queue) GetVirtualFinish(J int, G float64) float64 {
+	// The virtual finish time of request number J in the queue
+	// (counting from J=1 for the head) is J * G + (virtual start time).
+
+	// counting from J=1 for the head (eg: queue.requests[0] -> J=1) - J+1
+	jg := float64(J+1) * float64(G)
+	return jg + q.virtualStart
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/format/formatting.go

@@ -0,0 +1,231 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package format
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+
+	fcv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// This file provides an easy way to mark a value for formatting to
+// `%s` in full detail IF it is printed but without costing a lot of
+// CPU or memory if the value is NOT printed.  The API Priority and
+// Fairness API objects are formatted into JSON.  The other types of
+// objects here are formatted into golang source.
+
+// Stringer marks the given value for custom formatting by this package.
+type Stringer struct{ val interface{} }
+
+// Fmt marks the given value for custom formatting by this package.
+func Fmt(val interface{}) Stringer {
+	return Stringer{val}
+}
+
+// String formats to a string in full detail
+func (sr Stringer) String() string {
+	if sr.val == nil {
+		return "nil"
+	}
+	switch typed := sr.val.(type) {
+	case *fcv1a1.FlowSchema,
+		fcv1a1.FlowSchema,
+		fcv1a1.FlowSchemaSpec,
+		fcv1a1.FlowDistinguisherMethod,
+		*fcv1a1.FlowDistinguisherMethod,
+		*fcv1a1.PolicyRulesWithSubjects,
+		fcv1a1.PolicyRulesWithSubjects,
+		fcv1a1.Subject,
+		fcv1a1.ResourcePolicyRule,
+		fcv1a1.NonResourcePolicyRule,
+		fcv1a1.FlowSchemaCondition,
+		*fcv1a1.PriorityLevelConfiguration,
+		fcv1a1.PriorityLevelConfiguration,
+		fcv1a1.PriorityLevelConfigurationSpec,
+		*fcv1a1.LimitedPriorityLevelConfiguration,
+		fcv1a1.LimitedPriorityLevelConfiguration,
+		fcv1a1.LimitResponse,
+		*fcv1a1.QueuingConfiguration,
+		fcv1a1.QueuingConfiguration:
+		return ToJSON(sr.val)
+	case []user.Info:
+		return FmtUsers(typed)
+	case []*request.RequestInfo:
+		return FmtRequests(typed)
+	default:
+		return fmt.Sprintf("%#+v", sr.val)
+	}
+}
+
+// ToJSON converts using encoding/json and handles errors by
+// formatting them
+func ToJSON(val interface{}) string {
+	bs, err := json.Marshal(val)
+	str := string(bs)
+	if err != nil {
+		str = str + "<" + err.Error() + ">"
+	}
+	return str
+}
+
+// FmtPriorityLevelConfiguration returns a golang source expression
+// equivalent to the given value
+func FmtPriorityLevelConfiguration(pl *fcv1a1.PriorityLevelConfiguration) string {
+	if pl == nil {
+		return "nil"
+	}
+	var buf bytes.Buffer
+	buf.WriteString(fmt.Sprintf("&v1alpha1.PriorityLevelConfiguration{ObjectMeta: %#+v, Spec: ",
+		pl.ObjectMeta))
+	BufferPriorityLevelConfigurationSpec(&buf, &pl.Spec)
+	buf.WriteString(fmt.Sprintf(", Status: %#+v}", pl.Status))
+	return buf.String()
+}
+
+// FmtPriorityLevelConfigurationSpec returns a golang source
+// expression equivalent to the given value
+func FmtPriorityLevelConfigurationSpec(plSpec *fcv1a1.PriorityLevelConfigurationSpec) string {
+	var buf bytes.Buffer
+	BufferPriorityLevelConfigurationSpec(&buf, plSpec)
+	return buf.String()
+}
+
+// BufferPriorityLevelConfigurationSpec writes a golang source
+// expression for the given value to the given buffer
+func BufferPriorityLevelConfigurationSpec(buf *bytes.Buffer, plSpec *fcv1a1.PriorityLevelConfigurationSpec) {
+	buf.WriteString(fmt.Sprintf("v1alpha1.PriorityLevelConfigurationSpec{Type: %#v", plSpec.Type))
+	if plSpec.Limited != nil {
+		buf.WriteString(fmt.Sprintf(", Limited: &v1alpha1.LimitedPriorityLevelConfiguration{AssuredConcurrencyShares:%d, LimitResponse:v1alpha1.LimitResponse{Type:%#v", plSpec.Limited.AssuredConcurrencyShares, plSpec.Limited.LimitResponse.Type))
+		if plSpec.Limited.LimitResponse.Queuing != nil {
+			buf.WriteString(fmt.Sprintf(", Queuing:&%#+v", *plSpec.Limited.LimitResponse.Queuing))
+		}
+		buf.WriteString(" } }")
+	}
+	buf.WriteString("}")
+}
+
+// FmtFlowSchema produces a golang source expression of the value.
+func FmtFlowSchema(fs *fcv1a1.FlowSchema) string {
+	if fs == nil {
+		return "nil"
+	}
+	var buf bytes.Buffer
+	buf.WriteString(fmt.Sprintf("&v1alpha1.FlowSchema{ObjectMeta: %#+v, Spec: ",
+		fs.ObjectMeta))
+	BufferFlowSchemaSpec(&buf, &fs.Spec)
+	buf.WriteString(fmt.Sprintf(", Status: %#+v}", fs.Status))
+	return buf.String()
+}
+
+// FmtFlowSchemaSpec produces a golang source expression equivalent to
+// the given spec
+func FmtFlowSchemaSpec(fsSpec *fcv1a1.FlowSchemaSpec) string {
+	var buf bytes.Buffer
+	BufferFlowSchemaSpec(&buf, fsSpec)
+	return buf.String()
+}
+
+// BufferFlowSchemaSpec writes a golang source expression for the
+// given value to the given buffer
+func BufferFlowSchemaSpec(buf *bytes.Buffer, fsSpec *fcv1a1.FlowSchemaSpec) {
+	buf.WriteString(fmt.Sprintf("v1alpha1.FlowSchemaSpec{PriorityLevelConfiguration: %#+v, MatchingPrecedence: %d, DistinguisherMethod: ",
+		fsSpec.PriorityLevelConfiguration,
+		fsSpec.MatchingPrecedence))
+	if fsSpec.DistinguisherMethod == nil {
+		buf.WriteString("nil")
+	} else {
+		buf.WriteString(fmt.Sprintf("&%#+v", *fsSpec.DistinguisherMethod))
+	}
+	buf.WriteString(", Rules: []v1alpha1.PolicyRulesWithSubjects{")
+	for idx, rule := range fsSpec.Rules {
+		if idx > 0 {
+			buf.WriteString(", ")
+		}
+		BufferFmtPolicyRulesWithSubjectsSlim(buf, rule)
+	}
+	buf.WriteString("}}")
+}
+
+// FmtPolicyRulesWithSubjects produces a golang source expression of the value.
+func FmtPolicyRulesWithSubjects(rule fcv1a1.PolicyRulesWithSubjects) string {
+	return "v1alpha1.PolicyRulesWithSubjects" + FmtPolicyRulesWithSubjectsSlim(rule)
+}
+
+// FmtPolicyRulesWithSubjectsSlim produces a golang source expression
+// of the value but without the leading type name.  See above for an
+// example context where this is useful.
+func FmtPolicyRulesWithSubjectsSlim(rule fcv1a1.PolicyRulesWithSubjects) string {
+	var buf bytes.Buffer
+	BufferFmtPolicyRulesWithSubjectsSlim(&buf, rule)
+	return buf.String()
+}
+
+// BufferFmtPolicyRulesWithSubjectsSlim writes a golang source
+// expression for the given value to the given buffer but excludes the
+// leading type name
+func BufferFmtPolicyRulesWithSubjectsSlim(buf *bytes.Buffer, rule fcv1a1.PolicyRulesWithSubjects) {
+	buf.WriteString("{Subjects: []v1alpha1.Subject{")
+	for jdx, subj := range rule.Subjects {
+		if jdx > 0 {
+			buf.WriteString(", ")
+		}
+		buf.WriteString(fmt.Sprintf("{Kind: %q", subj.Kind))
+		if subj.User != nil {
+			buf.WriteString(fmt.Sprintf(", User: &%#+v", *subj.User))
+		}
+		if subj.Group != nil {
+			buf.WriteString(fmt.Sprintf(", Group: &%#+v", *subj.Group))
+		}
+		if subj.ServiceAccount != nil {
+			buf.WriteString(fmt.Sprintf(", ServiceAcount: &%#+v", *subj.ServiceAccount))
+		}
+		buf.WriteString("}")
+	}
+	buf.WriteString(fmt.Sprintf("}, ResourceRules: %#+v, NonResourceRules: %#+v}", rule.ResourceRules, rule.NonResourceRules))
+}
+
+// FmtUsers produces a golang source expression of the value.
+func FmtUsers(list []user.Info) string {
+	var buf bytes.Buffer
+	buf.WriteString("[]user.Info{")
+	for idx, member := range list {
+		if idx > 0 {
+			buf.WriteString(", ")
+		}
+		buf.WriteString(fmt.Sprintf("%#+v", member))
+	}
+	buf.WriteString("}")
+	return buf.String()
+}
+
+// FmtRequests produces a golang source expression of the value.
+func FmtRequests(list []*request.RequestInfo) string {
+	var buf bytes.Buffer
+	buf.WriteString("[]*request.RequestInfo{")
+	for idx, member := range list {
+		if idx > 0 {
+			buf.WriteString(", ")
+		}
+		buf.WriteString(fmt.Sprintf("%#+v", member))
+	}
+	buf.WriteString("}")
+	return buf.String()
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/formatting.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flowcontrol
+
+import (
+	"fmt"
+
+	fcfmt "k8s.io/apiserver/pkg/util/flowcontrol/format"
+)
+
+var _ fmt.GoStringer = RequestDigest{}
+
+// GoString produces a golang source expression of the value.
+func (rd RequestDigest) GoString() string {
+	return fmt.Sprintf("RequestDigest{RequestInfo: %#+v, User: %#+v}", rd.RequestInfo, rd.User)
+}
+
+var _ fmt.GoStringer = (*priorityLevelState)(nil)
+
+// GoString produces a golang source expression of the value.
+func (pls *priorityLevelState) GoString() string {
+	if pls == nil {
+		return "nil"
+	}
+	return fmt.Sprintf("&priorityLevelState{pl:%s, qsCompleter:%#+v, queues:%#+v, quiescing:%#v, numPending:%d}", fcfmt.Fmt(pls.pl), pls.qsCompleter, pls.queues, pls.quiescing, pls.numPending)
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/metrics/metrics.go

@@ -0,0 +1,198 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"strings"
+	"sync"
+	"time"
+
+	compbasemetrics "k8s.io/component-base/metrics"
+	"k8s.io/component-base/metrics/legacyregistry"
+	basemetricstestutil "k8s.io/component-base/metrics/testutil"
+)
+
+const (
+	namespace = "apiserver"
+	subsystem = "flowcontrol"
+)
+
+const (
+	priorityLevel = "priorityLevel"
+	flowSchema    = "flowSchema"
+)
+
+var (
+	queueLengthBuckets            = []float64{0, 10, 25, 50, 100, 250, 500, 1000}
+	requestDurationSecondsBuckets = []float64{0, 0.005, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10, 30}
+)
+
+var registerMetrics sync.Once
+
+// Register all metrics.
+func Register() {
+	registerMetrics.Do(func() {
+		for _, metric := range metrics {
+			legacyregistry.MustRegister(metric)
+		}
+	})
+}
+
+type resettable interface {
+	Reset()
+}
+
+// Reset all metrics to zero
+func Reset() {
+	for _, metric := range metrics {
+		rm := metric.(resettable)
+		rm.Reset()
+	}
+}
+
+// GatherAndCompare the given metrics with the given Prometheus syntax expected value
+func GatherAndCompare(expected string, metricNames ...string) error {
+	return basemetricstestutil.GatherAndCompare(legacyregistry.DefaultGatherer, strings.NewReader(expected), metricNames...)
+}
+
+var (
+	apiserverRejectedRequestsTotal = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "rejected_requests_total",
+			Help:      "Number of requests rejected by API Priority and Fairness system",
+		},
+		[]string{priorityLevel, flowSchema, "reason"},
+	)
+	apiserverDispatchedRequestsTotal = compbasemetrics.NewCounterVec(
+		&compbasemetrics.CounterOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "dispatched_requests_total",
+			Help:      "Number of requests released by API Priority and Fairness system for service",
+		},
+		[]string{priorityLevel, flowSchema},
+	)
+	apiserverCurrentInqueueRequests = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "current_inqueue_requests",
+			Help:      "Number of requests currently pending in queues of the API Priority and Fairness system",
+		},
+		[]string{priorityLevel, flowSchema},
+	)
+	apiserverRequestQueueLength = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_queue_length_after_enqueue",
+			Help:      "Length of queue in the API Priority and Fairness system, as seen by each request after it is enqueued",
+			Buckets:   queueLengthBuckets,
+		},
+		[]string{priorityLevel, flowSchema},
+	)
+	apiserverRequestConcurrencyLimit = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_concurrency_limit",
+			Help:      "Shared concurrency limit in the API Priority and Fairness system",
+		},
+		[]string{priorityLevel},
+	)
+	apiserverCurrentExecutingRequests = compbasemetrics.NewGaugeVec(
+		&compbasemetrics.GaugeOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "current_executing_requests",
+			Help:      "Number of requests currently executing in the API Priority and Fairness system",
+		},
+		[]string{priorityLevel, flowSchema},
+	)
+	apiserverRequestWaitingSeconds = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_wait_duration_seconds",
+			Help:      "Length of time a request spent waiting in its queue",
+			Buckets:   requestDurationSecondsBuckets,
+		},
+		[]string{priorityLevel, flowSchema, "execute"},
+	)
+	apiserverRequestExecutionSeconds = compbasemetrics.NewHistogramVec(
+		&compbasemetrics.HistogramOpts{
+			Namespace: namespace,
+			Subsystem: subsystem,
+			Name:      "request_execution_seconds",
+			Help:      "Duration of request execution in the API Priority and Fairness system",
+			Buckets:   requestDurationSecondsBuckets,
+		},
+		[]string{priorityLevel, flowSchema},
+	)
+	metrics = []compbasemetrics.Registerable{
+		apiserverRejectedRequestsTotal,
+		apiserverDispatchedRequestsTotal,
+		apiserverCurrentInqueueRequests,
+		apiserverRequestQueueLength,
+		apiserverRequestConcurrencyLimit,
+		apiserverCurrentExecutingRequests,
+		apiserverRequestWaitingSeconds,
+		apiserverRequestExecutionSeconds,
+	}
+)
+
+// AddRequestsInQueues adds the given delta to the gauge of the # of requests in the queues of the specified flowSchema and priorityLevel
+func AddRequestsInQueues(priorityLevel, flowSchema string, delta int) {
+	apiserverCurrentInqueueRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
+}
+
+// AddRequestsExecuting adds the given delta to the gauge of executing requests of the given flowSchema and priorityLevel
+func AddRequestsExecuting(priorityLevel, flowSchema string, delta int) {
+	apiserverCurrentExecutingRequests.WithLabelValues(priorityLevel, flowSchema).Add(float64(delta))
+}
+
+// UpdateSharedConcurrencyLimit updates the value for the concurrency limit in flow control
+func UpdateSharedConcurrencyLimit(priorityLevel string, limit int) {
+	apiserverRequestConcurrencyLimit.WithLabelValues(priorityLevel).Set(float64(limit))
+}
+
+// AddReject increments the # of rejected requests for flow control
+func AddReject(priorityLevel, flowSchema, reason string) {
+	apiserverRejectedRequestsTotal.WithLabelValues(priorityLevel, flowSchema, reason).Add(1)
+}
+
+// AddDispatch increments the # of dispatched requests for flow control
+func AddDispatch(priorityLevel, flowSchema string) {
+	apiserverDispatchedRequestsTotal.WithLabelValues(priorityLevel, flowSchema).Add(1)
+}
+
+// ObserveQueueLength observes the queue length for flow control
+func ObserveQueueLength(priorityLevel, flowSchema string, length int) {
+	apiserverRequestQueueLength.WithLabelValues(priorityLevel, flowSchema).Observe(float64(length))
+}
+
+// ObserveWaitingDuration observes the queue length for flow control
+func ObserveWaitingDuration(priorityLevel, flowSchema, execute string, waitTime time.Duration) {
+	apiserverRequestWaitingSeconds.WithLabelValues(priorityLevel, flowSchema, execute).Observe(waitTime.Seconds())
+}
+
+// ObserveExecutionDuration observes the execution duration for flow control
+func ObserveExecutionDuration(priorityLevel, flowSchema string, executionTime time.Duration) {
+	apiserverRequestExecutionSeconds.WithLabelValues(priorityLevel, flowSchema).Observe(executionTime.Seconds())
+}

vendor/k8s.io/apiserver/pkg/util/flowcontrol/rule.go

@@ -0,0 +1,203 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package flowcontrol
+
+import (
+	"strings"
+
+	fctypesv1a1 "k8s.io/api/flowcontrol/v1alpha1"
+	"k8s.io/apiserver/pkg/authentication/serviceaccount"
+	"k8s.io/apiserver/pkg/authentication/user"
+	"k8s.io/apiserver/pkg/endpoints/request"
+)
+
+// Tests whether a given request and FlowSchema match.  Nobody mutates
+// either input.
+func matchesFlowSchema(digest RequestDigest, flowSchema *fctypesv1a1.FlowSchema) bool {
+	for _, policyRule := range flowSchema.Spec.Rules {
+		if matchesPolicyRule(digest, &policyRule) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchesPolicyRule(digest RequestDigest, policyRule *fctypesv1a1.PolicyRulesWithSubjects) bool {
+	if !matchesASubject(digest.User, policyRule.Subjects) {
+		return false
+	}
+	if digest.RequestInfo.IsResourceRequest {
+		return matchesAResourceRule(digest.RequestInfo, policyRule.ResourceRules)
+	}
+	return matchesANonResourceRule(digest.RequestInfo, policyRule.NonResourceRules)
+}
+
+func matchesASubject(user user.Info, subjects []fctypesv1a1.Subject) bool {
+	for _, subject := range subjects {
+		if matchesSubject(user, subject) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchesSubject(user user.Info, subject fctypesv1a1.Subject) bool {
+	switch subject.Kind {
+	case fctypesv1a1.SubjectKindUser:
+		return subject.User != nil && (subject.User.Name == fctypesv1a1.NameAll || subject.User.Name == user.GetName())
+	case fctypesv1a1.SubjectKindGroup:
+		if subject.Group == nil {
+			return false
+		}
+		seek := subject.Group.Name
+		if seek == "*" {
+			return true
+		}
+		for _, userGroup := range user.GetGroups() {
+			if userGroup == seek {
+				return true
+			}
+		}
+		return false
+	case fctypesv1a1.SubjectKindServiceAccount:
+		if subject.ServiceAccount == nil {
+			return false
+		}
+		if subject.ServiceAccount.Name == fctypesv1a1.NameAll {
+			return serviceAccountMatchesNamespace(subject.ServiceAccount.Namespace, user.GetName())
+		}
+		return serviceaccount.MatchesUsername(subject.ServiceAccount.Namespace, subject.ServiceAccount.Name, user.GetName())
+	default:
+		return false
+	}
+}
+
+// serviceAccountMatchesNamespace checks whether the provided service account username matches the namespace, without
+// allocating. Use this when checking a service account namespace against a known string.
+// This is copied from `k8s.io/apiserver/pkg/authentication/serviceaccount::MatchesUsername` and simplified to not check the name part.
+func serviceAccountMatchesNamespace(namespace string, username string) bool {
+	const (
+		ServiceAccountUsernamePrefix    = "system:serviceaccount:"
+		ServiceAccountUsernameSeparator = ":"
+	)
+	if !strings.HasPrefix(username, ServiceAccountUsernamePrefix) {
+		return false
+	}
+	username = username[len(ServiceAccountUsernamePrefix):]
+
+	if !strings.HasPrefix(username, namespace) {
+		return false
+	}
+	username = username[len(namespace):]
+
+	return strings.HasPrefix(username, ServiceAccountUsernameSeparator)
+}
+
+func matchesAResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.ResourcePolicyRule) bool {
+	for _, rr := range rules {
+		if matchesResourcePolicyRule(ri, rr) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchesResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a1.ResourcePolicyRule) bool {
+	if !matchPolicyRuleVerb(policyRule.Verbs, ri.Verb) {
+		return false
+	}
+	if !matchPolicyRuleResource(policyRule.Resources, ri.Resource, ri.Subresource) {
+		return false
+	}
+	if !matchPolicyRuleAPIGroup(policyRule.APIGroups, ri.APIGroup) {
+		return false
+	}
+	if len(ri.Namespace) == 0 {
+		return policyRule.ClusterScope
+	}
+	return containsString(ri.Namespace, policyRule.Namespaces, fctypesv1a1.NamespaceEvery)
+}
+
+func matchesANonResourceRule(ri *request.RequestInfo, rules []fctypesv1a1.NonResourcePolicyRule) bool {
+	for _, rr := range rules {
+		if matchesNonResourcePolicyRule(ri, rr) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchesNonResourcePolicyRule(ri *request.RequestInfo, policyRule fctypesv1a1.NonResourcePolicyRule) bool {
+	if !matchPolicyRuleVerb(policyRule.Verbs, ri.Verb) {
+		return false
+	}
+	return matchPolicyRuleNonResourceURL(policyRule.NonResourceURLs, ri.Path)
+}
+
+func matchPolicyRuleVerb(policyRuleVerbs []string, requestVerb string) bool {
+	return containsString(requestVerb, policyRuleVerbs, fctypesv1a1.VerbAll)
+}
+
+func matchPolicyRuleNonResourceURL(policyRuleRequestURLs []string, requestPath string) bool {
+	for _, rulePath := range policyRuleRequestURLs {
+		if rulePath == fctypesv1a1.NonResourceAll || rulePath == requestPath {
+			return true
+		}
+		rulePrefix := strings.TrimSuffix(rulePath, "*")
+		if !strings.HasSuffix(rulePrefix, "/") {
+			rulePrefix = rulePrefix + "/"
+		}
+		if strings.HasPrefix(requestPath, rulePrefix) {
+			return true
+		}
+	}
+	return false
+}
+
+func matchPolicyRuleAPIGroup(policyRuleAPIGroups []string, requestAPIGroup string) bool {
+	return containsString(requestAPIGroup, policyRuleAPIGroups, fctypesv1a1.APIGroupAll)
+}
+
+func rsJoin(requestResource, requestSubresource string) string {
+	seekString := requestResource
+	if requestSubresource != "" {
+		seekString = requestResource + "/" + requestSubresource
+	}
+	return seekString
+}
+
+func matchPolicyRuleResource(policyRuleRequestResources []string, requestResource, requestSubresource string) bool {
+	return containsString(rsJoin(requestResource, requestSubresource), policyRuleRequestResources, fctypesv1a1.ResourceAll)
+}
+
+// containsString returns true if either `x` or `wildcard` is in
+// `list`.  The wildcard is not a pattern to match against `x`; rather
+// the presence of the wildcard in the list is the caller's way of
+// saying that all values of `x` should match the list.  This function
+// assumes that if `wildcard` is in `list` then it is the only member
+// of the list, which is enforced by validation.
+func containsString(x string, list []string, wildcard string) bool {
+	if len(list) == 1 && list[0] == wildcard {
+		return true
+	}
+	for _, y := range list {
+		if x == y {
+			return true
+		}
+	}
+	return false
+}

vendor/k8s.io/apiserver/pkg/util/shufflesharding/shufflesharding.go

@@ -0,0 +1,107 @@
+/*
+Copyright 2019 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package shufflesharding
+
+import (
+	"fmt"
+	"math"
+)
+
+// MaxHashBits is the max bit length which can be used from hash value.
+// If we use all bits of hash value, the critical(last) card shuffled by
+// Dealer will be uneven to 2:3 (first half:second half) at most,
+// in order to reduce this unevenness to 32:33, we set MaxHashBits to 60 here.
+const MaxHashBits = 60
+
+// RequiredEntropyBits makes a quick and slightly conservative estimate of the number
+// of bits of hash value that are consumed in shuffle sharding a deck of the given size
+// to a hand of the given size.  The result is meaningful only if
+// 1 <= handSize <= deckSize <= 1<<26.
+func RequiredEntropyBits(deckSize, handSize int) int {
+	return int(math.Ceil(math.Log2(float64(deckSize)) * float64(handSize)))
+}
+
+// Dealer contains some necessary parameters and provides some methods for shuffle sharding.
+// Dealer is thread-safe.
+type Dealer struct {
+	deckSize int
+	handSize int
+}
+
+// NewDealer will create a Dealer with the given deckSize and handSize, will return error when
+// deckSize or handSize is invalid as below.
+// 1. deckSize or handSize is not positive
+// 2. handSize is greater than deckSize
+// 3. deckSize is impractically large (greater than 1<<26)
+// 4. required entropy bits of deckSize and handSize is greater than MaxHashBits
+func NewDealer(deckSize, handSize int) (*Dealer, error) {
+	if deckSize <= 0 || handSize <= 0 {
+		return nil, fmt.Errorf("deckSize %d or handSize %d is not positive", deckSize, handSize)
+	}
+	if handSize > deckSize {
+		return nil, fmt.Errorf("handSize %d is greater than deckSize %d", handSize, deckSize)
+	}
+	if deckSize > 1<<26 {
+		return nil, fmt.Errorf("deckSize %d is impractically large", deckSize)
+	}
+	if RequiredEntropyBits(deckSize, handSize) > MaxHashBits {
+		return nil, fmt.Errorf("required entropy bits of deckSize %d and handSize %d is greater than %d", deckSize, handSize, MaxHashBits)
+	}
+
+	return &Dealer{
+		deckSize: deckSize,
+		handSize: handSize,
+	}, nil
+}
+
+// Deal shuffles a card deck and deals a hand of cards, using the given hashValue as the source of entropy.
+// The deck size and hand size are properties of the Dealer.
+// This function synchronously makes sequential calls to pick, one for each dealt card.
+// Each card is identified by an integer in the range [0, deckSize).
+// For example, for deckSize=128 and handSize=4 this function might call pick(14); pick(73); pick(119); pick(26).
+func (d *Dealer) Deal(hashValue uint64, pick func(int)) {
+	// 15 is the largest possible value of handSize
+	var remainders [15]int
+
+	for i := 0; i < d.handSize; i++ {
+		hashValueNext := hashValue / uint64(d.deckSize-i)
+		remainders[i] = int(hashValue - uint64(d.deckSize-i)*hashValueNext)
+		hashValue = hashValueNext
+	}
+
+	for i := 0; i < d.handSize; i++ {
+		card := remainders[i]
+		for j := i; j > 0; j-- {
+			if card >= remainders[j-1] {
+				card++
+			}
+		}
+		pick(card)
+	}
+}
+
+// DealIntoHand shuffles and deals according to the Dealer's parameters,
+// using the given hashValue as the source of entropy and then
+// returns the dealt cards as a slice of `int`.
+// If `hand` has the correct length as Dealer's handSize, it will be used as-is and no allocations will be made.
+// If `hand` is nil or too small, it will be extended (performing an allocation).
+// If `hand` is too large, a sub-slice will be returned.
+func (d *Dealer) DealIntoHand(hashValue uint64, hand []int) []int {
+	h := hand[:0]
+	d.Deal(hashValue, func(card int) { h = append(h, card) })
+	return h
+}

vendor/k8s.io/apiserver/pkg/util/webhook/authentication.go

@@ -26,6 +26,8 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
+	egressselector "k8s.io/apiserver/pkg/server/egressselector"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
 	clientcmdapi "k8s.io/client-go/tools/clientcmd/api"
@@ -38,6 +40,7 @@ type AuthenticationInfoResolverWrapper func(AuthenticationInfoResolver) Authenti
 // NewDefaultAuthenticationInfoResolverWrapper builds a default authn resolver wrapper
 func NewDefaultAuthenticationInfoResolverWrapper(
 	proxyTransport *http.Transport,
+	egressSelector *egressselector.EgressSelector,
 	kubeapiserverClientConfig *rest.Config) AuthenticationInfoResolverWrapper {
 
 	webhookAuthResolverWrapper := func(delegate AuthenticationInfoResolver) AuthenticationInfoResolver {
@@ -46,7 +49,23 @@ func NewDefaultAuthenticationInfoResolverWrapper(
 				if hostPort == "kubernetes.default.svc:443" {
 					return kubeapiserverClientConfig, nil
 				}
-				return delegate.ClientConfigFor(hostPort)
+				ret, err := delegate.ClientConfigFor(hostPort)
+				if err != nil {
+					return nil, err
+				}
+
+				if egressSelector != nil {
+					networkContext := egressselector.Master.AsNetworkContext()
+					var egressDialer utilnet.DialFunc
+					egressDialer, err = egressSelector.Lookup(networkContext)
+
+					if err != nil {
+						return nil, err
+					}
+
+					ret.Dial = egressDialer
+				}
+				return ret, nil
 			},
 			ClientConfigForServiceFunc: func(serviceName, serviceNamespace string, servicePort int) (*rest.Config, error) {
 				if serviceName == "kubernetes" && serviceNamespace == corev1.NamespaceDefault && servicePort == 443 {
@@ -56,10 +75,20 @@ func NewDefaultAuthenticationInfoResolverWrapper(
 				if err != nil {
 					return nil, err
 				}
-				if proxyTransport != nil && proxyTransport.DialContext != nil {
+
+				if egressSelector != nil {
+					networkContext := egressselector.Cluster.AsNetworkContext()
+					var egressDialer utilnet.DialFunc
+					egressDialer, err = egressSelector.Lookup(networkContext)
+					if err != nil {
+						return nil, err
+					}
+
+					ret.Dial = egressDialer
+				} else if proxyTransport != nil && proxyTransport.DialContext != nil {
 					ret.Dial = proxyTransport.DialContext
 				}
-				return ret, err
+				return ret, nil
 			},
 		}
 	}

vendor/k8s.io/apiserver/pkg/util/webhook/webhook.go

@@ -27,6 +27,7 @@ import (
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
 	"k8s.io/apimachinery/pkg/util/net"
+	utilnet "k8s.io/apimachinery/pkg/util/net"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -61,11 +62,11 @@ func DefaultShouldRetry(err error) bool {
 }
 
 // NewGenericWebhook creates a new GenericWebhook from the provided kubeconfig file.
-func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration) (*GenericWebhook, error) {
-	return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout)
+func NewGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) {
+	return newGenericWebhook(scheme, codecFactory, kubeConfigFile, groupVersions, initialBackoff, defaultRequestTimeout, customDial)
 }
 
-func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration) (*GenericWebhook, error) {
+func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFactory, kubeConfigFile string, groupVersions []schema.GroupVersion, initialBackoff, requestTimeout time.Duration, customDial utilnet.DialFunc) (*GenericWebhook, error) {
 	for _, groupVersion := range groupVersions {
 		if !scheme.IsVersionRegistered(groupVersion) {
 			return nil, fmt.Errorf("webhook plugin requires enabling extension resource: %s", groupVersion)
@@ -95,6 +96,8 @@ func newGenericWebhook(scheme *runtime.Scheme, codecFactory serializer.CodecFact
 	codec := codecFactory.LegacyCodec(groupVersions...)
 	clientConfig.ContentConfig.NegotiatedSerializer = serializer.NegotiatedSerializerWrapper(runtime.SerializerInfo{Serializer: codec})
 
+	clientConfig.Dial = customDial
+
 	restClient, err := rest.UnversionedRESTClientFor(clientConfig)
 	if err != nil {
 		return nil, err