vendor: revendor

This commit is contained in:
Sergiusz Urbaniak 2020-12-14 12:43:28 +01:00
parent 269295a414
commit 9f0440be0f
No known key found for this signature in database
GPG key ID: 44E6612519E13C39
669 changed files with 58447 additions and 20021 deletions

View file

@ -0,0 +1,96 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filterlatency
import (
"context"
"net/http"
"time"
utilclock "k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apiserver/pkg/endpoints/metrics"
apirequest "k8s.io/apiserver/pkg/endpoints/request"
)
type requestFilterRecordKeyType int
// requestFilterRecordKey is the context key for a request filter record struct.
const requestFilterRecordKey requestFilterRecordKeyType = iota
type requestFilterRecord struct {
name string
startedTimestamp time.Time
}
// withRequestFilterRecord attaches the given request filter record to the parent context.
func withRequestFilterRecord(parent context.Context, fr *requestFilterRecord) context.Context {
return apirequest.WithValue(parent, requestFilterRecordKey, fr)
}
// requestFilterRecordFrom returns the request filter record from the given context.
func requestFilterRecordFrom(ctx context.Context) *requestFilterRecord {
fr, _ := ctx.Value(requestFilterRecordKey).(*requestFilterRecord)
return fr
}
// TrackStarted measures the timestamp the given handler has started execution
// by attaching a handler to the chain.
func TrackStarted(handler http.Handler, name string) http.Handler {
return trackStarted(handler, name, utilclock.RealClock{})
}
// TrackCompleted measures the timestamp the given handler has completed execution and then
// it updates the corresponding metric with the filter latency duration.
func TrackCompleted(handler http.Handler) http.Handler {
return trackCompleted(handler, utilclock.RealClock{}, func(fr *requestFilterRecord, completedAt time.Time) {
metrics.RecordFilterLatency(fr.name, completedAt.Sub(fr.startedTimestamp))
})
}
func trackStarted(handler http.Handler, name string, clock utilclock.PassiveClock) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
ctx := r.Context()
if fr := requestFilterRecordFrom(ctx); fr != nil {
fr.name = name
fr.startedTimestamp = clock.Now()
handler.ServeHTTP(w, r)
return
}
fr := &requestFilterRecord{
name: name,
startedTimestamp: clock.Now(),
}
r = r.WithContext(withRequestFilterRecord(ctx, fr))
handler.ServeHTTP(w, r)
})
}
func trackCompleted(handler http.Handler, clock utilclock.PassiveClock, action func(*requestFilterRecord, time.Time)) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// The previous filter has just completed.
completedAt := clock.Now()
defer handler.ServeHTTP(w, r)
ctx := r.Context()
if fr := requestFilterRecordFrom(ctx); fr != nil {
action(fr, completedAt)
}
})
}

View file

@ -131,7 +131,11 @@ func createAuditEventAndAttachToContext(req *http.Request, policy policy.Checker
return req, nil, nil, nil
}
ev, err := audit.NewEventFromRequest(req, level, attribs)
requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(ctx)
if !ok {
requestReceivedTimestamp = time.Now()
}
ev, err := audit.NewEventFromRequest(req, requestReceivedTimestamp, level, attribs)
if err != nil {
return req, nil, nil, fmt.Errorf("failed to complete audit event from request: %v", err)
}

View file

@ -0,0 +1,40 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"net/http"
utilclock "k8s.io/apimachinery/pkg/util/clock"
"k8s.io/apiserver/pkg/endpoints/request"
)
// WithRequestReceivedTimestamp attaches the ReceivedTimestamp (the time the request reached
// the apiserver) to the context.
func WithRequestReceivedTimestamp(handler http.Handler) http.Handler {
return withRequestReceivedTimestampWithClock(handler, utilclock.RealClock{})
}
// The clock is passed as a parameter, handy for unit testing.
func withRequestReceivedTimestampWithClock(handler http.Handler, clock utilclock.PassiveClock) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
ctx := req.Context()
req = req.WithContext(request.WithReceivedTimestamp(ctx, clock.Now()))
handler.ServeHTTP(w, req)
})
}

View file

@ -0,0 +1,121 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package filters
import (
"errors"
"fmt"
"net/http"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/handlers/responsewriters"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/storageversion"
_ "k8s.io/component-base/metrics/prometheus/workqueue" // for workqueue metric registration
"k8s.io/klog/v2"
)
// WithStorageVersionPrecondition checks if the storage version barrier has
// completed, if not, it only passes the following API requests:
// 1. non-resource requests,
// 2. read requests,
// 3. write requests to the storageversion API,
// 4. create requests to the namespace API sent by apiserver itself,
// 5. write requests to the lease API in kube-system namespace,
// 6. resources whose StorageVersion is not pending update, including non-persisted resources.
func WithStorageVersionPrecondition(handler http.Handler, svm storageversion.Manager, s runtime.NegotiatedSerializer) http.Handler {
if svm == nil {
// TODO(roycaihw): switch to warning after the feature graduate to beta/GA
klog.V(2).Infof("Storage Version barrier is disabled")
return handler
}
return http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
if svm.Completed() {
handler.ServeHTTP(w, req)
return
}
ctx := req.Context()
requestInfo, found := request.RequestInfoFrom(ctx)
if !found {
responsewriters.InternalError(w, req, errors.New("no RequestInfo found in the context"))
return
}
// Allow non-resource requests
if !requestInfo.IsResourceRequest {
handler.ServeHTTP(w, req)
return
}
// Allow read requests
if requestInfo.Verb == "get" || requestInfo.Verb == "list" || requestInfo.Verb == "watch" {
handler.ServeHTTP(w, req)
return
}
// Allow writes to the storage version API
if requestInfo.APIGroup == "internal.apiserver.k8s.io" && requestInfo.Resource == "storageversions" {
handler.ServeHTTP(w, req)
return
}
// The system namespace is required for apiserver-identity lease to exist. Allow the apiserver
// itself to create namespaces.
// NOTE: with this exception, if the bootstrap client writes namespaces with a new version,
// and the upgraded apiserver dies before updating the StorageVersion for namespaces, the
// storage migrator won't be able to tell these namespaces are stored in a different version in etcd.
// Because the bootstrap client only creates system namespace and doesn't update them, this can
// only happen if the upgraded apiserver is the first apiserver that kicks off namespace creation,
// or if an upgraded server that joins an existing cluster has new system namespaces (other
// than kube-system, kube-public, kube-node-lease) that need to be created.
u, hasUser := request.UserFrom(ctx)
if requestInfo.APIGroup == "" && requestInfo.Resource == "namespaces" &&
requestInfo.Verb == "create" && hasUser &&
u.GetName() == user.APIServerUser && contains(u.GetGroups(), user.SystemPrivilegedGroup) {
handler.ServeHTTP(w, req)
return
}
// Allow writes to the lease API in kube-system. The storage version API depends on the
// apiserver-identity leases to operate. Leases in kube-system are either apiserver-identity
// lease (which gets garbage collected when stale) or leader-election leases (which gets
// periodically updated by system components). Both types of leases won't be stale in etcd.
if requestInfo.APIGroup == "coordination.k8s.io" && requestInfo.Resource == "leases" &&
requestInfo.Namespace == metav1.NamespaceSystem {
handler.ServeHTTP(w, req)
return
}
// If the resource's StorageVersion is not in the to-be-updated list, let it pass.
// Non-persisted resources are not in the to-be-updated list, so they will pass.
gr := schema.GroupResource{requestInfo.APIGroup, requestInfo.Resource}
if !svm.PendingUpdate(gr) {
handler.ServeHTTP(w, req)
return
}
gv := schema.GroupVersion{requestInfo.APIGroup, requestInfo.APIVersion}
responsewriters.ErrorNegotiated(apierrors.NewServiceUnavailable(fmt.Sprintf("wait for storage version registration to complete for resource: %v, last seen error: %v", gr, svm.LastUpdateError(gr))), s, gv, w, req)
})
}
func contains(s []string, e string) bool {
for _, a := range s {
if a == e {
return true
}
}
return false
}

View file

@ -30,7 +30,9 @@ import (
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/discovery"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/storageversion"
openapiproto "k8s.io/kube-openapi/pkg/util/proto"
)
@ -71,6 +73,7 @@ type APIGroupVersion struct {
Defaulter runtime.ObjectDefaulter
Linker runtime.SelfLinker
UnsafeConvertor runtime.ObjectConvertor
TypeConverter fieldmanager.TypeConverter
EquivalentResourceRegistry runtime.EquivalentResourceRegistry
@ -94,7 +97,7 @@ type APIGroupVersion struct {
// InstallREST registers the REST handlers (storage, watch, proxy and redirect) into a restful Container.
// It is expected that the provided path root prefix will serve all operations. Root MUST NOT end
// in a slash.
func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
func (g *APIGroupVersion) InstallREST(container *restful.Container) ([]*storageversion.ResourceInfo, error) {
prefix := path.Join(g.Root, g.GroupVersion.Group, g.GroupVersion.Version)
installer := &APIInstaller{
group: g,
@ -102,11 +105,24 @@ func (g *APIGroupVersion) InstallREST(container *restful.Container) error {
minRequestTimeout: g.MinRequestTimeout,
}
apiResources, ws, registrationErrors := installer.Install()
apiResources, resourceInfos, ws, registrationErrors := installer.Install()
versionDiscoveryHandler := discovery.NewAPIVersionHandler(g.Serializer, g.GroupVersion, staticLister{apiResources})
versionDiscoveryHandler.AddToWebService(ws)
container.Add(ws)
return utilerrors.NewAggregate(registrationErrors)
return removeNonPersistedResources(resourceInfos), utilerrors.NewAggregate(registrationErrors)
}
func removeNonPersistedResources(infos []*storageversion.ResourceInfo) []*storageversion.ResourceInfo {
var filtered []*storageversion.ResourceInfo
for _, info := range infos {
// if EncodingVersion is empty, then the apiserver does not
// need to register this resource via the storage version API,
// thus we can remove it.
if info != nil && len(info.EncodingVersion) > 0 {
filtered = append(filtered, info)
}
}
return filtered
}
// staticLister implements the APIResourceLister interface

View file

@ -44,6 +44,8 @@ import (
utiltrace "k8s.io/utils/trace"
)
var namespaceGVK = schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"}
func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Interface, includeName bool) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
// For performance tracking purposes.
@ -76,7 +78,6 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
ctx, cancel := context.WithTimeout(req.Context(), timeout)
defer cancel()
ctx = request.WithNamespace(ctx, namespace)
outputMediaType, _, err := negotiation.NegotiateOutputMediaType(req, scope.Serializer, scope)
if err != nil {
scope.err(err, w, req)
@ -128,17 +129,21 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
}
trace.Step("Conversion done")
// On create, get name from new object if unset
if len(name) == 0 {
_, name, _ = scope.Namer.ObjectName(obj)
}
if len(namespace) == 0 && *gvk == namespaceGVK {
namespace = name
}
ctx = request.WithNamespace(ctx, namespace)
ae := request.AuditEventFrom(ctx)
admit = admission.WithAudit(admit, ae)
audit.LogRequestObject(ae, obj, scope.Resource, scope.Subresource, scope.Serializer)
userInfo, _ := request.UserFrom(ctx)
// On create, get name from new object if unset
if len(name) == 0 {
_, name, _ = scope.Namer.ObjectName(obj)
}
trace.Step("About to store object in database")
admissionAttributes := admission.NewAttributesRecord(obj, nil, scope.Kind, namespace, name, scope.Resource, scope.Subresource, admission.Create, options, dryrun.IsDryRun(options.DryRun), userInfo)
requestFunc := func() (runtime.Object, error) {
@ -150,6 +155,8 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
options,
)
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finishRequest(timeout, func() (runtime.Object, error) {
if scope.FieldManager != nil {
liveObj, err := scope.Creater.New(scope.Kind)
@ -163,6 +170,8 @@ func createHandler(r rest.NamedCreater, scope *RequestScope, admit admission.Int
return nil, err
}
}
// Dedup owner references again after mutating admission happens
dedupOwnerReferencesAndAddWarning(obj, req.Context(), true)
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,
// it is safe to remove managedFields (which can be large) and try again.

View file

@ -141,7 +141,7 @@ func DeleteResource(r rest.GracefulDeleter, allowsOptions bool, scope *RequestSc
// that will break existing clients.
// Other cases where resource is not instantly deleted are: namespace deletion
// and pod graceful deletion.
if !wasDeleted && options.OrphanDependents != nil && *options.OrphanDependents == false {
if !wasDeleted && options.OrphanDependents != nil && !*options.OrphanDependents {
status = http.StatusAccepted
}
// if the rest.Deleter returns a nil object, fill out a status. Callers may return a valid

View file

@ -67,7 +67,7 @@ func (f *capManagersManager) capUpdateManagers(managed Managed) (newManaged Mana
// Gather all entries from updates
updaters := []string{}
for manager, fields := range managed.Fields() {
if fields.Applied() == false {
if !fields.Applied() {
updaters = append(updaters, manager)
}
}

View file

@ -27,7 +27,6 @@ import (
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"k8s.io/klog/v2"
openapiproto "k8s.io/kube-openapi/pkg/util/proto"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
"sigs.k8s.io/structured-merge-diff/v4/merge"
)
@ -67,50 +66,39 @@ type Manager interface {
// FieldManager updates the managed fields and merge applied
// configurations.
type FieldManager struct {
fieldManager Manager
fieldManager Manager
ignoreManagedFieldsFromRequestObject bool
}
// NewFieldManager creates a new FieldManager that decodes, manages, then re-encodes managedFields
// on update and apply requests.
func NewFieldManager(f Manager) *FieldManager {
return &FieldManager{f}
func NewFieldManager(f Manager, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
return &FieldManager{fieldManager: f, ignoreManagedFieldsFromRequestObject: ignoreManagedFieldsFromRequestObject}
}
// NewDefaultFieldManager creates a new FieldManager that merges apply requests
// and update managed fields for other types of requests.
func NewDefaultFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion) (*FieldManager, error) {
typeConverter, err := internal.NewTypeConverter(models, false)
if err != nil {
return nil, err
}
func NewDefaultFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool) (*FieldManager, error) {
f, err := NewStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
if err != nil {
return nil, fmt.Errorf("failed to create field manager: %v", err)
}
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
}
// NewDefaultCRDFieldManager creates a new FieldManager specifically for
// CRDs. This allows for the possibility of fields which are not defined
// in models, as well as having no models defined at all.
func NewDefaultCRDFieldManager(models openapiproto.Models, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, preserveUnknownFields bool) (_ *FieldManager, err error) {
var typeConverter internal.TypeConverter = internal.DeducedTypeConverter{}
if models != nil {
typeConverter, err = internal.NewTypeConverter(models, preserveUnknownFields)
if err != nil {
return nil, err
}
}
f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub, preserveUnknownFields)
func NewDefaultCRDFieldManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, hub schema.GroupVersion, ignoreManagedFieldsFromRequestObject bool) (_ *FieldManager, err error) {
f, err := NewCRDStructuredMergeManager(typeConverter, objectConverter, objectDefaulter, kind.GroupVersion(), hub)
if err != nil {
return nil, fmt.Errorf("failed to create field manager: %v", err)
}
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind), nil
return newDefaultFieldManager(f, typeConverter, objectConverter, objectCreater, kind, ignoreManagedFieldsFromRequestObject), nil
}
// newDefaultFieldManager is a helper function which wraps a Manager with certain default logic.
func newDefaultFieldManager(f Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind) *FieldManager {
func newDefaultFieldManager(f Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectCreater runtime.ObjectCreater, kind schema.GroupVersionKind, ignoreManagedFieldsFromRequestObject bool) *FieldManager {
f = NewStripMetaManager(f)
f = NewManagedFieldsUpdater(f)
f = NewBuildManagerInfoManager(f, kind.GroupVersion())
@ -119,36 +107,59 @@ func newDefaultFieldManager(f Manager, typeConverter internal.TypeConverter, obj
f = NewLastAppliedManager(f, typeConverter, objectConverter, kind.GroupVersion())
f = NewLastAppliedUpdater(f)
return NewFieldManager(f)
return NewFieldManager(f, ignoreManagedFieldsFromRequestObject)
}
func decodeLiveManagedFields(liveObj runtime.Object) (Managed, error) {
liveAccessor, err := meta.Accessor(liveObj)
if err != nil {
return nil, err
}
managed, err := internal.DecodeObjectManagedFields(liveAccessor.GetManagedFields())
if err != nil {
return internal.NewEmptyManaged(), nil
}
return managed, nil
}
func decodeManagedFields(liveObj, newObj runtime.Object, ignoreManagedFieldsFromRequestObject bool) (Managed, error) {
// We take the managedFields of the live object in case the request tries to
// manually set managedFields via a subresource.
if ignoreManagedFieldsFromRequestObject {
return decodeLiveManagedFields(liveObj)
}
// If the object doesn't have metadata, we should just return without trying to
// set the managedFields at all, so creates/updates/patches will work normally.
newAccessor, err := meta.Accessor(newObj)
if err != nil {
return nil, err
}
if isResetManagedFields(newAccessor.GetManagedFields()) {
return internal.NewEmptyManaged(), nil
}
managed, err := internal.DecodeObjectManagedFields(newAccessor.GetManagedFields())
// If the managed field is empty or we failed to decode it,
// let's try the live object. This is to prevent clients who
// don't understand managedFields from deleting it accidentally.
if err != nil || len(managed.Fields()) == 0 {
return decodeLiveManagedFields(liveObj)
}
return managed, nil
}
// Update is used when the object has already been merged (non-apply
// use-case), and simply updates the managed fields in the output
// object.
func (f *FieldManager) Update(liveObj, newObj runtime.Object, manager string) (object runtime.Object, err error) {
// If the object doesn't have metadata, we should just return without trying to
// set the managedFields at all, so creates/updates/patches will work normally.
newAccessor, err := meta.Accessor(newObj)
if err != nil {
return newObj, nil
}
// First try to decode the managed fields provided in the update,
// This is necessary to allow directly updating managed fields.
var managed Managed
if isResetManagedFields(newAccessor.GetManagedFields()) {
managed = internal.NewEmptyManaged()
} else if managed, err = internal.DecodeObjectManagedFields(newAccessor.GetManagedFields()); err != nil || len(managed.Fields()) == 0 {
liveAccessor, err := meta.Accessor(liveObj)
if err != nil {
return newObj, nil
}
// If the managed field is empty or we failed to decode it,
// let's try the live object. This is to prevent clients who
// don't understand managedFields from deleting it accidentally.
if managed, err = internal.DecodeObjectManagedFields(liveAccessor.GetManagedFields()); err != nil {
managed = internal.NewEmptyManaged()
}
managed, err := decodeManagedFields(liveObj, newObj, f.ignoreManagedFieldsFromRequestObject)
if err != nil {
return newObj, nil
}
internal.RemoveObjectManagedFields(liveObj)

View file

@ -30,12 +30,15 @@ import (
// definition's "extensions" map.
const groupVersionKindExtensionKey = "x-kubernetes-group-version-kind"
type gvkParser struct {
// GvkParser contains a Parser that allows introspecting the schema.
type GvkParser struct {
gvks map[schema.GroupVersionKind]string
parser typed.Parser
}
func (p *gvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
// Type returns a helper which can produce objects of the given type. Any
// errors are deferred until a further function is called.
func (p *GvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
typeName, ok := p.gvks[gvk]
if !ok {
return nil
@ -44,12 +47,15 @@ func (p *gvkParser) Type(gvk schema.GroupVersionKind) *typed.ParseableType {
return &t
}
func newGVKParser(models proto.Models, preserveUnknownFields bool) (*gvkParser, error) {
// NewGVKParser builds a GVKParser from a proto.Models. This
// will automatically find the proper version of the object, and the
// corresponding schema information.
func NewGVKParser(models proto.Models, preserveUnknownFields bool) (*GvkParser, error) {
typeSchema, err := schemaconv.ToSchemaWithPreserveUnknownFields(models, preserveUnknownFields)
if err != nil {
return nil, fmt.Errorf("failed to convert models to schema: %v", err)
}
parser := gvkParser{
parser := GvkParser{
gvks: map[schema.GroupVersionKind]string{},
}
parser.parser = typed.Parser{Schema: *typeSchema}

View file

@ -25,14 +25,13 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"sigs.k8s.io/structured-merge-diff/v4/fieldpath"
"sigs.k8s.io/structured-merge-diff/v4/merge"
)
type lastAppliedManager struct {
fieldManager Manager
typeConverter internal.TypeConverter
typeConverter TypeConverter
objectConverter runtime.ObjectConvertor
groupVersion schema.GroupVersion
}
@ -41,7 +40,7 @@ var _ Manager = &lastAppliedManager{}
// NewLastAppliedManager converts the client-side apply annotation to
// server-side apply managed fields
func NewLastAppliedManager(fieldManager Manager, typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, groupVersion schema.GroupVersion) Manager {
func NewLastAppliedManager(fieldManager Manager, typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, groupVersion schema.GroupVersion) Manager {
return &lastAppliedManager{
fieldManager: fieldManager,
typeConverter: typeConverter,

View file

@ -44,6 +44,7 @@ func NewManagedFieldsUpdater(fieldManager Manager) Manager {
// Update implements Manager.
func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Managed, manager string) (runtime.Object, Managed, error) {
self := "current-operation"
formerSet := managed.Fields()[manager]
object, managed, err := f.fieldManager.Update(liveObj, newObj, managed, self)
if err != nil {
return object, managed, err
@ -54,12 +55,15 @@ func (f *managedFieldsUpdater) Update(liveObj, newObj runtime.Object, managed Ma
if vs, ok := managed.Fields()[self]; ok {
delete(managed.Fields(), self)
managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
if previous, ok := managed.Fields()[manager]; ok {
managed.Fields()[manager] = fieldpath.NewVersionedSet(vs.Set().Union(previous.Set()), vs.APIVersion(), vs.Applied())
} else {
managed.Fields()[manager] = vs
}
// Update the time only if the manager's fieldSet has changed.
if formerSet == nil || !managed.Fields()[manager].Set().Equals(formerSet.Set()) {
managed.Times()[manager] = &metav1.Time{Time: time.Now().UTC()}
}
}
return object, managed, nil

View file

@ -29,7 +29,7 @@ import (
)
type structuredMergeManager struct {
typeConverter internal.TypeConverter
typeConverter TypeConverter
objectConverter runtime.ObjectConvertor
objectDefaulter runtime.ObjectDefaulter
groupVersion schema.GroupVersion
@ -41,7 +41,7 @@ var _ Manager = &structuredMergeManager{}
// NewStructuredMergeManager creates a new Manager that merges apply requests
// and update managed fields for other types of requests.
func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
func NewStructuredMergeManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (Manager, error) {
return &structuredMergeManager{
typeConverter: typeConverter,
objectConverter: objectConverter,
@ -49,7 +49,7 @@ func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConve
groupVersion: gv,
hubVersion: hub,
updater: merge.Updater{
Converter: internal.NewVersionConverter(typeConverter, objectConverter, hub), // This is the converter provided to SMD from k8s
Converter: newVersionConverter(typeConverter, objectConverter, hub), // This is the converter provided to SMD from k8s
},
}, nil
}
@ -57,7 +57,7 @@ func NewStructuredMergeManager(typeConverter internal.TypeConverter, objectConve
// NewCRDStructuredMergeManager creates a new Manager specifically for
// CRDs. This allows for the possibility of fields which are not defined
// in models, as well as having no models defined at all.
func NewCRDStructuredMergeManager(typeConverter internal.TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion, preserveUnknownFields bool) (_ Manager, err error) {
func NewCRDStructuredMergeManager(typeConverter TypeConverter, objectConverter runtime.ObjectConvertor, objectDefaulter runtime.ObjectDefaulter, gv schema.GroupVersion, hub schema.GroupVersion) (_ Manager, err error) {
return &structuredMergeManager{
typeConverter: typeConverter,
objectConverter: objectConverter,
@ -65,7 +65,7 @@ func NewCRDStructuredMergeManager(typeConverter internal.TypeConverter, objectCo
groupVersion: gv,
hubVersion: hub,
updater: merge.Updater{
Converter: internal.NewCRDVersionConverter(typeConverter, objectConverter, hub),
Converter: newCRDVersionConverter(typeConverter, objectConverter, hub),
},
}, nil
}

View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
package fieldmanager
import (
"fmt"
@ -22,6 +22,7 @@ import (
"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager/internal"
"k8s.io/kube-openapi/pkg/util/proto"
"sigs.k8s.io/structured-merge-diff/v4/typed"
"sigs.k8s.io/structured-merge-diff/v4/value"
@ -64,7 +65,7 @@ func (DeducedTypeConverter) TypedToObject(value *typed.TypedValue) (runtime.Obje
}
type typeConverter struct {
parser *gvkParser
parser *internal.GvkParser
}
var _ TypeConverter = &typeConverter{}
@ -73,7 +74,7 @@ var _ TypeConverter = &typeConverter{}
// will automatically find the proper version of the object, and the
// corresponding schema information.
func NewTypeConverter(models proto.Models, preserveUnknownFields bool) (TypeConverter, error) {
parser, err := newGVKParser(models, preserveUnknownFields)
parser, err := internal.NewGVKParser(models, preserveUnknownFields)
if err != nil {
return nil, err
}

View file

@ -14,7 +14,7 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package internal
package fieldmanager
import (
"k8s.io/apimachinery/pkg/runtime"
@ -35,7 +35,7 @@ type versionConverter struct {
var _ merge.Converter = &versionConverter{}
// NewVersionConverter builds a VersionConverter from a TypeConverter and an ObjectConvertor.
func NewVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
func newVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
return &versionConverter{
typeConverter: t,
objectConvertor: o,
@ -49,7 +49,7 @@ func NewVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.Gr
}
// NewCRDVersionConverter builds a VersionConverter for CRDs from a TypeConverter and an ObjectConvertor.
func NewCRDVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
func newCRDVersionConverter(t TypeConverter, o runtime.ObjectConvertor, h schema.GroupVersion) merge.Converter {
return &versionConverter{
typeConverter: t,
objectConvertor: o,

View file

@ -38,6 +38,7 @@ import (
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/strategicpatch"
"k8s.io/apimachinery/pkg/util/validation/field"
"k8s.io/apimachinery/pkg/util/yaml"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authorization/authorizer"
@ -49,7 +50,6 @@ import (
"k8s.io/apiserver/pkg/util/dryrun"
utilfeature "k8s.io/apiserver/pkg/util/feature"
utiltrace "k8s.io/utils/trace"
"sigs.k8s.io/yaml"
)
const (
@ -576,9 +576,14 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
default:
return nil, false, fmt.Errorf("%v: unimplemented patch type", p.patchType)
}
dedupOwnerReferencesTransformer := func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
// Dedup owner references after mutating admission happens
dedupOwnerReferencesAndAddWarning(obj, ctx, true)
return obj, nil
}
wasCreated := false
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission)
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission, dedupOwnerReferencesTransformer)
requestFunc := func() (runtime.Object, error) {
// Pass in UpdateOptions to override UpdateStrategy.AllowUpdateOnCreate
options := patchToUpdateOptions(p.options)
@ -592,11 +597,15 @@ func (p *patcher) patchResource(ctx context.Context, scope *RequestScope) (runti
// it is safe to remove managedFields (which can be large) and try again.
if isTooLargeError(err) && p.patchType != types.ApplyPatchType {
if _, accessorErr := meta.Accessor(p.restPatcher.New()); accessorErr == nil {
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil, p.applyPatch, p.applyAdmission, func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
accessor, _ := meta.Accessor(obj)
accessor.SetManagedFields(nil)
return obj, nil
})
p.updatedObjectInfo = rest.DefaultUpdatedObjectInfo(nil,
p.applyPatch,
p.applyAdmission,
dedupOwnerReferencesTransformer,
func(_ context.Context, obj, _ runtime.Object) (runtime.Object, error) {
accessor, _ := meta.Accessor(obj)
accessor.SetManagedFields(nil)
return obj, nil
})
result, err = requestFunc()
}
}

View file

@ -96,9 +96,11 @@ func SerializeObject(mediaType string, encoder runtime.Encoder, hw http.Response
err := encoder.Encode(object, w)
if err == nil {
err = w.Close()
if err == nil {
return
if err != nil {
// we cannot write an error to the writer anymore as the Encode call was successful.
utilruntime.HandleError(fmt.Errorf("apiserver was unable to close cleanly the response writer: %v", err))
}
return
}
// make a best effort to write the object if a failure is detected

View file

@ -31,12 +31,14 @@ import (
grpccodes "google.golang.org/grpc/codes"
grpcstatus "google.golang.org/grpc/status"
apiequality "k8s.io/apimachinery/pkg/api/equality"
"k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
metav1beta1 "k8s.io/apimachinery/pkg/apis/meta/v1beta1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apiserver/pkg/admission"
"k8s.io/apiserver/pkg/authorization/authorizer"
"k8s.io/apiserver/pkg/endpoints/handlers/fieldmanager"
@ -46,9 +48,21 @@ import (
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
utilfeature "k8s.io/apiserver/pkg/util/feature"
"k8s.io/apiserver/pkg/warning"
"k8s.io/klog/v2"
)
const (
// DuplicateOwnerReferencesWarningFormat is the warning that a client receives when a create/update request contains
// duplicate owner reference entries.
DuplicateOwnerReferencesWarningFormat = ".metadata.ownerReferences contains duplicate entries; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: %v"
// DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat indicates the duplication was observed
// after mutating admission.
// NOTE: For CREATE and UPDATE requests the API server dedups both before and after mutating admission.
// For PATCH request the API server only dedups after mutating admission.
DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat = ".metadata.ownerReferences contains duplicate entries after mutating admission happens; API server dedups owner references in 1.20+, and may reject such requests as early as 1.24; please fix your requests; duplicate UID(s) observed: %v"
)
// RequestScope encapsulates common fields across all RESTful handler methods.
type RequestScope struct {
Namer ScopeNamer
@ -323,11 +337,79 @@ func checkName(obj runtime.Object, name, namespace string, namer ScopeNamer) err
return nil
}
// dedupOwnerReferences dedups owner references over the entire entry.
// NOTE: We don't know enough about the existing cases of owner references
// sharing the same UID but different fields. Nor do we know what might break.
// In the future we may just dedup/reject owner references with the same UID.
func dedupOwnerReferences(refs []metav1.OwnerReference) ([]metav1.OwnerReference, []string) {
var result []metav1.OwnerReference
var duplicates []string
seen := make(map[types.UID]struct{})
for _, ref := range refs {
_, ok := seen[ref.UID]
// Short-circuit if we haven't seen the UID before. Otherwise
// check the entire list we have so far.
if !ok || !hasOwnerReference(result, ref) {
seen[ref.UID] = struct{}{}
result = append(result, ref)
} else {
duplicates = append(duplicates, string(ref.UID))
}
}
return result, duplicates
}
// hasOwnerReference returns true if refs has an item equal to ref. The function
// focuses on semantic equality instead of memory equality, to catch duplicates
// with different pointer addresses. The function uses apiequality.Semantic
// instead of implementing its own comparison, to tolerate API changes to
// metav1.OwnerReference.
// NOTE: This is expensive, but we accept it because we've made sure it only
// happens to owner references containing duplicate UIDs, plus typically the
// number of items in the list should be small.
func hasOwnerReference(refs []metav1.OwnerReference, ref metav1.OwnerReference) bool {
for _, r := range refs {
if apiequality.Semantic.DeepEqual(r, ref) {
return true
}
}
return false
}
// dedupOwnerReferencesAndAddWarning dedups owner references in the object metadata.
// If duplicates are found, the function records a warning to the provided context.
func dedupOwnerReferencesAndAddWarning(obj runtime.Object, requestContext context.Context, afterMutatingAdmission bool) {
accessor, err := meta.Accessor(obj)
if err != nil {
// The object doesn't have metadata. Nothing we need to do here.
return
}
refs := accessor.GetOwnerReferences()
deduped, duplicates := dedupOwnerReferences(refs)
if len(duplicates) > 0 {
// NOTE: For CREATE and UPDATE requests the API server dedups both before and after mutating admission.
// For PATCH request the API server only dedups after mutating admission.
format := DuplicateOwnerReferencesWarningFormat
if afterMutatingAdmission {
format = DuplicateOwnerReferencesAfterMutatingAdmissionWarningFormat
}
warning.AddWarning(requestContext, "", fmt.Sprintf(format,
strings.Join(duplicates, ", ")))
accessor.SetOwnerReferences(deduped)
}
}
// setObjectSelfLink sets the self link of an object as needed.
// TODO: remove the need for the namer LinkSetters by requiring objects implement either Object or List
// interfaces
func setObjectSelfLink(ctx context.Context, obj runtime.Object, req *http.Request, namer ScopeNamer) error {
if utilfeature.DefaultFeatureGate.Enabled(features.RemoveSelfLink) {
// Ensure that for empty lists we don't return <nil> items.
if meta.IsListType(obj) && meta.LenList(obj) == 0 {
if err := meta.SetList(obj, []runtime.Object{}); err != nil {
return err
}
}
return nil
}

View file

@ -153,6 +153,11 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
}
return newObj, nil
})
transformers = append(transformers, func(ctx context.Context, newObj, oldObj runtime.Object) (runtime.Object, error) {
// Dedup owner references again after mutating admission happens
dedupOwnerReferencesAndAddWarning(newObj, req.Context(), true)
return newObj, nil
})
}
createAuthorizerAttributes := authorizer.AttributesRecord{
@ -188,6 +193,8 @@ func UpdateResource(r rest.Updater, scope *RequestScope, admit admission.Interfa
wasCreated = created
return obj, err
}
// Dedup owner references before updating managed fields
dedupOwnerReferencesAndAddWarning(obj, req.Context(), false)
result, err := finishRequest(timeout, func() (runtime.Object, error) {
result, err := requestFunc()
// If the object wasn't committed to storage because it's serialized size was too large,

View file

@ -43,6 +43,7 @@ import (
utilwarning "k8s.io/apiserver/pkg/endpoints/warning"
"k8s.io/apiserver/pkg/features"
"k8s.io/apiserver/pkg/registry/rest"
"k8s.io/apiserver/pkg/storageversion"
utilfeature "k8s.io/apiserver/pkg/util/feature"
versioninfo "k8s.io/component-base/version"
)
@ -94,8 +95,9 @@ var toDiscoveryKubeVerb = map[string]string{
}
// Install handlers for API resources.
func (a *APIInstaller) Install() ([]metav1.APIResource, *restful.WebService, []error) {
func (a *APIInstaller) Install() ([]metav1.APIResource, []*storageversion.ResourceInfo, *restful.WebService, []error) {
var apiResources []metav1.APIResource
var resourceInfos []*storageversion.ResourceInfo
var errors []error
ws := a.newWebService()
@ -108,15 +110,18 @@ func (a *APIInstaller) Install() ([]metav1.APIResource, *restful.WebService, []e
}
sort.Strings(paths)
for _, path := range paths {
apiResource, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
apiResource, resourceInfo, err := a.registerResourceHandlers(path, a.group.Storage[path], ws)
if err != nil {
errors = append(errors, fmt.Errorf("error in registering resource: %s, %v", path, err))
}
if apiResource != nil {
apiResources = append(apiResources, *apiResource)
}
if resourceInfo != nil {
resourceInfos = append(resourceInfos, resourceInfo)
}
}
return apiResources, ws, errors
return apiResources, resourceInfos, ws, errors
}
// newWebService creates a new restful webservice with the api installer's prefix and version.
@ -182,7 +187,7 @@ func GetResourceKind(groupVersion schema.GroupVersion, storage rest.Storage, typ
return fqKindToRegister, nil
}
func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, error) {
func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storage, ws *restful.WebService) (*metav1.APIResource, *storageversion.ResourceInfo, error) {
admit := a.group.Admit
optionsExternalVersion := a.group.GroupVersion
@ -192,19 +197,19 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
resource, subresource, err := splitSubresource(path)
if err != nil {
return nil, err
return nil, nil, err
}
group, version := a.group.GroupVersion.Group, a.group.GroupVersion.Version
fqKindToRegister, err := GetResourceKind(a.group.GroupVersion, storage, a.group.Typer)
if err != nil {
return nil, err
return nil, nil, err
}
versionedPtr, err := a.group.Creater.New(fqKindToRegister)
if err != nil {
return nil, err
return nil, nil, err
}
defaultVersionedObject := indirectArbitraryPointer(versionedPtr)
kind := fqKindToRegister.Kind
@ -215,18 +220,18 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if isSubresource {
parentStorage, ok := a.group.Storage[resource]
if !ok {
return nil, fmt.Errorf("missing parent storage: %q", resource)
return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
}
scoper, ok := parentStorage.(rest.Scoper)
if !ok {
return nil, fmt.Errorf("%q must implement scoper", resource)
return nil, nil, fmt.Errorf("%q must implement scoper", resource)
}
namespaceScoped = scoper.NamespaceScoped()
} else {
scoper, ok := storage.(rest.Scoper)
if !ok {
return nil, fmt.Errorf("%q must implement scoper", resource)
return nil, nil, fmt.Errorf("%q must implement scoper", resource)
}
namespaceScoped = scoper.NamespaceScoped()
}
@ -255,7 +260,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
versionedExportOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ExportOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
if isNamedCreater {
@ -267,30 +272,30 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
list := lister.NewList()
listGVKs, _, err := a.group.Typer.ObjectKinds(list)
if err != nil {
return nil, err
return nil, nil, err
}
versionedListPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind(listGVKs[0].Kind))
if err != nil {
return nil, err
return nil, nil, err
}
versionedList = indirectArbitraryPointer(versionedListPtr)
}
versionedListOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("ListOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedCreateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("CreateOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedPatchOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("PatchOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedUpdateOptions, err := a.group.Creater.New(optionsExternalVersion.WithKind("UpdateOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
var versionedDeleteOptions runtime.Object
@ -299,7 +304,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if isGracefulDeleter {
versionedDeleteOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind("DeleteOptions"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedDeleterObject = indirectArbitraryPointer(versionedDeleteOptions)
@ -310,7 +315,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
versionedStatusPtr, err := a.group.Creater.New(optionsExternalVersion.WithKind("Status"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedStatus := indirectArbitraryPointer(versionedStatusPtr)
var (
@ -323,14 +328,14 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
getOptions, getSubpath, _ = getterWithOptions.NewGetOptions()
getOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(getOptions)
if err != nil {
return nil, err
return nil, nil, err
}
getOptionsInternalKind = getOptionsInternalKinds[0]
versionedGetOptions, err = a.group.Creater.New(a.group.GroupVersion.WithKind(getOptionsInternalKind.Kind))
if err != nil {
versionedGetOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(getOptionsInternalKind.Kind))
if err != nil {
return nil, err
return nil, nil, err
}
}
isGetter = true
@ -340,7 +345,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if isWatcher {
versionedWatchEventPtr, err := a.group.Creater.New(a.group.GroupVersion.WithKind("WatchEvent"))
if err != nil {
return nil, err
return nil, nil, err
}
versionedWatchEvent = indirectArbitraryPointer(versionedWatchEventPtr)
}
@ -356,7 +361,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if connectOptions != nil {
connectOptionsInternalKinds, _, err := a.group.Typer.ObjectKinds(connectOptions)
if err != nil {
return nil, err
return nil, nil, err
}
connectOptionsInternalKind = connectOptionsInternalKinds[0]
@ -364,7 +369,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if err != nil {
versionedConnectOptions, err = a.group.Creater.New(optionsExternalVersion.WithKind(connectOptionsInternalKind.Kind))
if err != nil {
return nil, err
return nil, nil, err
}
}
}
@ -388,7 +393,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
tableProvider, isTableProvider := storage.(rest.TableConvertor)
if isLister && !isTableProvider {
// All listers must implement TableProvider
return nil, fmt.Errorf("%q must implement TableConvertor", resource)
return nil, nil, fmt.Errorf("%q must implement TableConvertor", resource)
}
var apiResource metav1.APIResource
@ -398,7 +403,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
versioner := storageVersionProvider.StorageVersion()
gvk, err := getStorageVersionKind(versioner, storage, a.group.Typer)
if err != nil {
return nil, err
return nil, nil, err
}
apiResource.StorageVersionHash = discovery.StorageVersionHash(gvk.Group, gvk.Version, gvk.Kind)
}
@ -506,6 +511,30 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
}
var resourceInfo *storageversion.ResourceInfo
if utilfeature.DefaultFeatureGate.Enabled(features.StorageVersionAPI) &&
utilfeature.DefaultFeatureGate.Enabled(features.APIServerIdentity) &&
isStorageVersionProvider &&
storageVersionProvider.StorageVersion() != nil {
versioner := storageVersionProvider.StorageVersion()
encodingGVK, err := getStorageVersionKind(versioner, storage, a.group.Typer)
if err != nil {
return nil, nil, err
}
resourceInfo = &storageversion.ResourceInfo{
GroupResource: schema.GroupResource{
Group: a.group.GroupVersion.Group,
Resource: apiResource.Name,
},
EncodingVersion: encodingGVK.GroupVersion().String(),
// We record EquivalentResourceMapper first instead of calculate
// DecodableVersions immediately because API installation must
// be completed first for us to know equivalent APIs
EquivalentResourceMapper: a.group.EquivalentResourceRegistry,
}
}
// Create Routes for the actions.
// TODO: Add status documentation using Returns()
// Errors (see api/errors/errors.go as well as go-restful router):
@ -525,7 +554,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
for _, s := range a.group.Serializer.SupportedMediaTypes() {
if len(s.MediaTypeSubType) == 0 || len(s.MediaTypeType) == 0 {
return nil, fmt.Errorf("all serializers in the group Serializer must have MediaTypeType and MediaTypeSubType set: %s", s.MediaType)
return nil, nil, fmt.Errorf("all serializers in the group Serializer must have MediaTypeType and MediaTypeSubType set: %s", s.MediaType)
}
}
mediaTypes, streamMediaTypes := negotiation.MediaTypesForSerializer(a.group.Serializer)
@ -564,15 +593,16 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
if a.group.OpenAPIModels != nil && utilfeature.DefaultFeatureGate.Enabled(features.ServerSideApply) {
reqScope.FieldManager, err = fieldmanager.NewDefaultFieldManager(
a.group.OpenAPIModels,
a.group.TypeConverter,
a.group.UnsafeConvertor,
a.group.Defaulter,
a.group.Creater,
fqKindToRegister,
reqScope.HubGroupVersion,
isSubresource,
)
if err != nil {
return nil, fmt.Errorf("failed to create field manager: %v", err)
return nil, nil, fmt.Errorf("failed to create field manager: %v", err)
}
}
for _, action := range actions {
@ -593,6 +623,9 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
requestScope = "resource"
operationSuffix = operationSuffix + "WithPath"
}
if strings.Index(action.Path, "/{name}") != -1 || action.Verb == "POST" {
requestScope = "resource"
}
if action.AllNamespaces {
requestScope = "cluster"
operationSuffix = operationSuffix + "ForAllNamespaces"
@ -604,7 +637,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
kubeVerbs[kubeVerb] = struct{}{}
}
} else {
return nil, fmt.Errorf("unknown action verb for discovery: %s", action.Verb)
return nil, nil, fmt.Errorf("unknown action verb for discovery: %s", action.Verb)
}
routes := []*restful.RouteBuilder{}
@ -613,12 +646,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
if isSubresource {
parentStorage, ok := a.group.Storage[resource]
if !ok {
return nil, fmt.Errorf("missing parent storage: %q", resource)
return nil, nil, fmt.Errorf("missing parent storage: %q", resource)
}
fqParentKind, err := GetResourceKind(a.group.GroupVersion, parentStorage, a.group.Typer)
if err != nil {
return nil, err
return nil, nil, err
}
kind = fqParentKind.Kind
}
@ -677,12 +710,12 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Writes(producedObject)
if isGetterWithOptions {
if err := AddObjectParams(ws, route, versionedGetOptions); err != nil {
return nil, err
return nil, nil, err
}
}
if isExporter {
if err := AddObjectParams(ws, route, versionedExportOptions); err != nil {
return nil, err
return nil, nil, err
}
}
addParams(route, action.Params)
@ -704,7 +737,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusOK, "OK", versionedList).
Writes(versionedList)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
return nil, nil, err
}
switch {
case isLister && isWatcher:
@ -743,7 +776,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedUpdateOptions); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -774,7 +807,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Reads(metav1.Patch{}).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedPatchOptions); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -807,7 +840,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Reads(defaultVersionedObject).
Writes(producedObject)
if err := AddObjectParams(ws, route, versionedCreateOptions); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -837,7 +870,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
route.Reads(versionedDeleterObject)
route.ParameterNamed("body").Required(false)
if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
return nil, err
return nil, nil, err
}
}
addParams(route, action.Params)
@ -862,11 +895,11 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
route.Reads(versionedDeleterObject)
route.ParameterNamed("body").Required(false)
if err := AddObjectParams(ws, route, versionedDeleteOptions); err != nil {
return nil, err
return nil, nil, err
}
}
if err := AddObjectParams(ws, route, versionedListOptions, "watch", "allowWatchBookmarks"); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -889,7 +922,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -912,7 +945,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Returns(http.StatusOK, "OK", versionedWatchEvent).
Writes(versionedWatchEvent)
if err := AddObjectParams(ws, route, versionedListOptions); err != nil {
return nil, err
return nil, nil, err
}
addParams(route, action.Params)
routes = append(routes, route)
@ -939,7 +972,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
Writes(connectProducedObject)
if versionedConnectOptions != nil {
if err := AddObjectParams(ws, route, versionedConnectOptions); err != nil {
return nil, err
return nil, nil, err
}
}
addParams(route, action.Params)
@ -953,7 +986,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
}
}
default:
return nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)
return nil, nil, fmt.Errorf("unrecognized action verb: %s", action.Verb)
}
for _, route := range routes {
route.Metadata(ROUTE_META_GVK, metav1.GroupVersionKind{
@ -989,7 +1022,7 @@ func (a *APIInstaller) registerResourceHandlers(path string, storage rest.Storag
// Record the existence of the GVR and the corresponding GVK
a.group.EquivalentResourceRegistry.RegisterKindFor(reqScope.Resource, reqScope.Subresource, fqKindToRegister)
return &apiResource, nil
return &apiResource, resourceInfo, nil
}
// indirectArbitraryPointer returns *ptrToObject for an arbitrary pointer

View file

@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/types"
utilsets "k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apiserver/pkg/audit"
"k8s.io/apiserver/pkg/authentication/user"
"k8s.io/apiserver/pkg/endpoints/request"
"k8s.io/apiserver/pkg/features"
utilfeature "k8s.io/apiserver/pkg/util/feature"
@ -185,6 +186,36 @@ var (
},
[]string{"verb", "group", "version", "resource", "subresource", "scope", "component", "code"},
)
apiSelfRequestCounter = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_selfrequest_total",
Help: "Counter of apiserver self-requests broken out for each verb, API resource and subresource.",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "resource", "subresource"},
)
requestFilterDuration = compbasemetrics.NewHistogramVec(
&compbasemetrics.HistogramOpts{
Name: "apiserver_request_filter_duration_seconds",
Help: "Request filter latency distribution in seconds, for each filter type",
Buckets: []float64{0.0001, 0.0003, 0.001, 0.003, 0.01, 0.03, 0.1, 0.3, 1.0, 5.0},
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"filter"},
)
// requestAbortsTotal is a number of aborted requests with http.ErrAbortHandler
requestAbortsTotal = compbasemetrics.NewCounterVec(
&compbasemetrics.CounterOpts{
Name: "apiserver_request_aborts_total",
Help: "Number of requests which apiserver aborted possibly due to a timeout, for each group, version, verb, resource, subresource and scope",
StabilityLevel: compbasemetrics.ALPHA,
},
[]string{"verb", "group", "version", "resource", "subresource", "scope"},
)
kubectlExeRegexp = regexp.MustCompile(`^.*((?i:kubectl\.exe))`)
metrics = []resettableCollector{
@ -201,6 +232,9 @@ var (
currentInflightRequests,
currentInqueueRequests,
requestTerminationsTotal,
apiSelfRequestCounter,
requestFilterDuration,
requestAbortsTotal,
}
// these are the known (e.g. whitelisted/known) content types which we will report for
@ -290,6 +324,26 @@ func UpdateInflightRequestMetrics(phase string, nonmutating, mutating int) {
}
}
func RecordFilterLatency(name string, elapsed time.Duration) {
requestFilterDuration.WithLabelValues(name).Observe(elapsed.Seconds())
}
// RecordRequestAbort records that the request was aborted possibly due to a timeout.
func RecordRequestAbort(req *http.Request, requestInfo *request.RequestInfo) {
if requestInfo == nil {
requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
}
scope := CleanScope(requestInfo)
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
resource := requestInfo.Resource
subresource := requestInfo.Subresource
group := requestInfo.APIGroup
version := requestInfo.APIVersion
requestAbortsTotal.WithLabelValues(reportedVerb, group, version, resource, subresource, scope).Inc()
}
// RecordRequestTermination records that the request was terminated early as part of a resource
// preservation or apiserver self-defense mechanism (e.g. timeouts, maxinflight throttling,
// proxyHandler errors). RecordRequestTermination should only be called zero or one times
@ -299,20 +353,17 @@ func RecordRequestTermination(req *http.Request, requestInfo *request.RequestInf
requestInfo = &request.RequestInfo{Verb: req.Method, Path: req.URL.Path}
}
scope := CleanScope(requestInfo)
// We don't use verb from <requestInfo>, as for the healthy path
// MonitorRequest is called from InstrumentRouteFunc which is registered
// in installer.go with predefined list of verbs (different than those
// translated to RequestInfo).
// We don't use verb from <requestInfo>, as this may be propagated from
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
verb := canonicalVerb(strings.ToUpper(req.Method), scope)
// set verbs to a bounded set of known and expected verbs
if !validRequestMethods.Has(verb) {
verb = OtherRequestMethod
}
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
if requestInfo.IsResourceRequest {
requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
requestTerminationsTotal.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component, codeToString(code)).Inc()
} else {
requestTerminationsTotal.WithLabelValues(cleanVerb(verb, req), "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
requestTerminationsTotal.WithLabelValues(reportedVerb, "", "", "", requestInfo.Path, scope, component, codeToString(code)).Inc()
}
}
@ -324,12 +375,13 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
}
var g compbasemetrics.GaugeMetric
scope := CleanScope(requestInfo)
// We don't use verb from <requestInfo>, as for the healthy path
// MonitorRequest is called from InstrumentRouteFunc which is registered
// in installer.go with predefined list of verbs (different than those
// translated to RequestInfo).
// We don't use verb from <requestInfo>, as this may be propagated from
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
if requestInfo.IsResourceRequest {
g = longRunningRequestGauge.WithLabelValues(reportedVerb, requestInfo.APIGroup, requestInfo.APIVersion, requestInfo.Resource, requestInfo.Subresource, scope, component)
} else {
@ -343,11 +395,21 @@ func RecordLongRunning(req *http.Request, requestInfo *request.RequestInfo, comp
// MonitorRequest handles standard transformations for client and the reported verb and then invokes Monitor to record
// a request. verb must be uppercase to be backwards compatible with existing monitoring tooling.
func MonitorRequest(req *http.Request, verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, contentType string, httpCode, respSize int, elapsed time.Duration) {
reportedVerb := cleanVerb(verb, req)
// We don't use verb from <requestInfo>, as this may be propagated from
// InstrumentRouteFunc which is registered in installer.go with predefined
// list of verbs (different than those translated to RequestInfo).
// However, we need to tweak it e.g. to differentiate GET from LIST.
reportedVerb := cleanVerb(canonicalVerb(strings.ToUpper(req.Method), scope), req)
dryRun := cleanDryRun(req.URL)
elapsedSeconds := elapsed.Seconds()
cleanContentType := cleanContentType(contentType)
requestCounter.WithLabelValues(reportedVerb, dryRun, group, version, resource, subresource, scope, component, cleanContentType, codeToString(httpCode)).Inc()
// MonitorRequest happens after authentication, so we can trust the username given by the request
info, ok := request.UserFrom(req.Context())
if ok && info.GetName() == user.APIServerUser {
apiSelfRequestCounter.WithLabelValues(reportedVerb, resource, subresource).Inc()
}
if deprecated {
deprecatedRequestGauge.WithLabelValues(group, version, resource, subresource, removedRelease).Set(1)
audit.AddAuditAnnotation(req.Context(), deprecatedAnnotationKey, "true")
@ -365,8 +427,11 @@ func MonitorRequest(req *http.Request, verb, group, version, resource, subresour
// InstrumentRouteFunc works like Prometheus' InstrumentHandlerFunc but wraps
// the go-restful RouteFunction instead of a HandlerFunc plus some Kubernetes endpoint specific information.
func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, routeFunc restful.RouteFunction) restful.RouteFunction {
return restful.RouteFunction(func(request *restful.Request, response *restful.Response) {
now := time.Now()
return restful.RouteFunction(func(req *restful.Request, response *restful.Response) {
requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Request.Context())
if !ok {
requestReceivedTimestamp = time.Now()
}
delegate := &ResponseWriterDelegator{ResponseWriter: response.ResponseWriter}
@ -381,16 +446,19 @@ func InstrumentRouteFunc(verb, group, version, resource, subresource, scope, com
}
response.ResponseWriter = rw
routeFunc(request, response)
routeFunc(req, response)
MonitorRequest(request.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
MonitorRequest(req.Request, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
})
}
// InstrumentHandlerFunc works like Prometheus' InstrumentHandlerFunc but adds some Kubernetes endpoint specific information.
func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, component string, deprecated bool, removedRelease string, handler http.HandlerFunc) http.HandlerFunc {
return func(w http.ResponseWriter, req *http.Request) {
now := time.Now()
requestReceivedTimestamp, ok := request.ReceivedTimestampFrom(req.Context())
if !ok {
requestReceivedTimestamp = time.Now()
}
delegate := &ResponseWriterDelegator{ResponseWriter: w}
@ -405,7 +473,7 @@ func InstrumentHandlerFunc(verb, group, version, resource, subresource, scope, c
handler(w, req)
MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(now))
MonitorRequest(req, verb, group, version, resource, subresource, scope, component, deprecated, removedRelease, delegate.Header().Get("Content-Type"), delegate.Status(), delegate.ContentLength(), time.Since(requestReceivedTimestamp))
}
}
@ -440,7 +508,7 @@ func CleanScope(requestInfo *request.RequestInfo) string {
func canonicalVerb(verb string, scope string) string {
switch verb {
case "GET", "HEAD":
if scope != "resource" {
if scope != "resource" && scope != "" {
return "LIST"
}
return "GET"

View file

@ -0,0 +1,45 @@
/*
Copyright 2020 The Kubernetes Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package request
import (
"context"
"time"
)
type requestReceivedTimestampKeyType int
// requestReceivedTimestampKey is the ReceivedTimestamp (the time the request reached the apiserver)
// key for the context.
const requestReceivedTimestampKey requestReceivedTimestampKeyType = iota
// WithReceivedTimestamp returns a copy of parent context in which the ReceivedTimestamp
// (the time the request reached the apiserver) is set.
//
// If the specified ReceivedTimestamp is zero, no value is set and the parent context is returned as is.
func WithReceivedTimestamp(parent context.Context, receivedTimestamp time.Time) context.Context {
if receivedTimestamp.IsZero() {
return parent
}
return WithValue(parent, requestReceivedTimestampKey, receivedTimestamp)
}
// ReceivedTimestampFrom returns the value of the ReceivedTimestamp key from the specified context.
func ReceivedTimestampFrom(ctx context.Context) (time.Time, bool) {
info, ok := ctx.Value(requestReceivedTimestampKey).(time.Time)
return info, ok
}

View file

@ -77,7 +77,7 @@ var specialVerbsNoSubresources = sets.NewString("proxy")
// this list allows the parser to distinguish between a namespace subresource, and a namespaced resource
var namespaceSubresources = sets.NewString("status", "finalize")
// NamespaceSubResourcesForTest exports namespaceSubresources for testing in pkg/master/master_test.go, so we never drift
// NamespaceSubResourcesForTest exports namespaceSubresources for testing in pkg/controlplane/master_test.go, so we never drift
var NamespaceSubResourcesForTest = sets.NewString(namespaceSubresources.List()...)
type RequestInfoFactory struct {