vendor dependencies

vendor/k8s.io/apiserver/pkg/storage/storagebackend/OWNERS

@@ -1,3 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
 reviewers:
 - lavalamp
 - smarterclayton

vendor/k8s.io/apiserver/pkg/storage/storagebackend/config.go

@@ -25,24 +25,29 @@ import (
 
 const (
 	StorageTypeUnset = ""
-	StorageTypeETCD2 = "etcd2"
 	StorageTypeETCD3 = "etcd3"
 
 	DefaultCompactInterval = 5 * time.Minute
 )
 
-// Config is configuration for creating a storage backend.
-type Config struct {
-	// Type defines the type of storage backend, e.g. "etcd2", etcd3". Default ("") is "etcd3".
-	Type string
-	// Prefix is the prefix to all keys passed to storage.Interface methods.
-	Prefix string
+// TransportConfig holds all connection related info,  i.e. equal TransportConfig means equal servers we talk to.
+type TransportConfig struct {
 	// ServerList is the list of storage servers to connect with.
 	ServerList []string
 	// TLS credentials
 	KeyFile  string
 	CertFile string
 	CAFile   string
+}
+
+// Config is configuration for creating a storage backend.
+type Config struct {
+	// Type defines the type of storage backend. Default ("") is "etcd3".
+	Type string
+	// Prefix is the prefix to all keys passed to storage.Interface methods.
+	Prefix string
+	// Transport holds all connection related info, i.e. equal TransportConfig means equal servers we talk to.
+	Transport TransportConfig
 	// Quorum indicates that whether read operations should be quorum-level consistent.
 	Quorum bool
 	// Paging indicates whether the server implementation should allow paging (if it is
@@ -50,31 +55,27 @@ type Config struct {
 	// resource type not wishing to allow paging, and is not intended for end users to
 	// set.
 	Paging bool
-	// DeserializationCacheSize is the size of cache of deserialized objects.
-	// Currently this is only supported in etcd2.
-	// We will drop the cache once using protobuf.
-	DeserializationCacheSize int
 
 	Codec runtime.Codec
+	// EncodeVersioner is the same groupVersioner used to build the
+	// storage encoder. Given a list of kinds the input object might belong
+	// to, the EncodeVersioner outputs the gvk the object will be
+	// converted to before persisted in etcd.
+	EncodeVersioner runtime.GroupVersioner
 	// Transformer allows the value to be transformed prior to persisting into etcd.
 	Transformer value.Transformer
 
 	// CompactionInterval is an interval of requesting compaction from apiserver.
 	// If the value is 0, no compaction will be issued.
 	CompactionInterval time.Duration
-
 	// CountMetricPollPeriod specifies how often should count metric be updated
 	CountMetricPollPeriod time.Duration
 }
 
 func NewDefaultConfig(prefix string, codec runtime.Codec) *Config {
 	return &Config{
-		Prefix: prefix,
-		// Default cache size to 0 - if unset, its size will be set based on target
-		// memory usage.
-		DeserializationCacheSize: 0,
+		Prefix:             prefix,
 		Codec:              codec,
 		CompactionInterval: DefaultCompactInterval,
-		Quorum:             true,
 	}
 }

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd2.go

@@ -1,81 +0,0 @@
-/*
-Copyright 2016 The Kubernetes Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package factory
-
-import (
-	"net"
-	"net/http"
-	"time"
-
-	etcd2client "github.com/coreos/etcd/client"
-	"github.com/coreos/etcd/pkg/transport"
-
-	utilnet "k8s.io/apimachinery/pkg/util/net"
-	"k8s.io/apiserver/pkg/storage"
-	"k8s.io/apiserver/pkg/storage/etcd"
-	"k8s.io/apiserver/pkg/storage/storagebackend"
-)
-
-func newETCD2Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
-	tr, err := newTransportForETCD2(c.CertFile, c.KeyFile, c.CAFile)
-	if err != nil {
-		return nil, nil, err
-	}
-	client, err := newETCD2Client(tr, c.ServerList)
-	if err != nil {
-		return nil, nil, err
-	}
-	s := etcd.NewEtcdStorage(client, c.Codec, c.Prefix, c.Quorum, c.DeserializationCacheSize, etcd.IdentityTransformer)
-	return s, tr.CloseIdleConnections, nil
-}
-
-func newETCD2Client(tr *http.Transport, serverList []string) (etcd2client.Client, error) {
-	cli, err := etcd2client.New(etcd2client.Config{
-		Endpoints: serverList,
-		Transport: tr,
-	})
-	if err != nil {
-		return nil, err
-	}
-
-	return cli, nil
-}
-
-func newTransportForETCD2(certFile, keyFile, caFile string) (*http.Transport, error) {
-	info := transport.TLSInfo{
-		CertFile: certFile,
-		KeyFile:  keyFile,
-		CAFile:   caFile,
-	}
-	cfg, err := info.ClientConfig()
-	if err != nil {
-		return nil, err
-	}
-	// Copied from etcd.DefaultTransport declaration.
-	// TODO: Determine if transport needs optimization
-	tr := utilnet.SetTransportDefaults(&http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		DialContext: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-		}).DialContext,
-		TLSHandshakeTimeout: 10 * time.Second,
-		MaxIdleConnsPerHost: 500,
-		TLSClientConfig:     cfg,
-	})
-	return tr, nil
-}

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/etcd3.go

@@ -18,11 +18,18 @@ package factory
 
 import (
 	"context"
+	"fmt"
+	"path"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/coreos/etcd/clientv3"
 	"github.com/coreos/etcd/pkg/transport"
+	grpcprom "github.com/grpc-ecosystem/go-grpc-prometheus"
+	"google.golang.org/grpc"
 
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/apiserver/pkg/storage"
 	"k8s.io/apiserver/pkg/storage/etcd3"
 	"k8s.io/apiserver/pkg/storage/storagebackend"
@@ -39,7 +46,43 @@ const keepaliveTimeout = 10 * time.Second
 // on heavily loaded arm64 CPUs (issue #64649)
 const dialTimeout = 20 * time.Second
 
-func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+func newETCD3HealthCheck(c storagebackend.Config) (func() error, error) {
+	// constructing the etcd v3 client blocks and times out if etcd is not available.
+	// retry in a loop in the background until we successfully create the client, storing the client or error encountered
+
+	clientValue := &atomic.Value{}
+
+	clientErrMsg := &atomic.Value{}
+	clientErrMsg.Store("etcd client connection not yet established")
+
+	go wait.PollUntil(time.Second, func() (bool, error) {
+		client, err := newETCD3Client(c.Transport)
+		if err != nil {
+			clientErrMsg.Store(err.Error())
+			return false, nil
+		}
+		clientValue.Store(client)
+		clientErrMsg.Store("")
+		return true, nil
+	}, wait.NeverStop)
+
+	return func() error {
+		if errMsg := clientErrMsg.Load().(string); len(errMsg) > 0 {
+			return fmt.Errorf(errMsg)
+		}
+		client := clientValue.Load().(*clientv3.Client)
+		ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
+		defer cancel()
+		// See https://github.com/etcd-io/etcd/blob/master/etcdctl/ctlv3/command/ep_command.go#L118
+		_, err := client.Get(ctx, path.Join(c.Prefix, "health"))
+		if err == nil {
+			return nil
+		}
+		return fmt.Errorf("error getting data from etcd: %v", err)
+	}, nil
+}
+
+func newETCD3Client(c storagebackend.TransportConfig) (*clientv3.Client, error) {
 	tlsInfo := transport.TLSInfo{
 		CertFile: c.CertFile,
 		KeyFile:  c.KeyFile,
@@ -47,7 +90,7 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 	}
 	tlsConfig, err := tlsInfo.ClientConfig()
 	if err != nil {
-		return nil, nil, err
+		return nil, err
 	}
 	// NOTE: Client relies on nil tlsConfig
 	// for non-secure connections, update the implicit variable
@@ -58,25 +101,103 @@ func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, e
 		DialTimeout:          dialTimeout,
 		DialKeepAliveTime:    keepaliveTime,
 		DialKeepAliveTimeout: keepaliveTimeout,
-		Endpoints:            c.ServerList,
-		TLS:                  tlsConfig,
+		DialOptions: []grpc.DialOption{
+			grpc.WithUnaryInterceptor(grpcprom.UnaryClientInterceptor),
+			grpc.WithStreamInterceptor(grpcprom.StreamClientInterceptor),
+		},
+		Endpoints: c.ServerList,
+		TLS:       tlsConfig,
 	}
-	client, err := clientv3.New(cfg)
+
+	return clientv3.New(cfg)
+}
+
+type runningCompactor struct {
+	interval time.Duration
+	cancel   context.CancelFunc
+	client   *clientv3.Client
+	refs     int
+}
+
+var (
+	lock       sync.Mutex
+	compactors = map[string]*runningCompactor{}
+)
+
+// startCompactorOnce start one compactor per transport. If the interval get smaller on repeated calls, the
+// compactor is replaced. A destroy func is returned. If all destroy funcs with the same transport are called,
+// the compactor is stopped.
+func startCompactorOnce(c storagebackend.TransportConfig, interval time.Duration) (func(), error) {
+	lock.Lock()
+	defer lock.Unlock()
+
+	key := fmt.Sprintf("%v", c) // gives: {[server1 server2] keyFile certFile caFile}
+	if compactor, foundBefore := compactors[key]; !foundBefore || compactor.interval > interval {
+		compactorClient, err := newETCD3Client(c)
+		if err != nil {
+			return nil, err
+		}
+
+		if foundBefore {
+			// replace compactor
+			compactor.cancel()
+			compactor.client.Close()
+		} else {
+			// start new compactor
+			compactor = &runningCompactor{}
+			compactors[key] = compactor
+		}
+
+		ctx, cancel := context.WithCancel(context.Background())
+
+		compactor.interval = interval
+		compactor.cancel = cancel
+		compactor.client = compactorClient
+
+		etcd3.StartCompactor(ctx, compactorClient, interval)
+	}
+
+	compactors[key].refs++
+
+	return func() {
+		lock.Lock()
+		defer lock.Unlock()
+
+		compactor := compactors[key]
+		compactor.refs--
+		if compactor.refs == 0 {
+			compactor.cancel()
+			compactor.client.Close()
+			delete(compactors, key)
+		}
+	}, nil
+}
+
+func newETCD3Storage(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
+	stopCompactor, err := startCompactorOnce(c.Transport, c.CompactionInterval)
 	if err != nil {
 		return nil, nil, err
 	}
-	ctx, cancel := context.WithCancel(context.Background())
-	etcd3.StartCompactor(ctx, client, c.CompactionInterval)
+
+	client, err := newETCD3Client(c.Transport)
+	if err != nil {
+		stopCompactor()
+		return nil, nil, err
+	}
+
+	var once sync.Once
 	destroyFunc := func() {
-		cancel()
-		client.Close()
+		// we know that storage destroy funcs are called multiple times (due to reuse in subresources).
+		// Hence, we only destroy once.
+		// TODO: fix duplicated storage destroy calls higher level
+		once.Do(func() {
+			stopCompactor()
+			client.Close()
+		})
 	}
 	transformer := c.Transformer
 	if transformer == nil {
 		transformer = value.IdentityTransformer
 	}
-	if c.Quorum {
-		return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
-	}
-	return etcd3.NewWithNoQuorumRead(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
+	return etcd3.New(client, c.Codec, c.Prefix, transformer, c.Paging), destroyFunc, nil
 }

vendor/k8s.io/apiserver/pkg/storage/storagebackend/factory/factory.go

@@ -29,15 +29,23 @@ type DestroyFunc func()
 // Create creates a storage backend based on given config.
 func Create(c storagebackend.Config) (storage.Interface, DestroyFunc, error) {
 	switch c.Type {
-	case storagebackend.StorageTypeETCD2:
-		return newETCD2Storage(c)
+	case "etcd2":
+		return nil, nil, fmt.Errorf("%v is no longer a supported storage backend", c.Type)
 	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
-		// TODO: We have the following features to implement:
-		// - Support secure connection by using key, cert, and CA files.
-		// - Honor "https" scheme to support secure connection in gRPC.
-		// - Support non-quorum read.
 		return newETCD3Storage(c)
 	default:
 		return nil, nil, fmt.Errorf("unknown storage type: %s", c.Type)
 	}
 }
+
+// CreateHealthCheck creates a healthcheck function based on given config.
+func CreateHealthCheck(c storagebackend.Config) (func() error, error) {
+	switch c.Type {
+	case "etcd2":
+		return nil, fmt.Errorf("%v is no longer a supported storage backend", c.Type)
+	case storagebackend.StorageTypeUnset, storagebackend.StorageTypeETCD3:
+		return newETCD3HealthCheck(c)
+	default:
+		return nil, fmt.Errorf("unknown storage type: %s", c.Type)
+	}
+}