From 3b551fcdd6e36d01b9fc0f94ec0c8ecd31aa26c4 Mon Sep 17 00:00:00 2001 From: Prashant Balachandran Date: Mon, 13 Sep 2021 12:16:28 +0530 Subject: [PATCH] using new version of custom-metrics-apiserver adding changes to bring up metrics server correcting go dependencies Signed-off-by: Prashant Balachandran addressing code review comments, adding tls for metrics endpoint --- cmd/adapter/adapter.go | 46 +++++++++++++++++++ .../custom-metrics-apiserver-deployment.yaml | 15 ++++++ .../custom-metrics-apiserver-tls-config.yaml | 11 +++++ go.mod | 4 +- go.sum | 23 ++++++++++ 5 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 deploy/manifests/custom-metrics-apiserver-tls-config.yaml diff --git a/cmd/adapter/adapter.go b/cmd/adapter/adapter.go index 0169f40a..168217cb 100644 --- a/cmd/adapter/adapter.go +++ b/cmd/adapter/adapter.go @@ -22,12 +22,14 @@ import ( "flag" "fmt" "io/ioutil" + "net" "net/http" "net/url" "os" "strings" "time" + "github.com/prometheus/exporter-toolkit/web" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" openapinamer "k8s.io/apiserver/pkg/endpoints/openapi" @@ -38,6 +40,7 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/transport" "k8s.io/component-base/logs" + "k8s.io/component-base/metrics/legacyregistry" "k8s.io/klog/v2" customexternalmetrics "sigs.k8s.io/custom-metrics-apiserver/pkg/apiserver" @@ -55,6 +58,17 @@ import ( resprov "sigs.k8s.io/prometheus-adapter/pkg/resourceprovider" ) +type adapterLogger struct{} + +func (pl adapterLogger) Println(v ...interface{}) { + klog.Error(v...) +} + +func (pl adapterLogger) Log(v ...interface{}) error { + klog.Info(v...) + return nil +} + type PrometheusAdapter struct { basecmd.AdapterBase @@ -81,6 +95,9 @@ type PrometheusAdapter struct { // MetricsMaxAge is the period to query available metrics for MetricsMaxAge time.Duration + secureMetricsPort string + tlsConfig string + metricsHost string metricsConfig *adaptercfg.MetricsDiscoveryConfig } @@ -144,6 +161,12 @@ func (cmd *PrometheusAdapter) addFlags() { "interval at which to re-list the set of all available metrics from Prometheus") cmd.Flags().DurationVar(&cmd.MetricsMaxAge, "metrics-max-age", cmd.MetricsMaxAge, ""+ "period for which to query the set of available metrics from Prometheus") + cmd.Flags().StringVar(&cmd.secureMetricsPort, "secure-metrics-port", cmd.secureMetricsPort, + "Secure port for metrics") + cmd.Flags().StringVar(&cmd.metricsHost, "metrics-host", cmd.metricsHost, + "Host for exposing metrics") + cmd.Flags().StringVar(&cmd.tlsConfig, "tls-config", cmd.tlsConfig, + "TLS configuration for metrics server") } func (cmd *PrometheusAdapter) loadConfig() error { @@ -334,12 +357,35 @@ func main() { klog.Fatalf("unable to install resource metrics API: %v", err) } + config, err := cmd.Config() + if err != nil { + klog.Fatalf("unable to get configuration of custom metrics apiserver: %v", err) + } + + apiServerConfig(config) + cmd.setupMetricsServer() + // run the server if err := cmd.Run(stopCh); err != nil { klog.Fatalf("unable to run custom metrics adapter: %v", err) } } +func apiServerConfig(config *customexternalmetrics.Config) { + config.GenericConfig.EnableMetrics = false +} + +func (cmd PrometheusAdapter) setupMetricsServer() { + + mux := http.NewServeMux() + mux.Handle("/metrics", legacyregistry.Handler()) + + serverAddress := net.JoinHostPort(cmd.metricsHost, cmd.secureMetricsPort) + metricsServer := http.Server{Handler: mux, Addr: serverAddress} + + go web.ListenAndServe(&metricsServer, cmd.tlsConfig, adapterLogger{}) +} + // makeKubeconfigHTTPClient constructs an HTTP for connecting with the given auth options. func makeKubeconfigHTTPClient(inClusterAuth bool, kubeConfigPath string) (*http.Client, error) { // make sure we're not trying to use two different sources of auth diff --git a/deploy/manifests/custom-metrics-apiserver-deployment.yaml b/deploy/manifests/custom-metrics-apiserver-deployment.yaml index 0fc61d79..9d0175dd 100644 --- a/deploy/manifests/custom-metrics-apiserver-deployment.yaml +++ b/deploy/manifests/custom-metrics-apiserver-deployment.yaml @@ -29,6 +29,9 @@ spec: - --metrics-relist-interval=1m - --v=10 - --config=/etc/adapter/config.yaml + - --secure-metrics-port=9090 + - --metrics-host=0.0.0.0 + - --tls-config=/etc/tls/config.yaml ports: - containerPort: 6443 volumeMounts: @@ -40,6 +43,12 @@ spec: readOnly: true - mountPath: /tmp name: tmp-vol + - mountPath: /etc/tls + name: metricstlsconfig + readOnly: false + - mountPath: /etc/certs + name: servingcerts + readOnly: false volumes: - name: volume-serving-cert secret: @@ -49,3 +58,9 @@ spec: name: adapter-config - name: tmp-vol emptyDir: {} + - configMap: + name: metrics-tls-config + name: metricstlsconfig + - secret: + secretName: pa-metrics-serving-certs + name: servingcerts diff --git a/deploy/manifests/custom-metrics-apiserver-tls-config.yaml b/deploy/manifests/custom-metrics-apiserver-tls-config.yaml new file mode 100644 index 00000000..5fcea305 --- /dev/null +++ b/deploy/manifests/custom-metrics-apiserver-tls-config.yaml @@ -0,0 +1,11 @@ +apiVersion: v1 +data: + config.yaml: |- + "tls_server_config" : + "cert_file" : "/etc/certs/serving.crt" + "key_file" : "/etc/certs/serving.key" + "client_auth_type" : "VerifyClientCertIfGiven" +kind: ConfigMap +metadata: + name: metrics-tls-config + namespace: monitoring \ No newline at end of file diff --git a/go.mod b/go.mod index 19cc2c5d..eae97a38 100644 --- a/go.mod +++ b/go.mod @@ -6,9 +6,11 @@ require ( github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.15.0 github.com/prometheus/client_golang v1.11.0 - github.com/prometheus/common v0.26.0 + github.com/prometheus/common v0.29.0 + github.com/prometheus/exporter-toolkit v0.6.1 github.com/spf13/cobra v1.2.1 github.com/stretchr/testify v1.7.0 + golang.org/x/net v0.0.0-20210525063256-abc453219eb5 // indirect gopkg.in/yaml.v2 v2.4.0 k8s.io/api v0.22.0 k8s.io/apimachinery v0.22.0 diff --git a/go.sum b/go.sum index 178a12e5..cdc820ad 100644 --- a/go.sum +++ b/go.sum @@ -189,9 +189,11 @@ github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2 github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.9.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-kit/kit v0.10.0/go.mod h1:xUsJbQ/Fp4kEt7AFgCuvyX4a71u8h9jB8tj/ORgOZ7o= +github.com/go-kit/log v0.1.0 h1:DGJh0Sm43HbOeYDNnVZFl8BvcYVvjD5bqYJvp0REbwQ= github.com/go-kit/log v0.1.0/go.mod h1:zbhenjAZHb184qTLMA9ZjW7ThYL0H2mk7Q6pNt4vbaY= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= +github.com/go-logfmt/logfmt v0.5.0 h1:TrB8swr/68K7m9CcGut2g3UOihhbcbiMAYiuTXdEih4= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v0.2.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= @@ -367,6 +369,7 @@ github.com/jonboulle/clockwork v0.2.2 h1:UOGuzwb1PwsrDAObMuhUnj0p5ULPj8V/xJ7Kx9q github.com/jonboulle/clockwork v0.2.2/go.mod h1:Pkfl5aHPm1nk2H9h0bjmnJD/BcgbGXUBGnn1kMkgxc8= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= +github.com/jpillora/backoff v1.0.0 h1:uvFg412JmmHBHw7iwprIxkPMI+sGQ4kzOWsMeHnm2EA= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -437,6 +440,7 @@ github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8m github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= +github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f h1:KUppIJq7/+SVif2QVs3tOP0zanoHgBEVAwHxUSIzRqU= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mxk/go-flowrate v0.0.0-20140419014527-cca7078d478f/go.mod h1:ZdcZmHo+o7JKHSa8/e818NopupXU1YMK5fe1lsApnBw= github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= @@ -500,6 +504,14 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= github.com/pquerna/cachecontrol v0.0.0-20171018203845-0dec1b30a021/go.mod h1:prYjPmNq4d1NPVmpShWobRqXY3q7Vp+80DqgxxUrUIA= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924121150-c59102952133 h1:HiQnjSFpsulDVPEC9/JJzCO7cQZ+5+2cpxodmRzPjy4= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924121150-c59102952133/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924125505-db775ad8a2b3 h1:oQ9MH4BkVbYG7Rgm7hmC5xJ69N76VN1WYfoWnOt7yKk= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924125505-db775ad8a2b3/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924130734-ef442683046d h1:dF3MhzPXQ3+3dlyNNB2ZFC0QNFBiEe6jcMFsxAt6+3w= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924130734-ef442683046d/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924132849-9a845d6843df h1:w9/S7uweexBi4eyQyC0I4UYeKigzckAumq6+/YW+wp8= +github.com/prashbnair/exporter-toolkit v0.6.2-0.20210924132849-9a845d6843df/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= @@ -524,6 +536,10 @@ github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB8 github.com/prometheus/common v0.25.0/go.mod h1:H6QK/N6XVT42whUeIdI3dp36w49c+/iMDk7UAI2qm7Q= github.com/prometheus/common v0.26.0 h1:iMAkS2TDoNWnKM+Kopnx/8tnEStIfpYA0ur0xQzzhMQ= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= +github.com/prometheus/common v0.29.0 h1:3jqPBvKT4OHAbje2Ql7KeaaSicDBCxMYwEJU1zRJceE= +github.com/prometheus/common v0.29.0/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= +github.com/prometheus/exporter-toolkit v0.6.1 h1:Aqk75wQD92N9CqmTlZwjKwq6272nOGrWIbc8Z7+xQO0= +github.com/prometheus/exporter-toolkit v0.6.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= @@ -689,6 +705,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83 h1:/ZScEX8SfEmUGRHs0gxpqteO5nfNW6axyZbBdw9A12g= golang.org/x/crypto v0.0.0-20210220033148-5ea612d1eb83/go.mod h1:jdWPYTVW3xRLrWPugEBEK3UY2ZEsg3UU495nc5E+M+I= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e h1:gsTQYXdTw2Gq7RBsWvlQ91b+aEQ6bXFUngBGuR8sPpI= +golang.org/x/crypto v0.0.0-20210616213533-5ff15b29337e/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -780,6 +798,8 @@ golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96b golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023 h1:ADo5wSpq2gqaCGQWzk7S5vd//0iyyLeAratkEoG5dLE= golang.org/x/net v0.0.0-20210520170846-37e1c6afe023/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5 h1:wjuX4b5yYQnEQHzd+CBcrcC6OVR2J1CN6mUy0oSxIPo= +golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20170207211851-4464e7848382/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -794,6 +814,8 @@ golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602 h1:0Ja1LBD+yisY6RWM/BH7TJVXWsSjs2VwBSmvSX4HdBc= golang.org/x/oauth2 v0.0.0-20210402161424-2e8d93401602/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c h1:pkQiBZBvdos9qq4wBAHqlzuZHEXo07pqV06ef90u1WI= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/perf v0.0.0-20210220033136-40a54f11e909/go.mod h1:KRSrLY7jerMEa0Ih7gBheQ3FYDiSx6liMnniX1o3j2g= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -871,6 +893,7 @@ golang.org/x/sys v0.0.0-20210403161142-5e06dd20ab57/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22 h1:RqytpXGR1iVNX7psjB3ff8y7sNFinVFvkx1c8SjBkio= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=