From b92d30bd114749127b6276ac6c3ad539fa424432 Mon Sep 17 00:00:00 2001
From: Roberto Rojas <robertojrojas@gmail.com>
Date: Tue, 12 Nov 2019 16:47:42 -0500
Subject: [PATCH] Fixes issue #517: Determines Kubernetes version using the
REST API (#518)
* Fixes issue #517: Determines Kubernetes version using the REST API
* fixes
* fixes
* adds tests
* fixes
* added more tests
* kubernetes_version_test: Add a missing case for invalid certs
Signed-off-by: Simarpreet Singh <simar@linux.com>
* kubernetes_version_test: Remove un-needed casts
Signed-off-by: Simarpreet Singh <simar@linux.com>
* fixes as per PR review
* fixes as per PR review
---
cmd/common.go | 10 +-
cmd/common_test.go | 12 +-
cmd/kubernetes_version.go | 142 ++++++++++++++++++++
cmd/kubernetes_version_test.go | 233 +++++++++++++++++++++++++++++++++
cmd/util.go | 10 ++
5 files changed, 401 insertions(+), 6 deletions(-)
create mode 100644 cmd/kubernetes_version.go
create mode 100644 cmd/kubernetes_version_test.go
diff --git a/cmd/common.go b/cmd/common.go
index 8e6566f..0f9debd 100644
--- a/cmd/common.go
+++ b/cmd/common.go
@@ -235,17 +235,19 @@ func loadConfig(nodetype check.NodeType) string {
}
func mapToBenchmarkVersion(kubeToBenchmarkMap map[string]string, kv string) (string, error) {
+ kvOriginal := kv
cisVersion, found := kubeToBenchmarkMap[kv]
+ glog.V(2).Info(fmt.Sprintf("mapToBenchmarkVersion for k8sVersion: %q cisVersion: %q found: %t\n", kv, cisVersion, found))
for !found && (kv != defaultKubeVersion && !isEmpty(kv)) {
kv = decrementVersion(kv)
cisVersion, found = kubeToBenchmarkMap[kv]
- glog.V(2).Info(fmt.Sprintf("mapToBenchmarkVersion for cisVersion: %q found: %t\n", cisVersion, found))
+ glog.V(2).Info(fmt.Sprintf("mapToBenchmarkVersion for k8sVersion: %q cisVersion: %q found: %t\n", kv, cisVersion, found))
}
if !found {
- glog.V(1).Info(fmt.Sprintf("mapToBenchmarkVersion unable to find a match for: %q", kv))
+ glog.V(1).Info(fmt.Sprintf("mapToBenchmarkVersion unable to find a match for: %q", kvOriginal))
glog.V(3).Info(fmt.Sprintf("mapToBenchmarkVersion kubeToBenchmarkSMap: %#v", kubeToBenchmarkMap))
- return "", fmt.Errorf("Unable to find a matching Benchmark Version match for kubernetes version: %s", kubeVersion)
+ return "", fmt.Errorf("unable to find a matching Benchmark Version match for kubernetes version: %s", kvOriginal)
}
return cisVersion, nil
@@ -285,6 +287,8 @@ func getBenchmarkVersion(kubeVersion, benchmarkVersion string, v *viper.Viper) (
glog.V(2).Info(fmt.Sprintf("Mapped Kubernetes version: %s to Benchmark version: %s", kubeVersion, benchmarkVersion))
}
+
+ glog.V(1).Info(fmt.Sprintf("Kubernetes version: %q to Benchmark version: %q", kubeVersion, benchmarkVersion))
return benchmarkVersion, nil
}
diff --git a/cmd/common_test.go b/cmd/common_test.go
index 68bc36b..85e60a4 100644
--- a/cmd/common_test.go
+++ b/cmd/common_test.go
@@ -186,15 +186,16 @@ func TestMapToCISVersion(t *testing.T) {
kubeVersion string
succeed bool
exp string
+ expErr string
}{
- {kubeVersion: "1.9", succeed: false, exp: ""},
+ {kubeVersion: "1.9", succeed: false, exp: "", expErr: "unable to find a matching Benchmark Version match for kubernetes version: 1.9"},
{kubeVersion: "1.11", succeed: true, exp: "cis-1.3"},
{kubeVersion: "1.12", succeed: true, exp: "cis-1.3"},
{kubeVersion: "1.13", succeed: true, exp: "cis-1.4"},
{kubeVersion: "1.16", succeed: true, exp: "cis-1.4"},
{kubeVersion: "ocp-3.10", succeed: true, exp: "rh-0.7"},
{kubeVersion: "ocp-3.11", succeed: true, exp: "rh-0.7"},
- {kubeVersion: "unknown", succeed: false, exp: ""},
+ {kubeVersion: "unknown", succeed: false, exp: "", expErr: "unable to find a matching Benchmark Version match for kubernetes version: unknown"},
}
for _, c := range cases {
rv, err := mapToBenchmarkVersion(kubeToBenchmarkMap, c.kubeVersion)
@@ -210,9 +211,14 @@ func TestMapToCISVersion(t *testing.T) {
if c.exp != rv {
t.Errorf("[%q]- expected %q but Got %q", c.kubeVersion, c.exp, rv)
}
+
} else {
if c.exp != rv {
- t.Errorf("mapToBenchmarkVersion kubeversion: %q Got %q expected %s", c.kubeVersion, rv, c.exp)
+ t.Errorf("[%q]-mapToBenchmarkVersion kubeversion: %q Got %q expected %s", c.kubeVersion, c.kubeVersion, rv, c.exp)
+ }
+
+ if c.expErr != err.Error() {
+ t.Errorf("[%q]-mapToBenchmarkVersion expected Error: %q instead Got %q", c.kubeVersion, c.expErr, err.Error())
}
}
}
diff --git a/cmd/kubernetes_version.go b/cmd/kubernetes_version.go
new file mode 100644
index 0000000..be0ff8b
--- /dev/null
+++ b/cmd/kubernetes_version.go
@@ -0,0 +1,142 @@
+package cmd
+
+import (
+ "crypto/tls"
+ "encoding/json"
+ "encoding/pem"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "os"
+ "strings"
+
+ "github.com/golang/glog"
+)
+
+func getKubeVersionFromRESTAPI() (string, error) {
+ k8sVersionURL := getKubernetesURL()
+ serviceaccount := "/var/run/secrets/kubernetes.io/serviceaccount"
+ cacertfile := fmt.Sprintf("%s/ca.crt", serviceaccount)
+ tokenfile := fmt.Sprintf("%s/token", serviceaccount)
+
+ tlsCert, err := loadCertficate(cacertfile)
+ if err != nil {
+ return "", err
+ }
+
+ tb, err := ioutil.ReadFile(tokenfile)
+ if err != nil {
+ return "", err
+ }
+ token := strings.TrimSpace(string(tb))
+
+ data, err := getWebData(k8sVersionURL, token, tlsCert)
+ if err != nil {
+ return "", err
+ }
+
+ k8sVersion, err := extractVersion(data)
+ if err != nil {
+ return "", err
+ }
+ return k8sVersion, nil
+}
+
+func extractVersion(data []byte) (string, error) {
+ type versionResponse struct {
+ Major string
+ Minor string
+ GitVersion string
+ GitCommit string
+ GitTreeState string
+ BuildDate string
+ GoVersion string
+ Compiler string
+ Platform string
+ }
+
+ vrObj := &versionResponse{}
+ glog.V(2).Info(fmt.Sprintf("vd: %s\n", string(data)))
+ err := json.Unmarshal(data, vrObj)
+ if err != nil {
+ return "", err
+ }
+ glog.V(2).Info(fmt.Sprintf("vrObj: %#v\n", vrObj))
+
+ // Some provides return the minor version like "15+"
+ minor := strings.Replace(vrObj.Minor, "+", "", -1)
+ ver := fmt.Sprintf("%s.%s", vrObj.Major, minor)
+ return ver, nil
+}
+
+func getWebData(srvURL, token string, cacert *tls.Certificate) ([]byte, error) {
+ glog.V(2).Info(fmt.Sprintf("getWebData srvURL: %s\n", srvURL))
+
+ tlsConf := &tls.Config{
+ Certificates: []tls.Certificate{*cacert},
+ InsecureSkipVerify: true,
+ }
+ tr := &http.Transport{
+ TLSClientConfig: tlsConf,
+ }
+ client := &http.Client{Transport: tr}
+ req, err := http.NewRequest(http.MethodGet, srvURL, nil)
+ if err != nil {
+ return nil, err
+ }
+
+ authToken := fmt.Sprintf("Bearer %s", token)
+ glog.V(2).Info(fmt.Sprintf("getWebData AUTH TOKEN --[%q]--\n", authToken))
+ req.Header.Set("Authorization", authToken)
+
+ resp, err := client.Do(req)
+ if err != nil {
+ glog.V(2).Info(fmt.Sprintf("HTTP ERROR: %v\n", err))
+ return nil, err
+ }
+ defer resp.Body.Close()
+
+ if resp.StatusCode != http.StatusOK {
+ glog.V(2).Info(fmt.Sprintf("URL:[%s], StatusCode:[%d] \n Headers: %#v\n", srvURL, resp.StatusCode, resp.Header))
+ err = fmt.Errorf("URL:[%s], StatusCode:[%d]", srvURL, resp.StatusCode)
+ return nil, err
+ }
+
+ return ioutil.ReadAll(resp.Body)
+}
+
+func loadCertficate(certFile string) (*tls.Certificate, error) {
+ cacert, err := ioutil.ReadFile(certFile)
+ if err != nil {
+ return nil, err
+ }
+
+ var tlsCert tls.Certificate
+ block, _ := pem.Decode(cacert)
+ if block == nil {
+ return nil, fmt.Errorf("unable to Decode certificate")
+ }
+
+ glog.V(2).Info(fmt.Sprintf("Loading CA certificate"))
+ tlsCert.Certificate = append(tlsCert.Certificate, block.Bytes)
+ return &tlsCert, nil
+}
+
+func getKubernetesURL() string {
+ k8sVersionURL := "https://kubernetes.default.svc/version"
+
+ // The following provides flexibility to use
+ // K8S provided variables is situations where
+ // hostNetwork: true
+ if !isEmpty(os.Getenv("KUBE_BENCH_K8S_ENV")) {
+ k8sHost := os.Getenv("KUBERNETES_SERVICE_HOST")
+ k8sPort := os.Getenv("KUBERNETES_SERVICE_PORT_HTTPS")
+ if !isEmpty(k8sHost) && !isEmpty(k8sPort) {
+ return fmt.Sprintf("https://%s:%s/version", k8sHost, k8sPort)
+ }
+
+ glog.V(2).Info(fmt.Sprintf("KUBE_BENCH_K8S_ENV is set, but environment variables KUBERNETES_SERVICE_HOST or KUBERNETES_SERVICE_PORT_HTTPS are not set"))
+ }
+
+ return k8sVersionURL
+}
diff --git a/cmd/kubernetes_version_test.go b/cmd/kubernetes_version_test.go
new file mode 100644
index 0000000..1513f62
--- /dev/null
+++ b/cmd/kubernetes_version_test.go
@@ -0,0 +1,233 @@
+package cmd
+
+import (
+ "crypto/tls"
+ "fmt"
+ "io/ioutil"
+ "net/http"
+ "net/http/httptest"
+ "os"
+ "strconv"
+ "testing"
+)
+
+func TestLoadCertficate(t *testing.T) {
+ tmp, err := ioutil.TempDir("", "TestFakeLoadCertficate")
+ if err != nil {
+ t.Fatalf("unable to create temp directory: %v", err)
+ }
+ defer os.RemoveAll(tmp)
+
+ goodCertFile, _ := ioutil.TempFile(tmp, "good-cert-*")
+ _, _ = goodCertFile.Write([]byte(`-----BEGIN CERTIFICATE-----
+MIICyDCCAbCgAwIBAgIBADANBgkqhkiG9w0BAQsFADAVMRMwEQYDVQQDEwprdWJl
+cm5ldGVzMB4XDTE5MTEwODAxNDAwMFoXDTI5MTEwNTAxNDAwMFowFTETMBEGA1UE
+AxMKa3ViZXJuZXRlczCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMn6
+wjvhMc9e0MDwpQNhp8SPxmv1DsYJ4Btp1GeScIgKKDwppuoOmVizLiMNdV5+70yI
+MgNfm/gwFRNDOtN3R7msfZDD5Dd1vI6qRTP21DFOGVdysFdwqJTs0nGcmfvZEOtw
+9cjcsXrBi2Mg54v+X/pq2w51xajCGBt2+bpxJJ3WBiWqKYv0RQdNL0WZGm+V9BuP
+pHRWPBeLxuCzt5K3Gx+1QDy8o6Y4sSRPssWC4RhD9Hs5/9eeGRyZslLs+AuqdDLQ
+aziiSjHVtgCfRXE9nYVxaDIwTFuh+Q1IvtB36NRLyX47oya+BbX3PoCtSjA36RBb
+tcJfulr3oNHnb2ZlfcUCAwEAAaMjMCEwDgYDVR0PAQH/BAQDAgKkMA8GA1UdEwEB
+/wQFMAMBAf8wDQYJKoZIhvcNAQELBQADggEBAAeQDkbM6DilLkIVQDyxauETgJDV
+2AaVzYaAgDApQGAoYV6WIY7Exk4TlmLeKQjWt2s/GtthQWuzUDKTcEvWcG6gNdXk
+gzuCRRDMGu25NtG3m67w4e2RzW8Z/lzvbfyJZGoV2c6dN+yP9/Pw2MXlrnMWugd1
+jLv3UYZRHMpuNS8BJU74BuVzVPHd55RAl+bV8yemdZJ7pPzMvGbZ7zRXWODTDlge
+CQb9lY+jYErisH8Sq7uABFPvi7RaTh8SS7V7OxqHZvmttNTdZs4TIkk45JK7Y+Xq
+FAjB57z2NcIgJuVpQnGRYtr/JcH2Qdsq8bLtXaojUIWOOqoTDRLYozdMOOQ=
+-----END CERTIFICATE-----`))
+ badCertFile, _ := ioutil.TempFile(tmp, "bad-cert-*")
+
+ cases := []struct {
+ file string
+ fail bool
+ }{
+ {
+ file: "missing cert file",
+ fail: true,
+ },
+ {
+ file: badCertFile.Name(),
+ fail: true,
+ },
+ {
+ file: goodCertFile.Name(),
+ fail: false,
+ },
+ }
+
+ for id, c := range cases {
+ t.Run(strconv.Itoa(id), func(t *testing.T) {
+ tlsCert, err := loadCertficate(c.file)
+ if !c.fail {
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+
+ if tlsCert == nil {
+ t.Errorf("missing returned TLS Certificate")
+ }
+ } else {
+ if err == nil {
+ t.Errorf("Expected error")
+ }
+ }
+
+ })
+ }
+}
+
+func TestGetWebData(t *testing.T) {
+ okfn := func(w http.ResponseWriter, r *http.Request) {
+ _, _ = fmt.Fprintln(w, `{
+ "major": "1",
+ "minor": "15"}`)
+ }
+ errfn := func(w http.ResponseWriter, r *http.Request) {
+ http.Error(w, http.StatusText(http.StatusInternalServerError),
+ http.StatusInternalServerError)
+ }
+ token := "dummyToken"
+ var tlsCert tls.Certificate
+
+ cases := []struct {
+ fn http.HandlerFunc
+ fail bool
+ }{
+ {
+ fn: okfn,
+ fail: false,
+ },
+ {
+ fn: errfn,
+ fail: true,
+ },
+ }
+
+ for id, c := range cases {
+ t.Run(strconv.Itoa(id), func(t *testing.T) {
+ ts := httptest.NewServer(c.fn)
+ defer ts.Close()
+ data, err := getWebData(ts.URL, token, &tlsCert)
+ if !c.fail {
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+
+ if len(data) == 0 {
+ t.Errorf("missing data")
+ }
+ } else {
+ if err == nil {
+ t.Errorf("Expected error")
+ }
+ }
+ })
+ }
+
+}
+
+func TestExtractVersion(t *testing.T) {
+ okJSON := []byte(`{
+ "major": "1",
+ "minor": "15",
+ "gitVersion": "v1.15.3",
+ "gitCommit": "2d3c76f9091b6bec110a5e63777c332469e0cba2",
+ "gitTreeState": "clean",
+ "buildDate": "2019-08-20T18:57:36Z",
+ "goVersion": "go1.12.9",
+ "compiler": "gc",
+ "platform": "linux/amd64"
+ }`)
+
+ invalidJSON := []byte(`{
+ "major": "1",
+ "minor": "15",
+ "gitVersion": "v1.15.3",
+ "gitCommit": "2d3c76f9091b6bec110a5e63777c332469e0cba2",
+ "gitTreeState": "clean",`)
+
+ cases := []struct {
+ data []byte
+ fail bool
+ expectedVer string
+ }{
+ {
+ data: okJSON,
+ fail: false,
+ expectedVer: "1.15",
+ },
+ {
+ data: invalidJSON,
+ fail: true,
+ },
+ }
+
+ for id, c := range cases {
+ t.Run(strconv.Itoa(id), func(t *testing.T) {
+ ver, err := extractVersion(c.data)
+ if !c.fail {
+ if err != nil {
+ t.Errorf("unexpected error: %v", err)
+ }
+ if c.expectedVer != ver {
+ t.Errorf("Expected %q but Got %q", c.expectedVer, ver)
+ }
+ } else {
+ if err == nil {
+ t.Errorf("Expected error")
+ }
+ }
+ })
+ }
+}
+
+func TestGetKubernetesURL(t *testing.T) {
+
+ resetEnvs := func() {
+ os.Unsetenv("KUBE_BENCH_K8S_ENV")
+ os.Unsetenv("KUBERNETES_SERVICE_HOST")
+ os.Unsetenv("KUBERNETES_SERVICE_PORT_HTTPS")
+ }
+
+ setEnvs := func() {
+ os.Setenv("KUBE_BENCH_K8S_ENV", "1")
+ os.Setenv("KUBERNETES_SERVICE_HOST", "testHostServer")
+ os.Setenv("KUBERNETES_SERVICE_PORT_HTTPS", "443")
+ }
+
+ cases := []struct {
+ useDefault bool
+ expected string
+ }{
+ {
+ useDefault: true,
+ expected: "https://kubernetes.default.svc/version",
+ },
+ {
+ useDefault: false,
+ expected: "https://testHostServer:443/version",
+ },
+ }
+ for id, c := range cases {
+ t.Run(strconv.Itoa(id), func(t *testing.T) {
+ resetEnvs()
+ defer resetEnvs()
+ if !c.useDefault {
+ setEnvs()
+ }
+ k8sURL := getKubernetesURL()
+
+ if !c.useDefault {
+ if k8sURL != c.expected {
+ t.Errorf("Expected %q but Got %q", k8sURL, c.expected)
+ }
+ } else {
+ if k8sURL != c.expected {
+ t.Errorf("Expected %q but Got %q", k8sURL, c.expected)
+ }
+ }
+ })
+ }
+
+}
diff --git a/cmd/util.go b/cmd/util.go
index c55e753..2c2967e 100644
--- a/cmd/util.go
+++ b/cmd/util.go
@@ -78,12 +78,14 @@ func cleanIDs(list string) map[string]bool {
func ps(proc string) string {
// TODO: truncate proc to 15 chars
// See https://github.com/aquasecurity/kube-bench/issues/328#issuecomment-506813344
+ glog.V(2).Info(fmt.Sprintf("ps - proc: %q", proc))
cmd := exec.Command("/bin/ps", "-C", proc, "-o", "cmd", "--no-headers")
out, err := cmd.Output()
if err != nil {
continueWithError(fmt.Errorf("%s: %s", cmd.Args, err), "")
}
+ glog.V(2).Info(fmt.Sprintf("ps - returning: %q", string(out)))
return string(out)
}
@@ -206,7 +208,9 @@ func verifyBin(bin string) bool {
// but apiserver is not a match for kube-apiserver
reFirstWord := regexp.MustCompile(`^(\S*\/)*` + bin)
lines := strings.Split(out, "\n")
+ glog.V(2).Info(fmt.Sprintf("verifyBin - lines(%d)", len(lines)))
for _, l := range lines {
+ glog.V(2).Info(fmt.Sprintf("reFirstWord.Match(%s)\n\n\n\n", l))
if reFirstWord.Match([]byte(l)) {
return true
}
@@ -271,6 +275,12 @@ Alternatively, you can specify the version with --version
`
func getKubeVersion() (string, error) {
+
+ if k8sVer, err := getKubeVersionFromRESTAPI(); err == nil {
+ glog.V(2).Info(fmt.Sprintf("Kubernetes REST API Reported version: %s", k8sVer))
+ return k8sVer, nil
+ }
+
// These executables might not be on the user's path.
_, err := exec.LookPath("kubectl")
--
GitLab