From 92f763a46a9000a4206e1003e4acb2363591619c Mon Sep 17 00:00:00 2001
From: Felix Kunde <felix-kunde@gmx.de>
Date: Mon, 15 Jul 2019 12:06:56 +0200
Subject: [PATCH] make run.sh executable from within e2e

---
 .travis.yml                           |  4 +--
 Makefile                              | 21 ++++-------
 README.md                             |  2 +-
 delivery.yaml                         |  2 +-
 docs/administrator.md                 |  4 +--
 docs/developer.md                     | 27 +++++++-------
 docs/diagrams/pod.tex                 |  2 +-
 docs/index.md                         |  2 +-
 docs/reference/cluster_manifest.md    |  2 +-
 docs/reference/operator_parameters.md |  6 ++--
 e2e/Dockerfile                        | 11 +++---
 e2e/Makefile                          | 52 +++++++++++++++++++++++++++
 e2e/README.md                         | 46 ++++++++++++++++++++++++
 e2e/run.sh                            | 26 +++++++++++---
 e2e/tests/__init__.py                 |  2 ++
 e2e/tests/test_e2e.py                 | 10 +++---
 pkg/util/config/config.go             |  2 +-
 run_operator_locally.sh               |  2 +-
 ui/Makefile                           |  1 -
 ui/operator_ui/spiloutils.py          |  6 ++--
 20 files changed, 169 insertions(+), 61 deletions(-)
 create mode 100644 e2e/Makefile
 create mode 100644 e2e/README.md
 create mode 100644 e2e/tests/__init__.py

diff --git a/.travis.yml b/.travis.yml
index 7e77ffb4..9740b43c 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -15,9 +15,9 @@ before_install:
   - go get github.com/mattn/goveralls
 
 install:
-  - make deps e2e-tools e2e-build
+  - make deps
 
 script:
   - hack/verify-codegen.sh
   - travis_wait 20 goveralls -service=travis-ci -package ./pkg/... -v
-  - make e2e-run
+  - make e2e
diff --git a/Makefile b/Makefile
index 5d97c817..a78ec722 100644
--- a/Makefile
+++ b/Makefile
@@ -1,4 +1,4 @@
-.PHONY: clean local test linux macos docker push scm-source.json e2e-run e2e-tools e2e-build
+.PHONY: clean local test linux macos docker push scm-source.json e2e
 
 BINARY ?= postgres-operator
 BUILD_FLAGS ?= -v
@@ -34,7 +34,10 @@ ifdef CDP_PULL_REQUEST_NUMBER
 	CDP_TAG := -${CDP_BUILD_VERSION}
 endif
 
-KIND_PATH := $(GOPATH)/bin
+ifndef GOPATH
+	GOPATH := $(HOME)/go
+endif
+
 PATH := $(GOPATH)/bin:$(PATH)
 SHELL := env PATH=$(PATH) $(SHELL)
 
@@ -92,15 +95,5 @@ test:
 	hack/verify-codegen.sh
 	@go test ./...
 
-e2e-build:
-	docker build --tag="postgres-operator-e2e-tests" -f e2e/Dockerfile .
-
-e2e-tools:
-	# install pinned version of 'kind' 
-	# leave the name as is to avoid overwriting official binary named `kind`
-	wget https://github.com/kubernetes-sigs/kind/releases/download/v0.3.0/kind-linux-amd64
-	chmod +x kind-linux-amd64
-	mv kind-linux-amd64 $(KIND_PATH)
-
-e2e-run: docker
-	e2e/run.sh
+e2e:
+	cd e2e; make tools test
diff --git a/README.md b/README.md
index 724cee77..a2771efa 100644
--- a/README.md
+++ b/README.md
@@ -64,7 +64,7 @@ There is a browser-friendly version of this documentation at
 * [The Postgres experience on K8s](docs/user.md)
 * [The Postgres Operator UI](docs/operator-ui.md)
 * [DBA options - from RBAC to backup](docs/administrator.md)
-* [Debug and extend the operator](docs/developer.md)
+* [Build, debug and extend the operator](docs/developer.md)
 * [Configuration options](docs/reference/operator_parameters.md)
 * [Postgres manifest reference](docs/reference/cluster_manifest.md)
 * [Command-line options and environment variables](docs/reference/command_line_and_environment.md)
diff --git a/delivery.yaml b/delivery.yaml
index e10b7b2c..8c4db2b9 100644
--- a/delivery.yaml
+++ b/delivery.yaml
@@ -44,7 +44,7 @@ pipeline:
         - desc: 'Run e2e tests'
           cmd: |
             cd $OPERATOR_TOP_DIR/postgres-operator
-            make e2e-tools e2e-build e2e-run
+            make e2e
         - desc: 'Push docker image'
           cmd: |
             export PATH=$PATH:$HOME/go/bin
diff --git a/docs/administrator.md b/docs/administrator.md
index d613917e..5eaf3ff7 100644
--- a/docs/administrator.md
+++ b/docs/administrator.md
@@ -98,7 +98,7 @@ on `configmaps` resources). This is also done intentionally to avoid breaking
 things if someone decides to configure the same service account in the
 operator's ConfigMap to run Postgres clusters.
 
-### Give K8S users access to create/list `postgresqls`
+### Give K8s users access to create/list `postgresqls`
 
 By default `postgresql` custom resources can only be listed and changed by
 cluster admins. To allow read and/or write access to other human users apply
@@ -363,7 +363,7 @@ used internally in K8s.
 
 ## Logical backups
 
-The operator can manage k8s cron jobs to run logical backups of Postgres
+The operator can manage K8s cron jobs to run logical backups of Postgres
 clusters. The cron job periodically spawns a batch job that runs a single pod.
 The backup script within this pod's container can connect to a DB for a logical
 backup. The operator updates cron jobs during Sync if the job schedule changes;
diff --git a/docs/developer.md b/docs/developer.md
index 86b01bbf..a94edefc 100644
--- a/docs/developer.md
+++ b/docs/developer.md
@@ -96,7 +96,7 @@ kubectl get pod -l name=postgres-operator
 The operator employs K8s-provided code generation to obtain deep copy methods
 and K8s-like APIs for its custom resource definitions, namely the
 Postgres CRD and the operator CRD. The usage of the code generation follows
-conventions from the k8s community. Relevant scripts live in the `hack`
+conventions from the K8s community. Relevant scripts live in the `hack`
 directory:
 * `update-codegen.sh` triggers code generation for the APIs defined in `pkg/apis/acid.zalan.do/`,
 * `verify-codegen.sh` checks if the generated code is up-to-date (to be used within CI).
@@ -247,23 +247,20 @@ kubectl logs acid-minimal-cluster-0
 
 ## End-to-end tests
 
-The operator provides reference e2e (end-to-end) tests to ensure various infra
-parts work smoothly together. Each e2e execution tests a Postgres Operator image
-built from the current git branch. The test runner starts a [kind](https://kind.sigs.k8s.io/)
-(local k8s) cluster and Docker container with tests. The k8s API client from
-within the container connects to the `kind` cluster using the standard Docker
-`bridge` network. The tests utilize examples from `/manifests` (ConfigMap is
-used for the operator configuration) to avoid maintaining yet another set of
-configuration files. The kind cluster is deleted if tests complete successfully.
+The operator provides reference end-to-end tests (e2e) (as Docker image) to
+ensure various infrastructure parts work smoothly together. Each e2e execution
+tests a Postgres Operator image built from the current git branch. The test
+runner creates a new local K8s cluster using [kind](https://kind.sigs.k8s.io/),
+utilizes provided manifest examples, and runs e2e tests contained in the `tests`
+folder. The K8s API client in the container connects to the `kind` cluster via
+the standard Docker `bridge` network. The kind cluster is deleted if tests
+finish successfully or on each new run in case it still exists.
 
-End-to-end tests are executed automatically during builds:
+End-to-end tests are executed automatically during builds (for more details,
+see the [README](../e2e/README.md) in the `e2e` folder):
 
 ```bash
-# invoke them from the project's top directory
-make e2e-run
-
-# install kind and build test image before first run
-make e2e-tools e2e-build
+make e2e
 ```
 
 End-to-end tests are written in Python and use `flake8` for code quality.
diff --git a/docs/diagrams/pod.tex b/docs/diagrams/pod.tex
index f4451399..29138464 100644
--- a/docs/diagrams/pod.tex
+++ b/docs/diagrams/pod.tex
@@ -38,7 +38,7 @@
       node[k8s-label] (app-label) {App}
       node[k8s-label, right=.25cm of app-label] (role-label) {Role}
       node[k8s-label, right=.25cm of role-label] (custom-label) {Custom}
-      node[label, below of=role-label] (k8s-label-label) {K8S Labels}
+      node[label, below of=role-label] (k8s-label-label) {K8s Labels}
       node[border, behind path,
            fit=(app-label)(role-label)(custom-label)(k8s-label-label)
       ] (k8s-labels) {};  \& \&
diff --git a/docs/index.md b/docs/index.md
index a61fc5dd..c0e78ac3 100644
--- a/docs/index.md
+++ b/docs/index.md
@@ -46,7 +46,7 @@ used to complement it.
 Here is a diagram, that summarizes what would be created by the operator, when a
 new Postgres cluster CRD is submitted:
 
-![postgresql-operator](diagrams/operator.png "K8S resources, created by operator")
+![postgresql-operator](diagrams/operator.png "K8s resources, created by operator")
 
 This picture is not complete without an overview of what is inside a single
 cluster pod, so let's zoom in:
diff --git a/docs/reference/cluster_manifest.md b/docs/reference/cluster_manifest.md
index ac609e6d..51be5855 100644
--- a/docs/reference/cluster_manifest.md
+++ b/docs/reference/cluster_manifest.md
@@ -135,7 +135,7 @@ These parameters are grouped directly under  the `spec` key in the manifest.
   to S3. Default: false. Optional.
 
 * **logicalBackupSchedule**
-  Schedule for the logical backup k8s cron job. Please take
+  Schedule for the logical backup K8s cron job. Please take
   [the reference schedule format](https://kubernetes.io/docs/tasks/job/automated-tasks-with-cron-jobs/#schedule)
   into account. Optional. Default is: "30 00 \* \* \*"
 
diff --git a/docs/reference/operator_parameters.md b/docs/reference/operator_parameters.md
index 0528147a..3e21340d 100644
--- a/docs/reference/operator_parameters.md
+++ b/docs/reference/operator_parameters.md
@@ -158,8 +158,8 @@ configuration they are grouped under the `kubernetes` key.
 
 * **pod_service_account_role_binding_definition**
   This definition must bind pod service account to a role with permission
-  sufficient for the pods to start and for Patroni to access k8s endpoints;
-  service account on its own lacks any such rights starting with k8s v1.8. If
+  sufficient for the pods to start and for Patroni to access K8s endpoints;
+  service account on its own lacks any such rights starting with K8s v1.8. If
   not explicitly defined by the user, a simple definition that binds the
   account to the operator's own 'zalando-postgres-operator' cluster role will
   be used. The default is empty.
@@ -416,7 +416,7 @@ yet officially supported.
 
 ## Logical backup
 
-These parameters configure a k8s cron job managed by the operator to produce
+These parameters configure a K8s cron job managed by the operator to produce
 Postgres logical backups. In the CRD-based configuration those parameters are
 grouped under the `logical_backup` key.
 
diff --git a/e2e/Dockerfile b/e2e/Dockerfile
index bd646b67..236942d0 100644
--- a/e2e/Dockerfile
+++ b/e2e/Dockerfile
@@ -1,13 +1,11 @@
 FROM ubuntu:18.04
 LABEL maintainer="Team ACID @ Zalando <team-acid@zalando.de>"
 
-WORKDIR /e2e
-
 COPY manifests ./manifests
-COPY e2e/requirements.txt e2e/tests ./
+COPY requirements.txt tests ./
 
 RUN apt-get update \
-    && apt-get install --no-install-recommends -y \ 
+    && apt-get install --no-install-recommends -y \
            python3 \
            python3-setuptools \
            python3-pip \
@@ -19,4 +17,7 @@ RUN apt-get update \
     && apt-get clean \
     && rm -rf /var/lib/apt/lists/*
 
-CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]
\ No newline at end of file
+ARG VERSION=dev
+RUN sed -i "s/__version__ = .*/__version__ = '${VERSION}'/" ./__init__.py
+
+CMD ["python3", "-m", "unittest", "discover", "--start-directory", ".", "-v"]
diff --git a/e2e/Makefile b/e2e/Makefile
new file mode 100644
index 00000000..0a4f42ba
--- /dev/null
+++ b/e2e/Makefile
@@ -0,0 +1,52 @@
+.PHONY: clean copy docker push tools test
+
+BINARY ?= postgres-operator-e2e-tests
+BUILD_FLAGS ?= -v
+CGO_ENABLED ?= 0
+ifeq ($(RACE),1)
+	BUILD_FLAGS += -race -a
+    CGO_ENABLED=1
+endif
+
+LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS)
+LDFLAGS ?= -X=main.version=$(VERSION)
+
+IMAGE            ?= registry.opensource.zalan.do/acid/$(BINARY)
+VERSION          ?= $(shell git describe --tags --always --dirty)
+TAG              ?= $(VERSION)
+GITHEAD          = $(shell git rev-parse --short HEAD)
+GITURL           = $(shell git config --get remote.origin.url)
+GITSTATU         = $(shell git status --porcelain || echo 'no changes')
+TTYFLAGS         = $(shell test -t 0 && echo '-it')
+
+ifndef GOPATH
+	GOPATH := $(HOME)/go
+endif
+
+KIND_PATH := $(GOPATH)/bin
+PATH := $(GOPATH)/bin:$(PATH)
+
+default: tools
+
+clean:
+	rm -fr manifests
+
+copy: clean
+	mkdir manifests
+	cp ../manifests -r .
+
+docker: copy
+	docker build --build-arg "VERSION=$(VERSION)" -t "$(IMAGE):$(TAG)" .
+
+push: docker
+	docker push "$(IMAGE):$(TAG)"
+
+tools: docker
+	# install pinned version of 'kind'
+	# leave the name as is to avoid overwriting official binary named `kind`
+	wget https://github.com/kubernetes-sigs/kind/releases/download/v0.4.0/kind-linux-amd64
+	chmod +x kind-linux-amd64
+	mv kind-linux-amd64 $(KIND_PATH)
+
+test:
+	./run.sh
diff --git a/e2e/README.md b/e2e/README.md
new file mode 100644
index 00000000..1d611bcd
--- /dev/null
+++ b/e2e/README.md
@@ -0,0 +1,46 @@
+# Postgres Operator end-to-end tests
+
+End-to-end tests shall ensure that the Postgres Operator does its job when
+applying manifests against a Kubernetes (K8s) environment. A test runner
+Dockerfile is provided to run e2e tests without the need to install K8s and
+its runtime `kubectl` in advance. The test runner uses
+[kind](https://kind.sigs.k8s.io/) to create a local K8s cluster which runs on
+Docker.
+
+## Prerequisites
+
+Docker
+Go
+
+## Build test runner
+
+In the directory of the cloned Postgres Operator repository change to the e2e
+folder and run:
+
+```bash
+make
+```
+
+This will build the `postgres-operator-e2e-tests` image and download the kind
+runtime.
+
+## Run tests
+
+In the e2e folder you can invoke tests either with `make test` or with:
+
+```bash
+./run.sh
+```
+
+To run both the build and test step you can invoke `make e2e` from the parent
+directory.
+
+## Covered use cases
+
+The current tests are all bundled in [`test_e2e.py`](tests/test_e2e.py):
+
+* support for multiple namespaces
+* scale Postgres cluster up and down
+* taint-based eviction of Postgres pods
+* invoking logical backup cron job
+* uniqueness of master pod
diff --git a/e2e/run.sh b/e2e/run.sh
index 3ee27297..237960b8 100755
--- a/e2e/run.sh
+++ b/e2e/run.sh
@@ -6,11 +6,26 @@ set -o nounset
 set -o pipefail
 IFS=$'\n\t'
 
+cd $(dirname "$0");
+
 readonly cluster_name="postgres-operator-e2e-tests"
-readonly operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}"  | head -1)
-readonly e2e_test_image=${cluster_name}
 readonly kubeconfig_path="/tmp/kind-config-${cluster_name}"
 
+function pull_images(){
+
+  operator_tag=$(git describe --tags --always --dirty)
+  if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator:${operator_tag}) ]]
+  then
+    docker pull registry.opensource.zalan.do/acid/postgres-operator:latest
+  fi
+  if [[ -z $(docker images -q registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:${operator_tag}) ]]
+  then
+    docker pull registry.opensource.zalan.do/acid/postgres-operator-e2e-tests:latest
+  fi
+
+  operator_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator" --format "{{.Repository}}:{{.Tag}}" | head -1)
+  e2e_test_image=$(docker images --filter=reference="registry.opensource.zalan.do/acid/postgres-operator-e2e-tests" --format "{{.Repository}}:{{.Tag}}" | head -1)
+}
 
 function start_kind(){
 
@@ -20,8 +35,9 @@ function start_kind(){
     kind-linux-amd64 delete cluster --name ${cluster_name}
   fi
 
-  kind-linux-amd64 create cluster --name ${cluster_name} --config ./e2e/kind-cluster-postgres-operator-e2e-tests.yaml
+  kind-linux-amd64 create cluster --name ${cluster_name} --config kind-cluster-postgres-operator-e2e-tests.yaml
   kind-linux-amd64 load docker-image "${operator_image}" --name ${cluster_name}
+  kind-linux-amd64 load docker-image "${e2e_test_image}" --name ${cluster_name}
   KUBECONFIG="$(kind-linux-amd64 get kubeconfig-path --name=${cluster_name})"
   export KUBECONFIG
 }
@@ -36,11 +52,12 @@ function set_kind_api_server_ip(){
 }
 
 function run_tests(){
+
   docker run --rm --mount type=bind,source="$(readlink -f ${kubeconfig_path})",target=/root/.kube/config -e OPERATOR_IMAGE="${operator_image}" "${e2e_test_image}"
 }
 
 function clean_up(){
-  unset KUBECONFIG 
+  unset KUBECONFIG
   kind-linux-amd64 delete cluster --name ${cluster_name}
   rm -rf ${kubeconfig_path}
 }
@@ -49,6 +66,7 @@ function main(){
 
   trap "clean_up" QUIT TERM EXIT
 
+  pull_images
   start_kind
   set_kind_api_server_ip
   run_tests
diff --git a/e2e/tests/__init__.py b/e2e/tests/__init__.py
new file mode 100644
index 00000000..7b7cafe4
--- /dev/null
+++ b/e2e/tests/__init__.py
@@ -0,0 +1,2 @@
+# This version is replaced during release process.
+__version__ = '2019.0.dev1'
diff --git a/e2e/tests/test_e2e.py b/e2e/tests/test_e2e.py
index c232ba7a..52aa0549 100644
--- a/e2e/tests/test_e2e.py
+++ b/e2e/tests/test_e2e.py
@@ -11,7 +11,7 @@ from kubernetes import client, config
 
 class EndToEndTestCase(unittest.TestCase):
     '''
-    Test interaction of the operator with multiple k8s components.
+    Test interaction of the operator with multiple K8s components.
     '''
 
     # `kind` pods may stuck in the `Terminating` phase for a few minutes; hence high test timeout
@@ -21,15 +21,15 @@ class EndToEndTestCase(unittest.TestCase):
     @timeout_decorator.timeout(TEST_TIMEOUT_SEC)
     def setUpClass(cls):
         '''
-        Deploy operator to a "kind" cluster created by /e2e/run.sh using examples from /manifests.
+        Deploy operator to a "kind" cluster created by run.sh using examples from /manifests.
         This operator deployment is to be shared among all tests.
 
-        /e2e/run.sh deletes the 'kind' cluster after successful run along with all operator-related entities.
+        run.sh deletes the 'kind' cluster after successful run along with all operator-related entities.
         In the case of test failure the cluster will stay to enable manual examination;
-        next invocation of "make e2e-run" will re-create it.
+        next invocation of "make test" will re-create it.
         '''
 
-        # set a single k8s wrapper for all tests
+        # set a single K8s wrapper for all tests
         k8s = cls.k8s = K8s()
 
         # operator deploys pod service account there on start up
diff --git a/pkg/util/config/config.go b/pkg/util/config/config.go
index ab9e43fd..48e4a700 100644
--- a/pkg/util/config/config.go
+++ b/pkg/util/config/config.go
@@ -84,7 +84,7 @@ type Config struct {
 	LogicalBackup
 
 	WatchedNamespace string            `name:"watched_namespace"`    // special values: "*" means 'watch all namespaces', the empty string "" means 'watch a namespace where operator is deployed to'
-	EtcdHost         string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use k8s as a DCS
+	EtcdHost         string            `name:"etcd_host" default:""` // special values: the empty string "" means Patroni will use K8s as a DCS
 	DockerImage      string            `name:"docker_image" default:"registry.opensource.zalan.do/acid/spilo-11:1.5-p9"`
 	Sidecars         map[string]string `name:"sidecar_docker_images"`
 	// default name `operator` enables backward compatibility with the older ServiceAccountName field
diff --git a/run_operator_locally.sh b/run_operator_locally.sh
index 2b2fa59d..f5044dc1 100755
--- a/run_operator_locally.sh
+++ b/run_operator_locally.sh
@@ -4,7 +4,7 @@
 # Optionally re-build the operator binary beforehand to test local changes
 
 # Known limitations:
-# 1) minikube provides a single node k8s cluster. That is, you will not be able test functions like pod
+# 1) minikube provides a single node K8s cluster. That is, you will not be able test functions like pod
 #    migration between multiple nodes locally
 # 2) this script configures the operator via configmap, not the operator CRD
 
diff --git a/ui/Makefile b/ui/Makefile
index f1370c7a..e4eed45e 100644
--- a/ui/Makefile
+++ b/ui/Makefile
@@ -10,7 +10,6 @@ endif
 
 LOCAL_BUILD_FLAGS ?= $(BUILD_FLAGS)
 LDFLAGS ?= -X=main.version=$(VERSION)
-DOCKERDIR = docker
 
 IMAGE            ?= registry.opensource.zalan.do/acid/$(BINARY)
 VERSION          ?= $(shell git describe --tags --always --dirty)
diff --git a/ui/operator_ui/spiloutils.py b/ui/operator_ui/spiloutils.py
index a707ed73..5a7ad686 100644
--- a/ui/operator_ui/spiloutils.py
+++ b/ui/operator_ui/spiloutils.py
@@ -206,7 +206,7 @@ def create_postgresql(cluster, namespace, definition):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S create request failed")
+        logger.exception("K8s create request failed")
         return False
 
 
@@ -221,7 +221,7 @@ def apply_postgresql(cluster, namespace, resource_name, definition):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S create request failed")
+        logger.exception("K8s create request failed")
         return False
 
 
@@ -236,7 +236,7 @@ def remove_postgresql(cluster, namespace, resource_name):
         r.raise_for_status()
         return True
     except Exception as ex:
-        logger.exception("K8S delete request failed")
+        logger.exception("K8s delete request failed")
         return False
 
 
-- 
GitLab