From 37f5587085236988dde8acf3fad215a44d87d98b Mon Sep 17 00:00:00 2001
From: Stefan Prodan <stefan.prodan@gmail.com>
Date: Thu, 11 Feb 2021 15:20:19 +0200
Subject: [PATCH] Allow Flux to be deployed on tainted Kubernetes nodes
Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
---
cmd/flux/bootstrap.go | 4 ++++
cmd/flux/install.go | 13 ++++++++++---
docs/cmd/flux.md | 2 +-
docs/cmd/flux_bootstrap.md | 1 +
docs/cmd/flux_bootstrap_github.md | 1 +
docs/cmd/flux_bootstrap_gitlab.md | 1 +
docs/cmd/flux_install.md | 10 +++++++---
docs/guides/installation.md | 4 ++++
pkg/manifestgen/install/install_test.go | 5 +++++
pkg/manifestgen/install/options.go | 1 +
pkg/manifestgen/install/templates.go | 7 +++++++
11 files changed, 42 insertions(+), 7 deletions(-)
diff --git a/cmd/flux/bootstrap.go b/cmd/flux/bootstrap.go
index b31befd6..b81e3576 100644
--- a/cmd/flux/bootstrap.go
+++ b/cmd/flux/bootstrap.go
@@ -60,6 +60,7 @@ type bootstrapFlags struct {
requiredComponents []string
tokenAuth bool
clusterDomain string
+ tolerationKeys []string
}
const (
@@ -91,6 +92,8 @@ func init() {
bootstrapCmd.PersistentFlags().Var(&bootstrapArgs.logLevel, "log-level", bootstrapArgs.logLevel.Description())
bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.manifestsPath, "manifests", "", "path to the manifest directory")
bootstrapCmd.PersistentFlags().StringVar(&bootstrapArgs.clusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain")
+ bootstrapCmd.PersistentFlags().StringSliceVar(&bootstrapArgs.tolerationKeys, "toleration-keys", nil,
+ "list of toleration keys used to schedule the components pods onto nodes with matching taints")
bootstrapCmd.PersistentFlags().MarkHidden("manifests")
bootstrapCmd.PersistentFlags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64")
rootCmd.AddCommand(bootstrapCmd)
@@ -138,6 +141,7 @@ func generateInstallManifests(targetPath, namespace, tmpDir string, localManifes
Timeout: rootArgs.timeout,
TargetPath: targetPath,
ClusterDomain: bootstrapArgs.clusterDomain,
+ TolerationKeys: bootstrapArgs.tolerationKeys,
}
if localManifests == "" {
diff --git a/cmd/flux/install.go b/cmd/flux/install.go
index b0d301eb..5218055f 100644
--- a/cmd/flux/install.go
+++ b/cmd/flux/install.go
@@ -34,15 +34,18 @@ import (
var installCmd = &cobra.Command{
Use: "install",
- Short: "Install the toolkit components",
- Long: `The install command deploys the toolkit components in the specified namespace.
+ Short: "Install or upgrade Flux",
+ Long: `The install command deploys Flux in the specified namespace.
If a previous version is installed, then an in-place upgrade will be performed.`,
Example: ` # Install the latest version in the flux-system namespace
flux install --version=latest --namespace=flux-system
- # Dry-run install for a specific version and a series of components
+ # Install a specific version and a series of components
flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
+ # Install Flux onto tainted Kubernetes nodes
+ flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux
+
# Dry-run install with manifests preview
flux install --dry-run --verbose
@@ -66,6 +69,7 @@ var (
installArch flags.Arch
installLogLevel = flags.LogLevel(rootArgs.defaults.LogLevel)
installClusterDomain string
+ installTolerationKeys []string
)
func init() {
@@ -91,6 +95,8 @@ func init() {
installCmd.Flags().BoolVar(&installNetworkPolicy, "network-policy", rootArgs.defaults.NetworkPolicy,
"deny ingress access to the toolkit controllers from other namespaces using network policies")
installCmd.Flags().StringVar(&installClusterDomain, "cluster-domain", rootArgs.defaults.ClusterDomain, "internal cluster domain")
+ installCmd.Flags().StringSliceVar(&installTolerationKeys, "toleration-keys", nil,
+ "list of toleration keys used to schedule the components pods onto nodes with matching taints")
installCmd.Flags().MarkHidden("manifests")
installCmd.Flags().MarkDeprecated("arch", "multi-arch container image is now available for AMD64, ARMv7 and ARM64")
rootCmd.AddCommand(installCmd)
@@ -130,6 +136,7 @@ func installCmdRun(cmd *cobra.Command, args []string) error {
ManifestFile: fmt.Sprintf("%s.yaml", rootArgs.namespace),
Timeout: rootArgs.timeout,
ClusterDomain: installClusterDomain,
+ TolerationKeys: installTolerationKeys,
}
if installManifestsPath == "" {
diff --git a/docs/cmd/flux.md b/docs/cmd/flux.md
index 34b9ea85..6686573d 100644
--- a/docs/cmd/flux.md
+++ b/docs/cmd/flux.md
@@ -84,7 +84,7 @@ Command line utility for assembling Kubernetes CD pipelines the GitOps way.
* [flux delete](flux_delete.md) - Delete sources and resources
* [flux export](flux_export.md) - Export resources in YAML format
* [flux get](flux_get.md) - Get sources and resources
-* [flux install](flux_install.md) - Install the toolkit components
+* [flux install](flux_install.md) - Install or upgrade Flux
* [flux reconcile](flux_reconcile.md) - Reconcile sources and resources
* [flux resume](flux_resume.md) - Resume suspended resources
* [flux suspend](flux_suspend.md) - Suspend resources
diff --git a/docs/cmd/flux_bootstrap.md b/docs/cmd/flux_bootstrap.md
index 465edd96..6ee474cb 100644
--- a/docs/cmd/flux_bootstrap.md
+++ b/docs/cmd/flux_bootstrap.md
@@ -19,6 +19,7 @@ The bootstrap sub-commands bootstrap the toolkit components on the targeted Git
--network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
--registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd")
--token-auth when enabled, the personal access token will be used instead of SSH deploy key
+ --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints
-v, --version string toolkit version (default "latest")
--watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
```
diff --git a/docs/cmd/flux_bootstrap_github.md b/docs/cmd/flux_bootstrap_github.md
index 54c2fa67..ba185717 100644
--- a/docs/cmd/flux_bootstrap_github.md
+++ b/docs/cmd/flux_bootstrap_github.md
@@ -74,6 +74,7 @@ flux bootstrap github [flags]
--registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd")
--timeout duration timeout for this operation (default 5m0s)
--token-auth when enabled, the personal access token will be used instead of SSH deploy key
+ --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints
--verbose print generated objects
-v, --version string toolkit version (default "latest")
--watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
diff --git a/docs/cmd/flux_bootstrap_gitlab.md b/docs/cmd/flux_bootstrap_gitlab.md
index a718b648..5eb0c965 100644
--- a/docs/cmd/flux_bootstrap_gitlab.md
+++ b/docs/cmd/flux_bootstrap_gitlab.md
@@ -70,6 +70,7 @@ flux bootstrap gitlab [flags]
--registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd")
--timeout duration timeout for this operation (default 5m0s)
--token-auth when enabled, the personal access token will be used instead of SSH deploy key
+ --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints
--verbose print generated objects
-v, --version string toolkit version (default "latest")
--watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
diff --git a/docs/cmd/flux_install.md b/docs/cmd/flux_install.md
index 4765e8e0..faad5878 100644
--- a/docs/cmd/flux_install.md
+++ b/docs/cmd/flux_install.md
@@ -1,10 +1,10 @@
## flux install
-Install the toolkit components
+Install or upgrade Flux
### Synopsis
-The install command deploys the toolkit components in the specified namespace.
+The install command deploys Flux in the specified namespace.
If a previous version is installed, then an in-place upgrade will be performed.
```
@@ -17,9 +17,12 @@ flux install [flags]
# Install the latest version in the flux-system namespace
flux install --version=latest --namespace=flux-system
- # Dry-run install for a specific version and a series of components
+ # Install a specific version and a series of components
flux install --dry-run --version=v0.0.7 --components="source-controller,kustomize-controller"
+ # Install Flux onto tainted Kubernetes nodes
+ flux install --toleration-keys=node.kubernetes.io/dedicated-to-flux
+
# Dry-run install with manifests preview
flux install --dry-run --verbose
@@ -41,6 +44,7 @@ flux install [flags]
--log-level logLevel log level, available options are: (debug, info, error) (default info)
--network-policy deny ingress access to the toolkit controllers from other namespaces using network policies (default true)
--registry string container registry where the toolkit images are published (default "ghcr.io/fluxcd")
+ --toleration-keys strings list of toleration keys used to schedule the components pods onto nodes with matching taints
-v, --version string toolkit version (default "latest")
--watch-all-namespaces watch for custom resources in all namespaces, if set to false it will only watch the namespace where the toolkit is installed (default true)
```
diff --git a/docs/guides/installation.md b/docs/guides/installation.md
index c4f9bece..152335f2 100644
--- a/docs/guides/installation.md
+++ b/docs/guides/installation.md
@@ -70,6 +70,10 @@ flux bootstrap <GIT-PROVIDER> \
If you wish to install a specific version, use the Flux
[release tag](https://github.com/fluxcd/flux2/releases) e.g. `--version=v0.2.0`.
+If you wish to deploy the Flux components onto
+[tainted Kubernetes nodes](https://kubernetes.io/docs/concepts/scheduling-eviction/taint-and-toleration/),
+you can specify the toleration keys with `--toleration-keys=node.kubernetes.io/dedicated-to-flux`.
+
With `--path` you can configure the directory which will be used to reconcile the target cluster.
To control multiple clusters from the same Git repository, you have to set a unique path per
cluster e.g. `clusters/staging` and `clusters/production`:
diff --git a/pkg/manifestgen/install/install_test.go b/pkg/manifestgen/install/install_test.go
index 4fa6daf9..834b26d4 100644
--- a/pkg/manifestgen/install/install_test.go
+++ b/pkg/manifestgen/install/install_test.go
@@ -24,6 +24,7 @@ import (
func TestGenerate(t *testing.T) {
opts := MakeDefaultOptions()
+ opts.TolerationKeys = []string{"node.kubernetes.io/controllers"}
output, err := Generate(opts)
if err != nil {
t.Fatal(err)
@@ -36,5 +37,9 @@ func TestGenerate(t *testing.T) {
}
}
+ if !strings.Contains(output.Content, opts.TolerationKeys[0]) {
+ t.Errorf("toleration key '%s' not found", opts.TolerationKeys[0])
+ }
+
fmt.Println(output)
}
diff --git a/pkg/manifestgen/install/options.go b/pkg/manifestgen/install/options.go
index 0a15f821..a456007b 100644
--- a/pkg/manifestgen/install/options.go
+++ b/pkg/manifestgen/install/options.go
@@ -35,6 +35,7 @@ type Options struct {
Timeout time.Duration
TargetPath string
ClusterDomain string
+ TolerationKeys []string
}
func MakeDefaultOptions() Options {
diff --git a/pkg/manifestgen/install/templates.go b/pkg/manifestgen/install/templates.go
index 316435d1..1dcec5c9 100644
--- a/pkg/manifestgen/install/templates.go
+++ b/pkg/manifestgen/install/templates.go
@@ -137,6 +137,13 @@ spec:
imagePullSecrets:
- name: {{.ImagePullSecret}}
{{- end }}
+{{ if gt (len .TolerationKeys) 0 }}
+ tolerations:
+{{- range $i, $key := .TolerationKeys }}
+ - key: "{{$key}}"
+ operator: "Exists"
+{{- end }}
+{{- end }}
`
var labelsTmpl = `---
--
GitLab