diff --git a/Makefile b/Makefile index 473fbf17fa77897d64616b3bd3ec67fc5fa48de8..acf86ad8020fd736451b8384d979ce6bf077f3c0 100644 --- a/Makefile +++ b/Makefile @@ -21,7 +21,11 @@ cli-config: ## Configure koolbox CLI (To setup terraform values as well was toke .PHONY: deploy deploy: check-machine ## Deploy infrastructure on Hetzner Cloud cd ./terraform && make apply + date + sleep 300 + make kubernetes-install .PHONY: destroy destroy: check-machine ## Destroy infrastructure on Hetzner Cloud cd ./terraform && make destroy + for i in cp0{1..3}.$(TF_VAR_dns_domain); do ssh-keygen -R "$$i"; done diff --git a/README.md b/README.md index 928bad7b5017027545456340f3379c1076f14b1a..7b00575ee55a41aba44f231b5484a046eca79243 100644 --- a/README.md +++ b/README.md @@ -92,8 +92,9 @@ This toolchain is still under development. Before it will be used in production - [ ] Automate ingress-controller configuration for proxy-protocol - [ ] Automate hetzner cloud integration deployment ([hetzner-cloud-controller-manager](https://git.shivering-isles.com/github-mirror/hetznercloud/hcloud-cloud-controller-manager)) - [ ] Document usage and thoughts in repository and blog posts -- [ ] Automate deployment of Kubernetes -- [ ] Automate flux bootstrap +- [x] Automate deployment of Kubernetes +- [ ] Integrate OIDC-based authentication +- [x] Automate flux bootstrap - [ ] Automate flux OpenPGP bootstrap - [ ] Enforce SELinux on the deployed machines (Currently conflicts with Rook) - [ ] Encrypt root filesystems for all nodes @@ -103,6 +104,7 @@ This toolchain is still under development. Before it will be used in production - [ ] Move to immutable base-system - [ ] Automate system upgrades using Kubernetes - [ ] Automate system configuration using Kubernetes +- [ ] Automate Kubernetes upgrades - [ ] Integrate with [hcloud-dynfw](https://git.shivering-isles.com/sheogorath/hcloud-dynfw) - [ ] Automate deployment of [cluster autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler/cloudprovider/hetzner) diff --git a/terraform/.terraform.lock.hcl b/terraform/.terraform.lock.hcl index 6f546a0a2767702ecb3af25acf9bd9147ccb563b..52777e74f8b1a1540abab21c552c25c5ef63e360 100644 --- a/terraform/.terraform.lock.hcl +++ b/terraform/.terraform.lock.hcl @@ -99,23 +99,23 @@ provider "registry.terraform.io/hashicorp/template" { } provider "registry.terraform.io/hetznercloud/hcloud" { - version = "1.31.1" - constraints = "1.31.1" + version = "1.32.0" + constraints = "1.32.0" hashes = [ - "h1:Rg94ZvIoKP2IkMl+WflNsIgNOS1P29/Fwa39WZHPQvU=", - "zh:1ac55d8db278a85ee24a9269b0d85ee138242d9f8d9b9ba8b95dc4a02d659137", - "zh:4720d6d96f0603c988bd95c963aa014b0e1b07fdc0b2c76fe3cb521a7ba54f1a", - "zh:4c69e86d325de13247b887007b53f712ce53528d98c73f06ff0d757d1c6b52ac", - "zh:560517e62d6f14feda622268adc9cfc3045440367b58b73fdd954804b72ae4a3", - "zh:792e1b647dd583e42a5b65c104ffde7e8b77f173e08e62bf5ca6b4e901c10ff1", - "zh:8046990a2d7b5cb304a4d959196a5dc642b81fd158b1da50d1dd72039ba2093d", - "zh:885bb88cd934f68cbc2016c812b99a49fc3a358c19c82d14b9f3adde6d2497af", - "zh:9f8728f650a30afc5bba6c97d40decdb3fd846db35e68659a7967262427ffa6b", - "zh:a78b7369b6a077c8a82266515f1bbdfd1eaa98fc82fa3e34c1aa1bbadf4e5514", - "zh:aaf306f40b7c3f48732437f15366f4ce042e3885b914f19f4652ac9b600899b1", - "zh:af533eee1f85ce3126931f0c3c1fe455918f3525079e92e9d85ee391e42ff4fc", - "zh:b0ce67d5ee900127a14e616c1f7463b211204627742b4051c1b33f464b97679e", - "zh:b743cd1355ba7b37b60a66f79b0e779d8d6c8adc7bdec151d2b14994dec7b809", - "zh:cdb210a89af1bf1563f0c933acd14b86a6a01e6289231e317cf5704abf54c9e6", + "h1:idi73hJM5FTNXAKoAyr1+oVcqNk5Jqmq0GM+sGU7FiM=", + "zh:1957fda0a92ffe52bbfeb58aa21a2318eb63f4e1d312d86ebaf251e5a16f0c47", + "zh:206ddebfed83cc6c6c8300d008ceca0312f73baf621c8fa50c043d758dbf3c8d", + "zh:39aa1ff81a5e7f1e0b7e3283b7f8d3212f9ea03222d4835881537163f006d4a6", + "zh:57db72b5b70a34340896c5cdc036d694b248ccfc561e44b47aa1902559f91023", + "zh:610dd439f53c1825189ea999f606c989d88e4f60ce29a9466138cf3b321a105b", + "zh:72c019addc1a899604d3f9b281d402cde8f7a0f69ce207f298ddf7576c15dccb", + "zh:795b38ca4d54a1880a17dda45895861bb8780a257c1fa3a0d3b1ba5bbfbf1faa", + "zh:7e53a1596a7872d2258f65554d89f3c38395b102810a3c8276de46a9ce79ad69", + "zh:7ef56a5a4c8ad042f6a459b07a489049ec8a71de433a68f519599c2b4c1dc447", + "zh:93a0755435536eeec2bf7ea00dc03642e64374b925ed118fbd0313c899c67878", + "zh:955b83167abfcc0c7893b6e4554ace53583360ea5ffebb3ba61ece599ba9fb67", + "zh:9dccc7e73718c9aa5a47ea85d21f137b8136eed1a42846d0a803cc134cca1ae0", + "zh:b1007f8778997e2e2da78f074b96134eebb1c311ad287b9f6448d762d7ed2371", + "zh:d04421aeadeab1ea9b6870cf93aad2c1be5796f1b7545a9e115c31045f0704cd", ] } diff --git a/terraform/firewall.tf b/terraform/firewall.tf index bf85feb7ac312e735188bcf7780d75700e0f8c47..ec4916515ca094f56b42e749c4b9d8a0d01b2ca2 100644 --- a/terraform/firewall.tf +++ b/terraform/firewall.tf @@ -16,54 +16,68 @@ resource "hcloud_firewall" "k8s-node" { "::/0" ] } + rule { + description = "cAdvisor" + direction = "in" + protocol = "tcp" + port = "4194" + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] + } rule { description = "Kublet" direction = "in" protocol = "tcp" port = "10250" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] + } + rule { + description = "kube-proxy-metrics" + direction = "in" + protocol = "tcp" + port = "10249" + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Kubernetes NodePort" direction = "in" protocol = "tcp" port = "30000-32767" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Kubernetes NodePort" direction = "in" protocol = "udp" port = "30000-32767" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Calico BGP" direction = "in" protocol = "tcp" port = "179" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Calico VXLAN" direction = "in" protocol = "udp" port = "4789" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Calico Typha" direction = "in" protocol = "tcp" port = "5473" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Calico Wireguard" direction = "in" protocol = "udp" port = "51820" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } # Host level services, including the node exporter on ports 9100-9101. rule { @@ -71,7 +85,7 @@ resource "hcloud_firewall" "k8s-node" { direction = "in" protocol = "tcp" port = "9000-9999" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } # Host level services, including the node exporter on ports 9100-9101. rule { @@ -79,7 +93,7 @@ resource "hcloud_firewall" "k8s-node" { direction = "in" protocol = "udp" port = "9000-9999" - source_ips = [for s in concat(module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } } @@ -87,7 +101,7 @@ resource "hcloud_firewall" "k8s-node" { resource "hcloud_firewall" "k8s-master" { name = "k8s-master" apply_to { - label_selector = "k8s.io/master" + label_selector = "k8s.io/controlplane" } # ICMP is always a good idea @@ -107,28 +121,35 @@ resource "hcloud_firewall" "k8s-master" { direction = "in" protocol = "tcp" port = "6443" - source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.nodes.ipv4_addresses) : "${s}/32"] + source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "etcd" direction = "in" protocol = "tcp" - port = "2379-2381" - source_ips = [for s in module.nodes.ipv4_addresses : "${s}/32"] + port = "2380-2381" + source_ips = [for s in module.controllers.ipv4_addresses : "${s}/32"] + } + rule { + description = "etcd-metrics" + direction = "in" + protocol = "tcp" + port = "2379" + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { - description = "kube-scheduler" + description = "kube-scheduler-metrics" direction = "in" protocol = "tcp" port = "10251" - source_ips = [for s in module.nodes.ipv4_addresses : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { - description = "kube-controller-manager" + description = "kube-controller-manager-metrics" direction = "in" protocol = "tcp" port = "10252" - source_ips = [for s in module.nodes.ipv4_addresses : "${s}/32"] + source_ips = [for s in concat(module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } } @@ -155,13 +176,13 @@ resource "hcloud_firewall" "k8s-ingress" { direction = "in" protocol = "tcp" port = "32080" - source_ips = [for s in [hcloud_load_balancer.lb.ipv4] : "${s}/32"] + source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } rule { description = "Public HTTPS" direction = "in" protocol = "tcp" port = "32443" - source_ips = [for s in [hcloud_load_balancer.lb.ipv4] : "${s}/32"] + source_ips = [for s in concat([hcloud_load_balancer.lb.ipv4], module.controllers.ipv4_addresses, module.workers.ipv4_addresses) : "${s}/32"] } } diff --git a/terraform/loadbalancer.tf b/terraform/loadbalancer.tf index f01dca818304fd3be5c6c6d45a2c1261e39aa95c..3ff60529b48f2dea71a1f3ac193d316a5ba06ba3 100644 --- a/terraform/loadbalancer.tf +++ b/terraform/loadbalancer.tf @@ -7,7 +7,7 @@ resource "hcloud_load_balancer" "lb" { resource "hcloud_load_balancer_target" "lb_target_master" { type = "label_selector" load_balancer_id = hcloud_load_balancer.lb.id - label_selector = "k8s.io/master" + label_selector = "k8s.io/controlplane" use_private_ip = false } diff --git a/terraform/main.tf b/terraform/main.tf index cd1dd2a206c5869a571559e29d96970324b9b128..8570e34bb5f3ec9c87fc41b7a371e329c4cc19ff 100644 --- a/terraform/main.tf +++ b/terraform/main.tf @@ -6,17 +6,17 @@ resource "hcloud_placement_group" "k8s" { } } -module "nodes" { +module "controllers" { source = "./modules/hcloud_instance" instance_count = var.replicas_nodes location = var.location - name = "node" + name = "cp" dns_domain = var.dns_domain dns_zone_id = var.dns_zone_id image = var.image labels = { "k8s.io/node" = "true", - "k8s.io/master" = "true", + "k8s.io/controlplane" = "true", "k8s.io/ingress" = "true", } placement_group_id = hcloud_placement_group.k8s.id @@ -30,16 +30,17 @@ module "workers" { source = "./modules/hcloud_instance" instance_count = var.replicas_worker location = var.location - name = "worker" + name = "wk" dns_domain = var.dns_domain dns_zone_id = var.dns_zone_id image = var.image labels = { "k8s.io/node" = "true", - "k8s.io/worker" = "true" + "k8s.io/worker" = "true", + "k8s.io/ingress" = "true", } placement_group_id = hcloud_placement_group.k8s.id ssh_keys = data.hcloud_ssh_keys.all_keys.ssh_keys.*.name - server_type = "cx21" + server_type = "cx21-ceph" user_data = file("templates/cloud-init.tpl") } diff --git a/terraform/ssh.tf b/terraform/ssh.tf index 15ac889188ae2a562a2ca2f89ed377e8c9167e2b..26f71c7d5bd056e27e73b65733ed14ea4b76149a 100644 --- a/terraform/ssh.tf +++ b/terraform/ssh.tf @@ -5,9 +5,6 @@ data "hcloud_ssh_keys" "all_keys" { data "http" "myipv4" { url = "https://api4.ipify.org" } -data "http" "myipv6" { - url = "https://api6.ipify.org" -} resource "hcloud_firewall" "k8s-ssh" { name = "k8s-ssh" @@ -34,7 +31,6 @@ resource "hcloud_firewall" "k8s-ssh" { port = "22" source_ips = [ "${chomp(data.http.myipv4.body)}/32", - "${replace(chomp(data.http.myipv6.body), "/^([0-9a-f]+:[0-9a-f]+:[0-9a-f]+:[0-9a-f]+):.*/", "$1")}::/64", ] } } diff --git a/terraform/templates/cloud-init.tpl b/terraform/templates/cloud-init.tpl index 983a4a3c3269ab5fbc67d2e516c5a97123b338c4..617ddc4a4fc9d9eef05dbd05deaaf6f618c900a0 100644 --- a/terraform/templates/cloud-init.tpl +++ b/terraform/templates/cloud-init.tpl @@ -29,15 +29,16 @@ sysctl --system dnf install -y iptables +# Disable systemd-resolved for CoreDNS +rm -f /etc/resolv.conf +cp /run/systemd/resolve/resolv.conf /etc/resolv.conf +systemctl disable --now systemd-resolved + # Prepare NetworkManager for Calico cat <<EOF | sudo tee /etc/NetworkManager/conf.d/calico.conf [keyfile] unmanaged-devices=interface-name:cali*;interface-name:tunl*;interface-name:vxlan.calico;interface-name:wireguard.cali EOF - -# Disable systemd-resolved for CoreDNS -systemctl disable --now systemd-resolved -rm /etc/resolv.conf systemctl restart NetworkManager reboot diff --git a/terraform/versions.tf b/terraform/versions.tf index 98407af4638033cf43344a12bf593509afbd45ca..75b41e7cdc57b2ffb3968174aab57322c8d8dc5e 100644 --- a/terraform/versions.tf +++ b/terraform/versions.tf @@ -6,7 +6,7 @@ terraform { } hcloud = { source = "hetznercloud/hcloud" - version = "1.31.1" + version = "1.32.0" } template = { source = "hashicorp/template" diff --git a/utils/flux.mk b/utils/flux.mk new file mode 100644 index 0000000000000000000000000000000000000000..a2b8f69e130717515de9d4aa95d90d02dc9211ed --- /dev/null +++ b/utils/flux.mk @@ -0,0 +1,22 @@ +.PHONY: flux-preflight +flux-preflight: + # Checking for required variables + env | grep -Pe '^GITLAB_TOKEN' > /dev/null + +.PHONY: flux-watch +flux-watch: ## flux: Show currently deployed resources an their status in all namespaces + koolbox flux get all --all-namespaces + +.PHONY: flux-update-git +flux-update-git: ## flux: Reload flux-system repository + koolbox flux reconcile source git flux-system + +.PHONY: flux-bootstrap +flux-bootstrap: flux-preflight + flux bootstrap gitlab \ + --hostname=git.shivering-isles.com \ + --ssh-hostname=git.shivering-isles.com:2222 \ + --ssh-key-algorithm ed25519 \ + --owner=shivering-isles \ + --repository=infrastructure-gitops \ + --path=clusters/k8s01 diff --git a/utils/git.mk b/utils/git.mk new file mode 100644 index 0000000000000000000000000000000000000000..dece38bf7e98bcb6b44e44a1b97f1549881ba6e1 --- /dev/null +++ b/utils/git.mk @@ -0,0 +1,5 @@ +.PHONY: commit +commit: ## Commit changes to git, push them to the remote and inform flux about it + git commit -v + git push + make flux-update-git diff --git a/utils/kubernetes-init.mk b/utils/kubernetes-init.mk new file mode 100644 index 0000000000000000000000000000000000000000..a179d73914f3e5779cdc3ee80ee6075733a881b6 --- /dev/null +++ b/utils/kubernetes-init.mk @@ -0,0 +1,40 @@ +.PHONY: ssh-init-hosts +ssh-init-hosts: + ssh -o StrictHostKeyChecking=no cp01.$(TF_VAR_dns_domain) echo "Hello World" + ssh -o StrictHostKeyChecking=no cp02.$(TF_VAR_dns_domain) echo "Hello World" + ssh -o StrictHostKeyChecking=no cp03.$(TF_VAR_dns_domain) echo "Hello World" + +.PHONY: kubeadm-init +kubeadm-init: + ssh cp01.$(TF_VAR_dns_domain) kubeadm init --control-plane-endpoint "api.$(TF_VAR_dns_domain):6443" --upload-certs --pod-network-cidr "192.168.0.0/16" + ssh cp01.$(TF_VAR_dns_domain) systemctl enable kubelet.service + +.PHONY: kubeadm-copy-config +kubeadm-copy-config: ## Copy Kubernetes admin config from cp01 to the local machine + scp cp01.$(TF_VAR_dns_domain):/etc/kubernetes/admin.conf /root/.kube/config + +.PHONY: kubeadm-join-masters +kubeadm-join-masters: + ssh cp02.$(TF_VAR_dns_domain) $$(ssh cp01.$(TF_VAR_dns_domain) kubeadm token create --ttl 1h --print-join-command --certificate-key "$$(ssh cp01.$(TF_VAR_dns_domain) kubeadm init phase upload-certs --upload-certs | tail -1)" | tail -1) + ssh cp02.$(TF_VAR_dns_domain) systemctl enable kubelet.service + ssh cp03.$(TF_VAR_dns_domain) $$(ssh cp01.$(TF_VAR_dns_domain) kubeadm token create --ttl 1h --print-join-command --certificate-key "$$(ssh cp01.$(TF_VAR_dns_domain) kubeadm init phase upload-certs --upload-certs | tail -1)" | tail -1) + ssh cp03.$(TF_VAR_dns_domain) systemctl enable kubelet.service + +.PHONY: kubectl-remove-first-master-taints +kubectl-remove-first-master-taints: + kubectl taint nodes cp01.$(TF_VAR_dns_domain) node-role.kubernetes.io/master- + +.PHONY: kubectl-remove-all-master-taints +kubectl-remove-all-master-taints: + kubectl taint nodes --all node-role.kubernetes.io/master- || true + +.PHONY: kubectl-delete-wrong-subnet +kubectl-delete-wrong-subnet: + kubectl get pods --all-namespaces -o wide | grep 10.85. | awk '{print "-n " $$1 " " $$2}' | xargs -L 1 kubectl delete pod + +.PHONY: kubectl-prepare-hcloud-csi +kubectl-prepare-hcloud-csi: + kubectl -n kube-system create secret generic --from-literal token=$CLOUD_TOKEN --dry-run=client -o yaml hcloud-csi | kubectl apply -f - + +.PHONY: kubernetes-install +kubernetes-install: ssh-init-hosts kubeadm-init kubeadm-copy-config kubectl-remove-first-master-taints kubectl-prepare-hcloud-csi flux-bootstrap kubeadm-join-masters kubectl-remove-all-master-taints kubectl-delete-wrong-subnet