diff --git a/os/.gitignore b/os/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..d874ad67cc881f97e9cbc302a3bf49e5c2686e46 --- /dev/null +++ b/os/.gitignore @@ -0,0 +1 @@ +*.tar diff --git a/os/Containerfile b/os/Containerfile new file mode 100644 index 0000000000000000000000000000000000000000..b83a29ae9fda9352bfcf39fb19c76b47d553eba9 --- /dev/null +++ b/os/Containerfile @@ -0,0 +1,7 @@ +# Minimal HTTP server to serve the OSTree setup +# Taken from https://www.osbuild.org/guides/user-guide/building-ostree-images.html#setting-up-an-http-server +FROM registry.access.redhat.com/ubi8/ubi +RUN yum -y install httpd && yum clean all +ADD *.tar *.ks /var/www/html +EXPOSE 80 +CMD ["/usr/sbin/httpd", "-D", "FOREGROUND"] diff --git a/os/Makefile b/os/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..e84102754ce409b21f97e6a44a6445699462def1 --- /dev/null +++ b/os/Makefile @@ -0,0 +1,25 @@ + +include ../utils/help.mk +.PHONY: preflight +preflight: + # Check for required tools + command -v podman > /dev/null + command -v composer-cli > /dev/null + command -v awk > /dev/null + composer-cli status show | grep osbuild-composer + +.PHONY: build-kOS +build-kOS: preflight + composer-cli sources add kubernetes-repo.toml + composer-cli blueprints push kOS.toml + COMPOSE_ID=$$(composer-cli compose start kOS fedora-iot-commit | awk '{print $$2}') + while composer-cli compose list running | grep "$${COMPOSE_ID} RUNNING"; do sleep 30; done + composer-cli compose list + composer-cli compose image "$${COMPOSE_ID}" + +.PHONY: build-container +build-container: build-kOS + podman build -t ostree . + +.PHONY: build +build: build-kOS ## Building the base OS using osbuild / composer-cli diff --git a/os/README.md b/os/README.md new file mode 100644 index 0000000000000000000000000000000000000000..b5b918bae774b788083c9551c19288ec86c87a5e --- /dev/null +++ b/os/README.md @@ -0,0 +1,71 @@ +kOS - And OS born from chaos and need +=== + +kOS (spoken chaos) is an opinionated OS build to provide a stable base for the +overall Shivering-Isles infrastructure. The focus is to make it easy to set up +and run a vanilla Kubernetes cluster with SELinux, cri-o and full disk +encryption. + +The target plattform is `x86_64` on baremetal for now and is based on Fedora. + +Build process +--- + +As with everything: + +``` +make build +``` + +The build process uses [`osbuild`](https://www.osbuild.org) or more precisely +`composer-cli` to build an OSTree commit. Further a kickstart file is generated, +which can be loaded from any Fedora ISO, which should make installations easy +and achievable on baremetal. + +It uses an `fedora-iot-commit` type and produces a Fedora IoT image which will +container all we need. There will be work to fix the branding and also obey the +official Fedora Trademark guidelines. + +Problems during development / TODOs +--- + +During the process of fiddling this kOS together, there were various problems +and annoyances regarding the build process. These shall be documented here as +either todos or lessons learnd so no one has to face them again. + +### Modularity support + +There is none. At least non I could find. + +My solution to this, was to include all the packages I needed from modularity to +my COPR repository. + +### COPR support + +COPR is integrated like [any 3rd-party +repo](https://www.osbuild.org/guides/user-guide/managing-repositories.html#custom-3rd-party-repositories) +note that this file is in a custom TOML format, the type has to be `yum-baseurl` +and osbuild doesn't support `$releaseserver` or `$basearch`. The good news is +that you can enable `check_ssl`. + +### GPG check for COPR packages + +Validating GPG keys is definitely important, sadly, I couldn't pull it off with +this custom format yet. [Some +docs](https://github.com/osbuild/weldr-client#package-sources) indicate that +`gpgkey_urls` shoudl work, but this has only ended up in an exit code 1 from +[`rpmkeys`](https://github.com/osbuild/osbuild/blob/a96ee0f8ba6af47f1352e354355768ae2b56200c/stages/org.osbuild.rpm#L249-L252) +for me. + +### unprivileged, containerized builds + +Currently there is +[`docker-compose.yaml`](https://github.com/osbuild/osbuild-composer/blob/09f57b6c2f8199a0a8a9365dfb56b3e52e7a60e4/docker-compose.yml), +but it required a privileged container. for now I opted for a VM in which the +package was installed natively. Since it was a Fedora Silverblue setup, it was +already easy to test an image, by just deploying it from localhost and boot into +it. + +### Build ISO for installation + +I just didn't have time to look into this yet. diff --git a/os/kOS.toml b/os/kOS.toml new file mode 100644 index 0000000000000000000000000000000000000000..679ce19d6a9db93e95f2b312522e699740b8a9e0 --- /dev/null +++ b/os/kOS.toml @@ -0,0 +1,82 @@ +name = "kOS" +description = "A base system for my Kubernetes Cluster" +version = "0.3.3" +distro = "fedora-35" + +# Kubeadm to bootstrap vanilla Kubernetes clusters +[[packages]] +name = "kubernetes-kubeadm" +version = "1.22.*" + +# kubectl for master nodes, just in case +[[packages]] +name = "kubernetes-client" +version = "1.22.*" + +# A complicated name for kubelet +[[packages]] +name = "kubernetes-node" +version = "1.22.*" + +# CRI-compatible container runtime manager +[[packages]] +name = "cri-o" +version = "1.22.*" + +# cri-o only recommends runc as runtime, it's a good start… +[[packages]] +name = "runc" +version = "*" + +# Debugging local containers, is much easier with a CLI +[[packages]] +name = "cri-tools" +version = "1.22.*" + +# Kubernetes wants this… +[[packages]] +name = "iptables" +version = "*" + +# Needed to deploy LUKS unlocking via TPM by clevis, might be obsoleted by systemd-cryptsetup +# https://0pointer.net/blog/unlocking-luks2-volumes-with-tpm2-fido2-pkcs11-security-hardware-on-systemd-248.html +[[packages]] +name = "clevis-dracut" +version = "*" + +# Basic cockpit setup, but disabled by default, still nice if web administration is wanted +[[packages]] +name = "cockpit" +version = "*" + +# Provide metrics for cockpit, in case this node should be managed by cockpit +[[packages]] +name = "cockpit-pcp" +version = "*" + +# Time matters. +[[packages]] +name = "chrony" +version = "*" + +# iscsi will be needed for longhorn, if another CSI is used, it still won't hurt +[[packages]] +name = "iscsi-initiator-utils" +version = "*" + +# Allows node_exporter to collect temperature sensor data +[[packages]] +name = "lm_sensors" +version = "*" + +[customizations.services] +# Enable essential services for Kubernetes (kubelet needs to be enabled after initalizing the cluster) +enabled = ["sshd", "crio", "chronyd", "iscsid"] +# Disable services that (might) come with Fedora IoT but are not needed, like zezere +disabled = ["postfix", "firewalld", "zezere_ignition.timer", "zezere_ignition_banner", "rsyslogd"] + +# Name is program, let's use the most amazing locale to make everyone upset +[customizations.locale] +languages = ["en_GB.UTF-8"] +keyboard = "de" + diff --git a/os/kubernetes-repo.toml b/os/kubernetes-repo.toml new file mode 100644 index 0000000000000000000000000000000000000000..9ecb9f18615a3967cad65c12c297fa5c81545a95 --- /dev/null +++ b/os/kubernetes-repo.toml @@ -0,0 +1,16 @@ +id = "k8s-1.22" +name = "copr-Kubernetes-1.22" +# Since fedora 35 is hardcoded in the URL, we only support Fedora 35 +distros = ["fedora-35"] +# Has nothing to do with the type in the regular .repo file +type = "yum-baseurl" +# Remove all variables from the URL +url = "https://download.copr.fedorainfracloud.org/results/sheogorath/kubernetes-1.22/fedora-35-x86_64/" + +# This seems to be non-functional :/ +# gpgkey_urls = ["https://download.copr.fedorainfracloud.org/results/sheogorath/kubernetes-1.22/pubkey.gpg"] +# Since above doesn't work yet, this needs to be disabled +check_gpg = false +# At least there is TLS +check_ssl = true +system = false