diff --git a/.github/workflows/publish_on_master.yml b/.github/workflows/publish_on_master.yml index 3b064d1e7d7584a68e5e96c19bb7f0a5a34fad1c..58563ec32030d17450871cd6825e0e0b51c53572 100644 --- a/.github/workflows/publish_on_master.yml +++ b/.github/workflows/publish_on_master.yml @@ -26,3 +26,7 @@ jobs: tags: ${{ github.repository_owner }}/hcloud-csi-driver:latest cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max + - name: "make docker plugin" + run: | + cd deploy/docker-swarm/pkg + make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=latest-swarm diff --git a/.github/workflows/publish_on_tag.yml b/.github/workflows/publish_on_tag.yml index 5d6d2dbaf72050b508f1809fe1d56bdab5d5a927..6ef2a14215f8b59da1641fa8fc762f577b2b5f98 100644 --- a/.github/workflows/publish_on_tag.yml +++ b/.github/workflows/publish_on_tag.yml @@ -42,6 +42,12 @@ jobs: tags: ${{ github.repository_owner }}/hcloud-csi-driver:${{ steps.release_version.outputs.value}} cache-from: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache cache-to: type=registry,ref=${{ github.repository_owner }}/hcloud-csi-driver:buildcache,mode=max + - name: "make docker plugin" + env: + RELEASE_VERSION: ${{ steps.release_version.outputs.value}} + run: | + cd deploy/docker-swarm/pkg + make push PLUGIN_NAME=${{ github.repository_owner }}/hcloud-csi-driver PLUGIN_TAG=$RELEASE_VERSION-swarm - name: Run GoReleaser uses: goreleaser/goreleaser-action@v2 with: diff --git a/cmd/aio/README.md b/cmd/aio/README.md new file mode 100644 index 0000000000000000000000000000000000000000..99852fb773702bdc3e804e3fce2cd3ef48d3f74c --- /dev/null +++ b/cmd/aio/README.md @@ -0,0 +1,3 @@ +This contains an all in one binary (aio). This is required +for orchestrators such as Docker Swarm which need all endpoints in a single +API. \ No newline at end of file diff --git a/cmd/aio/main.go b/cmd/aio/main.go new file mode 100644 index 0000000000000000000000000000000000000000..5c58139af0fb356c687b13165f9f762e0325e1be --- /dev/null +++ b/cmd/aio/main.go @@ -0,0 +1,125 @@ +package main + +import ( + "fmt" + "os" + "strconv" + "strings" + + proto "github.com/container-storage-interface/spec/lib/go/csi" + "github.com/go-kit/kit/log" + "github.com/go-kit/kit/log/level" + "github.com/hetznercloud/csi-driver/api" + "github.com/hetznercloud/csi-driver/app" + "github.com/hetznercloud/csi-driver/driver" + "github.com/hetznercloud/csi-driver/volumes" + "github.com/hetznercloud/hcloud-go/hcloud/metadata" +) + +var logger log.Logger + +func main() { + logger = app.CreateLogger() + + m := app.CreateMetrics(logger) + + hcloudClient, err := app.CreateHcloudClient(m.Registry(), logger) + if err != nil { + level.Error(logger).Log( + "msg", "failed to initialize hcloud client", + "err", err, + ) + os.Exit(1) + } + + metadataClient := metadata.NewClient(metadata.WithInstrumentation(m.Registry())) + + // node + serverID, err := metadataClient.InstanceID() + if err != nil { + level.Error(logger).Log("msg", "failed to fetch server ID from metadata service", "err", err) + os.Exit(1) + } + + serverAZ, err := metadataClient.AvailabilityZone() + if err != nil { + level.Error(logger).Log("msg", "failed to fetch server availability-zone from metadata service", "err", err) + os.Exit(1) + } + parts := strings.Split(serverAZ, "-") + if len(parts) != 2 { + level.Error(logger).Log("msg", fmt.Sprintf("unexpected server availability zone: %s", serverAZ), "err", err) + os.Exit(1) + } + serverLocation := parts[0] + + level.Info(logger).Log("msg", "Fetched data from metadata service", "id", serverID, "location", serverLocation) + + volumeMountService := volumes.NewLinuxMountService( + log.With(logger, "component", "linux-mount-service"), + ) + volumeResizeService := volumes.NewLinuxResizeService( + log.With(logger, "component", "linux-resize-service"), + ) + volumeStatsService := volumes.NewLinuxStatsService( + log.With(logger, "component", "linux-stats-service"), + ) + nodeService := driver.NewNodeService( + log.With(logger, "component", "driver-node-service"), + strconv.Itoa(serverID), + serverLocation, + volumeMountService, + volumeResizeService, + volumeStatsService, + ) + + // controller + volumeService := volumes.NewIdempotentService( + log.With(logger, "component", "idempotent-volume-service"), + api.NewVolumeService( + log.With(logger, "component", "api-volume-service"), + hcloudClient, + ), + ) + controllerService := driver.NewControllerService( + log.With(logger, "component", "driver-controller-service"), + volumeService, + serverLocation, + ) + + // common + identityService := driver.NewIdentityService( + log.With(logger, "component", "driver-identity-service"), + ) + + // common + listener, err := app.CreateListener() + if err != nil { + level.Error(logger).Log( + "msg", "failed to create listener", + "err", err, + ) + os.Exit(1) + } + + grpcServer := app.CreateGRPCServer(logger, m.UnaryServerInterceptor()) + + // controller + proto.RegisterControllerServer(grpcServer, controllerService) + // common + proto.RegisterIdentityServer(grpcServer, identityService) + // node + proto.RegisterNodeServer(grpcServer, nodeService) + + m.InitializeMetrics(grpcServer) + + identityService.SetReady(true) + + if err := grpcServer.Serve(listener); err != nil { + level.Error(logger).Log( + "msg", "grpc server failed", + "err", err, + ) + os.Exit(1) + } +} diff --git a/deploy/docker-swarm/.gitignore b/deploy/docker-swarm/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..2d8a3023d5fe3e1d801e3604e71843e59ca8138f --- /dev/null +++ b/deploy/docker-swarm/.gitignore @@ -0,0 +1 @@ +plugin \ No newline at end of file diff --git a/deploy/docker-swarm/README.md b/deploy/docker-swarm/README.md new file mode 100644 index 0000000000000000000000000000000000000000..1c55e72573de8ea41cbdef5096b82224c8b86e7d --- /dev/null +++ b/deploy/docker-swarm/README.md @@ -0,0 +1,104 @@ +# Docker Swarm Hetzner CSI plugin + +Currently in Beta. Please consult the Docker Swarm documentation +for cluster volumes (=CSI) support at https://github.com/moby/moby/blob/master/docs/cluster_volumes.md + +The community is tracking the state of support for CSI in Docker Swarm over at https://github.com/olljanat/csi-plugins-for-docker-swarm + +## How to install the plugin + +Run the following steps on all nodes (especially master nodes). +The simplest way to achieve this + +1. Create a read+write API token in the [Hetzner Cloud Console](https://console.hetzner.cloud/). + +2. Install the plugin + +Note that docker plugins without a tag in the alias currently get `:latest` appended. To prevent this from happening, we will use +the fake tag `:swarm` instead. + +```bash +docker plugin install --disable --alias hetznercloud/hcloud-csi-driver:swarm --grant-all-permissions hetznercloud/hcloud-csi-driver:<version>-swarm +``` + +3. Set HCLOUD_TOKEN + +```bash +docker plugin set hetznercloud/hcloud-csi-driver:swarm HCLOUD_TOKEN=<your token> +``` + +4. Enable plugin + +```bash +docker plugin enable hetznercloud/hcloud-csi-driver:swarm +``` + +## How to create a volume + +Example: Create a volume wih size 50G in Nuremberg: + +```bash +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing onewriter --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 +``` + +We can now use this in a service: + +```bash +docker service create --name hcloud-debug-serv1 --mount type=cluster,src=hcloud-debug1,dst=/srv/www nginx:alpine +``` + +Note that only scope `single` is supported as Hetzner Cloud volumes can only be attached to one node at a time + +We can however share the volume on multiple containers on the same host: + +```bash +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes 50G --type mount --sharing all --scope single hcloud-debug1 --topology-required csi.hetzner.cloud/location=nbg1 +``` + +After creation we can now use this volume with `--sharing all` in more than one replica: + +```bash +docker service create --name hcloud-debug-serv2 --mount type=cluster,src=hcloud-debug2,dst=/srv/www nginx:alpine +docker service scale hcloud-debug-serv2=2 +``` + +## How to resize a docker swarm Hetzner CSI volume + +Currently, the Docker Swarm CSI support does not come with support for volume resizing. See [this ticket](https://github.com/moby/moby/issues/44985) for the current state on the Docker side. +The following explains a step by step guide on how to do this manually instead. + +Please test the following on a Swarm with the same version as your target cluster +as this strongly depends on the logic of `docker volume rm -f` not deleting the cloud volume. + +### Steps + +1. Drain Volume + +``` +docker volume update <volume-name> --availability drain +``` + +This way, we ensure that all services stop using the volume. + +2. Force remove volume on cluster + +``` +docker volume rm -f <volume-name> +``` + +4. Resize Volume in Hetzner UI +5. Attach Volume to temporary server manually +6. Run resize2fs manually +7. Detach Volume from temporary server manually +8. Recreate Volume with new size to make it known to Swarm again + +``` +docker volume create --driver hetznercloud/hcloud-csi-driver:swarm --required-bytes <new-size> --type mount --sharing onewriter --scope single <volume-name> +``` + +9. Verify that volume exists again: + +``` +docker volume ls --cluster +``` + diff --git a/deploy/docker-swarm/pkg/Dockerfile b/deploy/docker-swarm/pkg/Dockerfile new file mode 100644 index 0000000000000000000000000000000000000000..cc98e664291d1a3b72bf99abfe14c05261d8073d --- /dev/null +++ b/deploy/docker-swarm/pkg/Dockerfile @@ -0,0 +1,15 @@ +FROM golang:1.19 as builder +WORKDIR /csi +ADD go.mod go.sum /csi/ +RUN go mod download +ADD . /csi/ +RUN ls -al +RUN GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -o aio.bin github.com/hetznercloud/csi-driver/cmd/aio + +FROM --platform=linux/amd64 alpine:3.15 +RUN apk add --no-cache ca-certificates e2fsprogs xfsprogs blkid xfsprogs-extra e2fsprogs-extra btrfs-progs cryptsetup +ENV GOTRACEBACK=all +RUN mkdir -p /plugin +COPY --from=builder /csi/aio.bin /plugin + +ENTRYPOINT [ "/plugin/aio.bin" ] diff --git a/deploy/docker-swarm/pkg/LICENSE b/deploy/docker-swarm/pkg/LICENSE new file mode 100644 index 0000000000000000000000000000000000000000..5dea4c0ef69ee845d98870d50369d20d93d0f220 --- /dev/null +++ b/deploy/docker-swarm/pkg/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2018 Leo Antunes (base packaging code from https://github.com/costela/docker-volume-hetzner) + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. \ No newline at end of file diff --git a/deploy/docker-swarm/pkg/Makefile b/deploy/docker-swarm/pkg/Makefile new file mode 100644 index 0000000000000000000000000000000000000000..d3215e9065f2a1fba94a0ee2c0c4a0298a6aa07a --- /dev/null +++ b/deploy/docker-swarm/pkg/Makefile @@ -0,0 +1,31 @@ +PLUGIN_NAME = hetznercloud/hcloud-csi-driver +PLUGIN_TAG ?= $(shell git describe --tags --exact-match 2> /dev/null || echo dev)-swarm + +all: create + +clean: + @rm -rf ./plugin + @docker container rm -vf tmp_plugin_build || true + +rootfs: clean + docker image build -f Dockerfile -t ${PLUGIN_NAME}:rootfs ../../../ + mkdir -p ./plugin/rootfs + docker container create --name tmp_plugin_build ${PLUGIN_NAME}:rootfs + docker container export tmp_plugin_build | tar -x -C ./plugin/rootfs + cp config.json ./plugin/ + docker container rm -vf tmp_plugin_build + +create: rootfs + docker plugin rm -f ${PLUGIN_NAME}:${PLUGIN_TAG} 2> /dev/null || true + docker plugin create ${PLUGIN_NAME}:${PLUGIN_TAG} ./plugin + +enable: create + docker plugin enable ${PLUGIN_NAME}:${PLUGIN_TAG} + +push: create + docker plugin push ${PLUGIN_NAME}:${PLUGIN_TAG} + +push_latest: create + docker plugin push ${PLUGIN_NAME}:latest-swarm + +.PHONY: clean rootfs create enable push diff --git a/deploy/docker-swarm/pkg/README.md b/deploy/docker-swarm/pkg/README.md new file mode 100644 index 0000000000000000000000000000000000000000..93880433700261f98b79764148b49697d096eb6d --- /dev/null +++ b/deploy/docker-swarm/pkg/README.md @@ -0,0 +1,6 @@ +a lot in this directory comes from work originally done +by other awesome people. + +Before CSI support, Docker Swarm volumes +were graciously supported by @costela over at: +https://github.com/costela/docker-volume-hetzner \ No newline at end of file diff --git a/deploy/docker-swarm/pkg/config.json b/deploy/docker-swarm/pkg/config.json new file mode 100644 index 0000000000000000000000000000000000000000..d518ed27f3b263886fe1252625857c65de0e93c7 --- /dev/null +++ b/deploy/docker-swarm/pkg/config.json @@ -0,0 +1,68 @@ +{ + "description": "Hetzner csi-driver plugin for Docker", + "documentation": "https://github.com/hetznercloud/csi-driver", + "entrypoint": [ + "/plugin/aio.bin" + ], + "env": [ + { + "name": "HCLOUD_TOKEN", + "description": "authentication token to use when accessing the Hetzner Cloud API", + "settable": [ + "value" + ], + "value": "" + }, + { + "name": "CSI_ENDPOINT", + "description": "the CSI endpoint to listen to internally", + "settable": [], + "value": "unix:///run/docker/plugins/hetzner-csi.sock" + }, + { + "name": "LOG_LEVEL", + "description": "the log level to use", + "settable": [ + "value" + ], + "value": "debug" + }, + { + "name": "FORCE_STAGING_SUPPORT", + "description": "workaround: force staging support to make Docker 23.0.0 work without https://github.com/moby/swarmkit/pull/3116", + "settable": ["value"], + "value": "true" + } + ], + "interface": { + "socket": "hetzner-csi.sock", + "types": [ + "docker.csicontroller/1.0", + "docker.csinode/1.0" + ] + }, + "linux": { + "allowAllDevices": true, + "capabilities": [ + "CAP_SYS_ADMIN", + "CAP_CHOWN" + ] + }, + "mounts": [ + { + "description": "used to access the dynamically attached block devices", + "destination": "/dev", + "options": [ + "rbind", + "rshared" + ], + "name": "dev", + "source": "/dev/", + "type": "bind" + } + ], + "network": { + "type": "host" + }, + "propagatedmount": "/data/published" +} \ No newline at end of file diff --git a/driver/node.go b/driver/node.go index 8ec2478cdd3fa79d40bd9b959f55abc179865d7e..e818371029a3f7a671ffadb5dc4cc983724e02f9 100644 --- a/driver/node.go +++ b/driver/node.go @@ -3,6 +3,7 @@ package driver import ( "context" "fmt" + "os" proto "github.com/container-storage-interface/spec/lib/go/csi" "github.com/go-kit/kit/log" @@ -19,6 +20,11 @@ type NodeService struct { volumeMountService volumes.MountService volumeResizeService volumes.ResizeService volumeStatsService volumes.StatsService + // enable volume staging api to workaround + // docker CSI support not working properly + // if a plugin does not support staging + // see https://github.com/moby/swarmkit/pull/3116 + forceVolumeStaging bool } func NewNodeService( @@ -36,17 +42,20 @@ func NewNodeService( volumeMountService: volumeMountService, volumeResizeService: volumeResizeService, volumeStatsService: volumeStatsService, + forceVolumeStaging: os.Getenv("FORCE_STAGING_SUPPORT") == "true", } } const encryptionPassphraseKey = "encryption-passphrase" func (s *NodeService) NodeStageVolume(ctx context.Context, req *proto.NodeStageVolumeRequest) (*proto.NodeStageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "not supported") + // while we dont do anything here, Swarm 23.0.1 might require this + return &proto.NodeStageVolumeResponse{}, nil } func (s *NodeService) NodeUnstageVolume(ctx context.Context, req *proto.NodeUnstageVolumeRequest) (*proto.NodeUnstageVolumeResponse, error) { - return nil, status.Error(codes.Unimplemented, "not supported") + // while we dont do anything here, Swarm 23.0.1 might require this + return &proto.NodeUnstageVolumeResponse{}, nil } func (s *NodeService) NodePublishVolume(ctx context.Context, req *proto.NodePublishVolumeRequest) (*proto.NodePublishVolumeResponse, error) { @@ -148,24 +157,37 @@ func (s *NodeService) NodeGetVolumeStats(ctx context.Context, req *proto.NodeGet } func (s *NodeService) NodeGetCapabilities(ctx context.Context, req *proto.NodeGetCapabilitiesRequest) (*proto.NodeGetCapabilitiesResponse, error) { - resp := &proto.NodeGetCapabilitiesResponse{ - Capabilities: []*proto.NodeServiceCapability{ - { - Type: &proto.NodeServiceCapability_Rpc{ - Rpc: &proto.NodeServiceCapability_RPC{ - Type: proto.NodeServiceCapability_RPC_EXPAND_VOLUME, - }, + capabilities := []*proto.NodeServiceCapability{ + { + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_EXPAND_VOLUME, }, }, - { - Type: &proto.NodeServiceCapability_Rpc{ - Rpc: &proto.NodeServiceCapability_RPC{ - Type: proto.NodeServiceCapability_RPC_GET_VOLUME_STATS, - }, + }, + { + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_GET_VOLUME_STATS, }, }, }, } + + if s.forceVolumeStaging { + capabilities = append(capabilities, &proto.NodeServiceCapability{ + Type: &proto.NodeServiceCapability_Rpc{ + Rpc: &proto.NodeServiceCapability_RPC{ + Type: proto.NodeServiceCapability_RPC_STAGE_UNSTAGE_VOLUME, + }, + }, + }) + } + + resp := &proto.NodeGetCapabilitiesResponse{ + Capabilities: capabilities, + } + return resp, nil }