diff --git a/chart/.snapshots/default.yaml b/chart/.snapshots/default.yaml index 3a0fce4f969e1b950e0fc8ea600f3f9ec1123cdf..3b00f7f20c63892167a54ce63004df280541bae6 100644 --- a/chart/.snapshots/default.yaml +++ b/chart/.snapshots/default.yaml @@ -310,6 +310,7 @@ spec: args: - --feature-gates=Topology=true - --default-fstype=ext4 + - --extra-create-metadata volumeMounts: - name: socket-dir mountPath: /run/csi diff --git a/chart/.snapshots/example-prod.yaml b/chart/.snapshots/example-prod.yaml index cce60ea5627b3579ac0d3e3d9f955bf4dbbba43f..f06774487c63d08cd1fcba7e5eb8493bd76e7a86 100644 --- a/chart/.snapshots/example-prod.yaml +++ b/chart/.snapshots/example-prod.yaml @@ -413,6 +413,7 @@ spec: args: - --feature-gates=Topology=true - --default-fstype=ext4 + - --extra-create-metadata - --leader-election - --leader-election-namespace=kube-system volumeMounts: @@ -442,6 +443,8 @@ spec: value: unix:///run/csi/socket - name: HCLOUD_VOLUME_DEFAULT_LOCATION value: "nbg1" + - name: HCLOUD_VOLUME_EXTRA_LABELS + value: "cluster=mycluster,env=production,team=devops" - name: METRICS_ENDPOINT value: "0.0.0.0:9189" - name: ENABLE_METRICS diff --git a/chart/.snapshots/full.yaml b/chart/.snapshots/full.yaml index 6bd9fc64f930fc7a0883f261bc64e95a1d41d6c0..ccdebd136952cdce2c3f9233cb60f4c7a991702c 100644 --- a/chart/.snapshots/full.yaml +++ b/chart/.snapshots/full.yaml @@ -535,6 +535,7 @@ spec: args: - --feature-gates=Topology=true - --default-fstype=ext4 + - --extra-create-metadata - --leader-election - --leader-election-namespace=namespace-override volumeMounts: diff --git a/chart/example-prod.values.yaml b/chart/example-prod.values.yaml index 2735a93112ae69c16238fd0b0f5b781b2e684a2f..3fef7da4cd318fe04455a103a8cdf4311946e564 100644 --- a/chart/example-prod.values.yaml +++ b/chart/example-prod.values.yaml @@ -1,6 +1,10 @@ controller: replicaCount: 2 hcloudVolumeDefaultLocation: nbg1 + volumeExtraLabels: + env: production + team: devops + cluster: mycluster priorityClassName: "system-cluster-critical" resources: csiAttacher: diff --git a/chart/templates/controller/deployment.yaml b/chart/templates/controller/deployment.yaml index a7b52d087cbb8a13caf05d107a9b99a08b6cdc08..74db6e452d22aa7caaade639c3e37eccb19e5726 100644 --- a/chart/templates/controller/deployment.yaml +++ b/chart/templates/controller/deployment.yaml @@ -118,6 +118,7 @@ spec: args: - --feature-gates=Topology=true - --default-fstype=ext4 + - --extra-create-metadata {{- if $enableLeaderElection }} - --leader-election - --leader-election-namespace={{ include "common.names.namespace" . }} @@ -147,6 +148,14 @@ spec: - name: HCLOUD_VOLUME_DEFAULT_LOCATION value: {{ .Values.controller.hcloudVolumeDefaultLocation | quote }} {{- end }} + {{- if .Values.controller.volumeExtraLabels }} + {{- $pairs := list }} + {{- range $key, $value := .Values.controller.volumeExtraLabels }} + {{- $pairs = append $pairs (printf "%s=%s" $key $value) }} + {{- end }} + - name: HCLOUD_VOLUME_EXTRA_LABELS + value: {{ join "," $pairs | quote }} + {{- end }} {{- if .Values.metrics.enabled }} - name: METRICS_ENDPOINT value: "0.0.0.0:{{ .Values.controller.containerPorts.metrics }}" diff --git a/chart/values.schema.json b/chart/values.schema.json index 2d01f9c002d75786c632524e704ec2e25d6d5efd..2d3ee678b349448b2e47307511c3c5a415bf412c 100644 --- a/chart/values.schema.json +++ b/chart/values.schema.json @@ -72,6 +72,12 @@ "hcloudVolumeDefaultLocation": { "type": "string" }, + "volumeExtraLabels": { + "type": "object", + "additionalProperties": { + "type": "string" + } + }, "image": { "properties": { "csiAttacher": { diff --git a/chart/values.yaml b/chart/values.yaml index 1ad3638b15cea11cfcd3acad1915cc30398b1683..530bc0d6ede9e36c069c5fd3860792a6eb708509 100644 --- a/chart/values.yaml +++ b/chart/values.yaml @@ -172,6 +172,10 @@ controller: ## hcloudVolumeDefaultLocation: "" + ## @param controller.volumeExtraLabels Specifies default labels to apply to all newly created volumes. The value must be a map in the format key: value. + ## + volumeExtraLabels: {} + ## @param controller.containerPorts.metrics controller metrics container port ## @param controller.containerPorts.healthz controller healthz container port ## diff --git a/cmd/aio/main.go b/cmd/aio/main.go index 84abb09d852f4e3b96bc36419b0d610f929ba9ce..2ac9df2849ad25cb5f2667830f78227342a46c07 100644 --- a/cmd/aio/main.go +++ b/cmd/aio/main.go @@ -12,6 +12,7 @@ import ( "github.com/hetznercloud/csi-driver/internal/api" "github.com/hetznercloud/csi-driver/internal/app" "github.com/hetznercloud/csi-driver/internal/driver" + "github.com/hetznercloud/csi-driver/internal/utils" "github.com/hetznercloud/csi-driver/internal/volumes" "github.com/hetznercloud/hcloud-go/v2/hcloud/metadata" ) @@ -83,11 +84,18 @@ func main() { hcloudClient, ), ) + + extraVolumeLabels, err := utils.ConvertLabelsToMap(os.Getenv("HCLOUD_VOLUME_EXTRA_LABELS")) + if err != nil { + logger.Error("could not parse extra labels for volumes", "error", err) + os.Exit(1) + } controllerService := driver.NewControllerService( logger.With("component", "driver-controller-service"), volumeService, serverLocation, enableProvidedByTopology, + extraVolumeLabels, ) // common diff --git a/cmd/controller/main.go b/cmd/controller/main.go index 9816d441fcaab608c1450350d530709bb56c9071..dc9f241c103b24e7a8a32c87e27686eacbe79f03 100644 --- a/cmd/controller/main.go +++ b/cmd/controller/main.go @@ -9,6 +9,7 @@ import ( "github.com/hetznercloud/csi-driver/internal/api" "github.com/hetznercloud/csi-driver/internal/app" "github.com/hetznercloud/csi-driver/internal/driver" + "github.com/hetznercloud/csi-driver/internal/utils" "github.com/hetznercloud/csi-driver/internal/volumes" "github.com/hetznercloud/hcloud-go/v2/hcloud/metadata" ) @@ -70,11 +71,17 @@ func main() { hcloudClient, ), ) + extraVolumeLabels, err := utils.ConvertLabelsToMap(os.Getenv("HCLOUD_VOLUME_EXTRA_LABELS")) + if err != nil { + logger.Error("could not parse extra labels for volumes", "error", err) + os.Exit(1) + } controllerService := driver.NewControllerService( logger.With("component", "driver-controller-service"), volumeService, location, enableProvidedByTopology, + extraVolumeLabels, ) identityService := driver.NewIdentityService( diff --git a/deploy/kubernetes/hcloud-csi.yml b/deploy/kubernetes/hcloud-csi.yml index 4e437180684534782121626a8657aa1676f41697..b25118f4606d2fa26764f8d5285119924d2c694b 100644 --- a/deploy/kubernetes/hcloud-csi.yml +++ b/deploy/kubernetes/hcloud-csi.yml @@ -343,6 +343,7 @@ spec: args: - --feature-gates=Topology=true - --default-fstype=ext4 + - --extra-create-metadata volumeMounts: - name: socket-dir mountPath: /run/csi diff --git a/docs/kubernetes/README.md b/docs/kubernetes/README.md index 9ef0b8ef337a50e6d238090dccb5b214810d0af9..04240ad674faf48f515eabc041e2fd1d5db2817f 100644 --- a/docs/kubernetes/README.md +++ b/docs/kubernetes/README.md @@ -155,6 +155,38 @@ During the initialization of the CSI controller, the default location for all vo 3. If neither of the above is set, the `KUBE_NODE_NAME` environment variable defaults to the name of the node where the CSI controller is scheduled. This node name is then used to query the Hetzner API for a matching server and its location. 4. As a final fallback, the [Hetzner metadata service](https://docs.hetzner.cloud/#server-metadata) is queried to obtain the server ID, which is then used to fetch the location from the Hetzner API. +### Volume Labels + +It is possible to set labels for all newly created volumes. By default, all volumes are labeled as follows: + +- `csi.storage.k8s.io/pvc/name` +- `csi.storage.k8s.io/pvc/namespace` +- `csi.storage.k8s.io/pv/name` +- `csi.hetzner.cloud/created-by=csi-driver` + +To add extra labels to all created volumes set `HCLOUD_VOLUME_EXTRA_LABELS` in the format `key=value,...`. +This is also configurable from the Helm chart by the value `controller.volumeExtraLabels`, e.g: + +```yaml +controller: + volumeExtraLabels: + cluster: myCluster + env: prod +``` + +It is also possible to set only labels on specific volumes created by a storage class. To do this, you need to set `labels` in the format `key=value,...` as `extraParameters` inside the storage class. + +There is an example to set the `labels` for the storage class over the Helm chart values: + +```yaml +storageClasses: + - name: hcloud-volumes + defaultStorageClass: true + reclaimPolicy: Delete + extraParameters: + labels: cluster=myCluster,env=prod +``` + ## Upgrading To upgrade the csi-driver version, you just need to apply the new manifests to your cluster. diff --git a/internal/api/volume.go b/internal/api/volume.go index 128cecf473b99f72ce58ad437490937b8a86d780..903d0bb2c6b44cbb30107fb55eff8816141cbede 100644 --- a/internal/api/volume.go +++ b/internal/api/volume.go @@ -51,7 +51,9 @@ func (s *VolumeService) Create(ctx context.Context, opts volumes.CreateOpts) (*c Name: opts.Name, Size: opts.MinSize, Location: &hcloud.Location{Name: opts.Location}, + Labels: opts.Labels, }) + if err != nil { s.logger.Info( "failed to create volume", diff --git a/internal/driver/controller.go b/internal/driver/controller.go index 2591a9200f3bb53842a63384b0444f8e4b1a6ba2..7393389c038f80738ae670be84c423d438363e15 100644 --- a/internal/driver/controller.go +++ b/internal/driver/controller.go @@ -5,16 +5,31 @@ import ( "errors" "fmt" "log/slog" + "maps" "strconv" + "strings" proto "github.com/container-storage-interface/spec/lib/go/csi" "google.golang.org/grpc/codes" "google.golang.org/grpc/status" "github.com/hetznercloud/csi-driver/internal/csi" + "github.com/hetznercloud/csi-driver/internal/utils" "github.com/hetznercloud/csi-driver/internal/volumes" ) +const ( + parameterKeyPVCName = "csi.storage.k8s.io/pvc/name" + parameterKeyPVCNamespace = "csi.storage.k8s.io/pvc/namespace" + parameterKeyPVName = "csi.storage.k8s.io/pv/name" + parameterKeyLabels = "labels" + + tagKeyCreatedForClaimName = "csi.storage.k8s.io/pvc/name" + tagKeyCreatedForClaimNamespace = "csi.storage.k8s.io/pvc/namespace" + tagKeyCreatedForVolumeName = "csi.storage.k8s.io/pv/name" + tagKeyCreatedBy = "csi.hetzner.cloud/created-by" +) + type ControllerService struct { proto.UnimplementedControllerServer @@ -22,6 +37,7 @@ type ControllerService struct { volumeService volumes.Service location string enableProvidedByTopology bool + extraVolumeLabels map[string]string } func NewControllerService( @@ -29,12 +45,14 @@ func NewControllerService( volumeService volumes.Service, location string, enableProvidedByTopology bool, + extraVolumeLabels map[string]string, ) *ControllerService { return &ControllerService{ logger: logger, volumeService: volumeService, location: location, enableProvidedByTopology: enableProvidedByTopology, + extraVolumeLabels: extraVolumeLabels, } } @@ -66,12 +84,38 @@ func (s *ControllerService) CreateVolume(ctx context.Context, req *proto.CreateV location = *loc } + var volumeLabels = map[string]string{ + tagKeyCreatedBy: "csi-driver", + } + + maps.Copy(volumeLabels, s.extraVolumeLabels) + + for key, value := range req.GetParameters() { + switch strings.ToLower(key) { + case parameterKeyPVCName: + volumeLabels[tagKeyCreatedForClaimName] = value + case parameterKeyPVCNamespace: + volumeLabels[tagKeyCreatedForClaimNamespace] = value + case parameterKeyPVName: + volumeLabels[tagKeyCreatedForVolumeName] = value + case parameterKeyLabels: + customLabels, err := utils.ConvertLabelsToMap(value) + if err != nil { + return nil, status.Errorf(codes.InvalidArgument, "Invalid format of parameter labels: %s", err) + } + maps.Copy(volumeLabels, customLabels) + default: + s.logger.Warn(fmt.Sprintf("invalid parameter key %s for CreateVolume", key)) + } + } + // Create the volume. The service handles idempotency as required by the CSI spec. volume, err := s.volumeService.Create(ctx, volumes.CreateOpts{ Name: req.Name, MinSize: minSize, MaxSize: maxSize, Location: location, + Labels: volumeLabels, }) if err != nil { s.logger.Error( diff --git a/internal/driver/controller_test.go b/internal/driver/controller_test.go index d1f7c0b56b29f9d30311a6e794349d899a0be1ab..35b17f28fd5e03f1fd79faf2d60fcca484e04c92 100644 --- a/internal/driver/controller_test.go +++ b/internal/driver/controller_test.go @@ -34,6 +34,7 @@ func newControllerServiceTestEnv() *controllerServiceTestEnv { volumeService, "testloc", false, + map[string]string{"clusterName": "myCluster"}, ), volumeService: volumeService, } @@ -57,6 +58,18 @@ func TestControllerServiceCreateVolume(t *testing.T) { if opts.Location != "testloc" { t.Errorf("unexpected location passed to volume service: %s", opts.Location) } + if v, ok := opts.Labels["clusterName"]; !ok || v != "myCluster" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } + if v, ok := opts.Labels[tagKeyCreatedForClaimName]; !ok || v != "pvc-name" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } + if v, ok := opts.Labels[tagKeyCreatedForClaimNamespace]; !ok || v != "default" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } + if v, ok := opts.Labels[tagKeyCreatedForVolumeName]; !ok || v != "pv-name" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } return &csi.Volume{ ID: 1, Name: opts.Name, @@ -71,6 +84,85 @@ func TestControllerServiceCreateVolume(t *testing.T) { RequiredBytes: MinVolumeSize*GB + 100, LimitBytes: 2 * MinVolumeSize * GB, }, + Parameters: map[string]string{ + parameterKeyPVCName: "pvc-name", + parameterKeyPVCNamespace: "default", + parameterKeyPVName: "pv-name", + }, + VolumeCapabilities: []*proto.VolumeCapability{ + { + AccessType: &proto.VolumeCapability_Mount{ + Mount: &proto.VolumeCapability_MountVolume{}, + }, + AccessMode: &proto.VolumeCapability_AccessMode{ + Mode: proto.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + } + resp, err := env.service.CreateVolume(env.ctx, req) + if err != nil { + t.Fatal(err) + } + if resp.Volume.VolumeId != "1" { + t.Errorf("unexpected value for VolumeId: %s", resp.Volume.VolumeId) + } + if resp.Volume.CapacityBytes != (MinVolumeSize+1)*1024*1024*1024 { + t.Errorf("unexpected value for CapacityBytes: %d", resp.Volume.CapacityBytes) + } + if len(resp.Volume.AccessibleTopology) == 1 { + top := resp.Volume.AccessibleTopology[0] + if loc := top.Segments[TopologySegmentLocation]; loc != "testloc" { + t.Errorf("unexpected location segment in topology: %s", loc) + } + if env.service.enableProvidedByTopology { + if provider := top.Segments[ProvidedByLabel]; provider != "cloud" { + t.Errorf("unexpected provider segment in topology: %s", provider) + } + } + } else { + t.Errorf("unexpected number of topologies: %d", len(resp.Volume.AccessibleTopology)) + } +} +func TestControllerServiceCreateVolumeWithParameterLabels(t *testing.T) { + env := newControllerServiceTestEnv() + + env.service.enableProvidedByTopology = true + + env.volumeService.CreateFunc = func(ctx context.Context, opts volumes.CreateOpts) (*csi.Volume, error) { + if opts.Name != "testvol" { + t.Errorf("unexpected name passed to volume service: %s", opts.Name) + } + if opts.MinSize != MinVolumeSize+1 { + t.Errorf("unexpected min size passed to volume service: %d", opts.MinSize) + } + if opts.MaxSize != 2*MinVolumeSize { + t.Errorf("unexpected max size passed to volume service: %d", opts.MaxSize) + } + if opts.Location != "testloc" { + t.Errorf("unexpected location passed to volume service: %s", opts.Location) + } + if v, ok := opts.Labels["test"]; !ok || v != "test" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } + if v, ok := opts.Labels["clusterName"]; !ok || v != "myCluster" { + t.Errorf("unexpected labels passed to volume service: %s", opts.Labels) + } + return &csi.Volume{ + ID: 1, + Name: opts.Name, + Size: opts.MinSize, + Location: opts.Location, + }, nil + } + + req := &proto.CreateVolumeRequest{ + Name: "testvol", + CapacityRange: &proto.CapacityRange{ + RequiredBytes: MinVolumeSize*GB + 100, + LimitBytes: 2 * MinVolumeSize * GB, + }, + Parameters: map[string]string{"labels": "test=test"}, VolumeCapabilities: []*proto.VolumeCapability{ { AccessType: &proto.VolumeCapability_Mount{ @@ -190,6 +282,28 @@ func TestControllerServiceCreateVolumeInputErrors(t *testing.T) { }, Code: codes.InvalidArgument, }, + { + Name: "invalid label", + Req: &proto.CreateVolumeRequest{ + Name: "test", + CapacityRange: &proto.CapacityRange{ + RequiredBytes: 5*GB + 100, + LimitBytes: 10 * GB, + }, + Parameters: map[string]string{"labels": "=test"}, + VolumeCapabilities: []*proto.VolumeCapability{ + { + AccessType: &proto.VolumeCapability_Mount{ + Mount: &proto.VolumeCapability_MountVolume{}, + }, + AccessMode: &proto.VolumeCapability_AccessMode{ + Mode: proto.VolumeCapability_AccessMode_SINGLE_NODE_WRITER, + }, + }, + }, + }, + Code: codes.InvalidArgument, + }, { Name: "empty capabilities", Req: &proto.CreateVolumeRequest{ diff --git a/internal/driver/sanity_test.go b/internal/driver/sanity_test.go index 4b34609cc71b81a71e61ab66bd0d311e357abe3c..51d4dcfa3eb42ffe53421437dd100ef6dbba4d41 100644 --- a/internal/driver/sanity_test.go +++ b/internal/driver/sanity_test.go @@ -47,6 +47,7 @@ func TestSanity(t *testing.T) { volumeService, "testloc", false, + map[string]string{"clusterName": "myCluster"}, ) identityService := NewIdentityService( diff --git a/internal/utils/labels.go b/internal/utils/labels.go new file mode 100644 index 0000000000000000000000000000000000000000..6d1045a6e1e9cf90e1e6e821ec08e18a5a2f59de --- /dev/null +++ b/internal/utils/labels.go @@ -0,0 +1,27 @@ +package utils + +import ( + "errors" + "strings" +) + +func ConvertLabelsToMap(labelsString string) (map[string]string, error) { + result := map[string]string{} + splitFn := func(c rune) bool { + return c == ',' + } + vals := strings.FieldsFunc(labelsString, splitFn) + for _, val := range vals { + pair := strings.SplitN(val, "=", 2) + key := strings.TrimSpace(pair[0]) + if key == "" { + return nil, errors.New("empty key") + } + value := "" + if len(pair) > 1 { + value = strings.TrimSpace(pair[1]) + } + result[strings.TrimSpace(pair[0])] = value + } + return result, nil +} diff --git a/internal/utils/labels_test.go b/internal/utils/labels_test.go new file mode 100644 index 0000000000000000000000000000000000000000..56363eb1a7b97dc9723380aca03935dacd149422 --- /dev/null +++ b/internal/utils/labels_test.go @@ -0,0 +1,35 @@ +package utils + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestConvertLabelsToMap(t *testing.T) { + tests := []struct { + name string + env string + expected map[string]string + errMessage string + }{ + {"valid", "test1=test1", map[string]string{"test1": "test1"}, ""}, + {"mutiple items", "test1=test1,test2=test2", map[string]string{"test1": "test1", "test2": "test2"}, ""}, + {"empty", "", map[string]string{}, ""}, + {"multiple colons", "test1=test1=test1", map[string]string{"test1": "test1=test1"}, ""}, + {"space", "test1= test1", map[string]string{"test1": "test1"}, ""}, + {"no value", "test1=", map[string]string{"test1": ""}, ""}, + {"space key", "=test1", nil, "empty key"}, + {"no equal sign", "test1", map[string]string{"test1": ""}, ""}, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + result, err := ConvertLabelsToMap(test.env) + assert.Equal(t, test.expected, result) + if err != nil { + assert.Equal(t, err.Error(), test.errMessage) + } + }) + } +} diff --git a/internal/volumes/idempotency_test.go b/internal/volumes/idempotency_test.go index 67fa333e448beff137e7b947ecc76621f4ad7f82..448672576ba96250fba2ba43b6bc1a28f3697468 100644 --- a/internal/volumes/idempotency_test.go +++ b/internal/volumes/idempotency_test.go @@ -3,6 +3,7 @@ package volumes_test import ( "context" "io" + "reflect" "testing" "github.com/hetznercloud/csi-driver/internal/csi" @@ -29,7 +30,7 @@ func TestIdempotentServiceCreateNew(t *testing.T) { volumeService := &mock.VolumeService{ CreateFunc: func(ctx context.Context, opts volumes.CreateOpts) (*csi.Volume, error) { - if opts != creatingOpts { + if !reflect.DeepEqual(opts, creatingOpts) { t.Errorf("unexpected options: %v", opts) } return creatingVolume, nil diff --git a/internal/volumes/service.go b/internal/volumes/service.go index f67baa02fa8399bc353437bdcf80e1a2d6df0af0..db3029df1909995cb207761df4077f976fc80b4e 100644 --- a/internal/volumes/service.go +++ b/internal/volumes/service.go @@ -35,4 +35,5 @@ type CreateOpts struct { MinSize int MaxSize int Location string + Labels map[string]string }