From a3953111d4c9a47b0005a5454508aea2a10ed828 Mon Sep 17 00:00:00 2001 From: Luiz Aoqui Date: Mon, 10 Jul 2023 20:05:31 -0400 Subject: [PATCH] volumes: rename existing volume resources Rename `nomad_volume` to `nomad_csi_volume_registration` and `nomad_external_volume` to `nomad_csi_volume` to better reflect their intended use. The deprecated top-level fields `access_mode` and `attachment_mode` were not moved to the new resource. The previous resources are now marked as deprecated. --- nomad/helper/types.go | 12 + nomad/provider.go | 28 +- nomad/resource_csi_volume.go | 410 +++++++++++++ nomad/resource_csi_volume_registration.go | 556 ++++++++++++++++++ nomad/resource_csi_volume_test.go | 229 ++++++++ nomad/resource_external_volume.go | 2 + nomad/resource_volume.go | 2 + website/docs/r/csi_volume.html.markdown | 122 ++++ .../r/csi_volume_registration.html.markdown | 121 ++++ website/docs/r/external_volume.html.markdown | 3 + website/docs/r/volume.html.markdown | 3 + website/nomad.erb | 6 + 12 files changed, 1481 insertions(+), 13 deletions(-) create mode 100644 nomad/helper/types.go create mode 100644 nomad/resource_csi_volume.go create mode 100644 nomad/resource_csi_volume_registration.go create mode 100644 nomad/resource_csi_volume_test.go create mode 100644 website/docs/r/csi_volume.html.markdown create mode 100644 website/docs/r/csi_volume_registration.html.markdown diff --git a/nomad/helper/types.go b/nomad/helper/types.go new file mode 100644 index 00000000..d93a9daf --- /dev/null +++ b/nomad/helper/types.go @@ -0,0 +1,12 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package helper + +func ToMapStringString(m any) map[string]string { + mss := map[string]string{} + for k, v := range m.(map[string]any) { + mss[k] = v.(string) + } + return mss +} diff --git a/nomad/provider.go b/nomad/provider.go index 882e559f..f5a7237f 100644 --- a/nomad/provider.go +++ b/nomad/provider.go @@ -163,19 +163,21 @@ func Provider() *schema.Provider { }, ResourcesMap: map[string]*schema.Resource{ - "nomad_acl_auth_method": resourceACLAuthMethod(), - "nomad_acl_binding_rule": resourceACLBindingRule(), - "nomad_acl_policy": resourceACLPolicy(), - "nomad_acl_role": resourceACLRole(), - "nomad_acl_token": resourceACLToken(), - "nomad_external_volume": resourceExternalVolume(), - "nomad_job": resourceJob(), - "nomad_namespace": resourceNamespace(), - "nomad_quota_specification": resourceQuotaSpecification(), - "nomad_sentinel_policy": resourceSentinelPolicy(), - "nomad_volume": resourceVolume(), - "nomad_scheduler_config": resourceSchedulerConfig(), - "nomad_variable": resourceVariable(), + "nomad_acl_auth_method": resourceACLAuthMethod(), + "nomad_acl_binding_rule": resourceACLBindingRule(), + "nomad_acl_policy": resourceACLPolicy(), + "nomad_acl_role": resourceACLRole(), + "nomad_acl_token": resourceACLToken(), + "nomad_csi_volume": resourceCSIVolume(), + "nomad_csi_volume_registration": resourceCSIVolumeRegistration(), + "nomad_external_volume": resourceExternalVolume(), + "nomad_job": resourceJob(), + "nomad_namespace": resourceNamespace(), + "nomad_quota_specification": resourceQuotaSpecification(), + "nomad_sentinel_policy": resourceSentinelPolicy(), + "nomad_volume": resourceVolume(), + "nomad_scheduler_config": resourceSchedulerConfig(), + "nomad_variable": resourceVariable(), }, } } diff --git a/nomad/resource_csi_volume.go b/nomad/resource_csi_volume.go new file mode 100644 index 00000000..800b7a9d --- /dev/null +++ b/nomad/resource_csi_volume.go @@ -0,0 +1,410 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package nomad + +import ( + "bytes" + "errors" + "fmt" + "hash/crc32" + "log" + + "github.com/dustin/go-humanize" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/hashicorp/terraform-provider-nomad/nomad/helper" +) + +func resourceCSIVolume() *schema.Resource { + return &schema.Resource{ + Create: resourceCSIVolumeCreate, + Update: resourceCSIVolumeCreate, + Delete: resourceCSIVolumeDelete, + + // Once created, CSI volumes are automatically registered as a + // normal volume. + Read: resourceCSIVolumeRead, + + Schema: map[string]*schema.Schema{ + "namespace": { + ForceNew: true, + Description: "The namespace in which to create the volume.", + Optional: true, + Default: "default", + Type: schema.TypeString, + }, + + "volume_id": { + ForceNew: true, + Description: "The unique ID of the volume, how jobs will refer to the volume.", + Required: true, + Type: schema.TypeString, + }, + + "name": { + Description: "The display name of the volume.", + Required: true, + Type: schema.TypeString, + }, + + "plugin_id": { + ForceNew: true, + Description: "The ID of the CSI plugin that manages this volume.", + Required: true, + Type: schema.TypeString, + }, + + "snapshot_id": { + ForceNew: true, + Description: "The snapshot ID to restore when creating this volume. Storage provider must support snapshots. Conflicts with 'clone_id'.", + Optional: true, + Type: schema.TypeString, + ConflictsWith: []string{"clone_id"}, + }, + + "clone_id": { + ForceNew: true, + Description: "The volume ID to clone when creating this volume. Storage provider must support cloning. Conflicts with 'snapshot_id'.", + Optional: true, + Type: schema.TypeString, + ConflictsWith: []string{"snapshot_id"}, + }, + + "capacity_min": { + ForceNew: true, + Description: "Defines how small the volume can be. The storage provider may return a volume that is larger than this value.", + Optional: true, + Type: schema.TypeString, + }, + + "capacity_max": { + ForceNew: true, + Description: "Defines how large the volume can be. The storage provider may return a volume that is smaller than this value.", + Optional: true, + Type: schema.TypeString, + }, + + "capability": { + ForceNew: true, + Description: "Capabilities intended to be used in a job. At least one capability must be provided.", + Required: true, + Type: schema.TypeSet, + MinItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "access_mode": { + Description: "Defines whether a volume should be available concurrently.", + Type: schema.TypeString, + Required: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringInSlice([]string{ + "single-node-reader-only", + "single-node-writer", + "multi-node-reader-only", + "multi-node-single-writer", + "multi-node-multi-writer", + }, false), + }, + }, + "attachment_mode": { + Description: "The storage API that will be used by the volume.", + Required: true, + Type: schema.TypeString, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringInSlice([]string{ + "block-device", + "file-system", + }, false), + }, + }, + }, + }, + Set: func(v interface{}) int { + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s-", m["access_mode"].(string))) + buf.WriteString(fmt.Sprintf("%s-", m["attachment_mode"].(string))) + + i := int(crc32.ChecksumIEEE(buf.Bytes())) + if i >= 0 { + return i + } + if -i >= 0 { + return -i + } + // i == MinInt + return 0 + }, + }, + + "mount_options": { + Description: "Options for mounting 'block-device' volumes without a pre-formatted file system.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "fs_type": { + Description: "The file system type.", + Type: schema.TypeString, + Optional: true, + }, + "mount_flags": { + Description: "The flags passed to mount.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + }, + }, + }, + }, + + "secrets": { + Description: "An optional key-value map of strings used as credentials for publishing and unpublishing volumes.", + Optional: true, + Type: schema.TypeMap, + Sensitive: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "parameters": { + Description: "An optional key-value map of strings passed directly to the CSI plugin to configure the volume.", + Optional: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "topology_request": { + Description: "Specify locations (region, zone, rack, etc.) where the provisioned volume is accessible from.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "required": { + Description: "Required topologies indicate that the volume must be created in a location accessible from all the listed topologies.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "topology": { + Description: "Defines the location for the volume.", + Required: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "segments": { + Description: "Define the attributes for the topology request.", + Required: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + }, + }, + }, + "preferred": { + Description: "Preferred topologies indicate that the volume should be created in a location accessible from some of the listed topologies.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "topology": { + Description: "Defines the location for the volume.", + Required: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "segments": { + Description: "Define the attributes for the topology request.", + Required: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + + "controller_required": { + Computed: true, + Type: schema.TypeBool, + }, + + "controllers_expected": { + Computed: true, + Type: schema.TypeInt, + }, + + "controllers_healthy": { + Computed: true, + Type: schema.TypeInt, + }, + + "plugin_provider": { + Computed: true, + Type: schema.TypeString, + }, + + "plugin_provider_version": { + Computed: true, + Type: schema.TypeString, + }, + + "nodes_healthy": { + Computed: true, + Type: schema.TypeInt, + }, + + "nodes_expected": { + Computed: true, + Type: schema.TypeInt, + }, + + "schedulable": { + Computed: true, + Type: schema.TypeBool, + }, + "topologies": { + Computed: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "segments": { + Computed: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + }, + } +} + +func resourceCSIVolumeCreate(d *schema.ResourceData, meta interface{}) error { + providerConfig := meta.(ProviderConfig) + client := providerConfig.client + + // Parse capacities from human-friendly string to number. + capacityMin, err := humanize.ParseBytes(d.Get("capacity_min").(string)) + if err != nil { + return fmt.Errorf("invalid value 'capacity_min': %v", err) + } + + capacityMax, err := humanize.ParseBytes(d.Get("capacity_max").(string)) + if err != nil { + return fmt.Errorf("invalid value 'capacity_max': %v", err) + } + + // Parse capabilities set. + capabilities, err := parseCSIVolumeCapabilities(d.Get("capability")) + if err != nil { + return fmt.Errorf("failed to unpack capabilities: %v", err) + } + + topologyRequest, err := parseCSIVolumeTopologyRequest(d.Get("topology_request")) + if err != nil { + return fmt.Errorf("failed to unpack topology request: %v", err) + } + + volume := &api.CSIVolume{ + ID: d.Get("volume_id").(string), + PluginID: d.Get("plugin_id").(string), + Name: d.Get("name").(string), + SnapshotID: d.Get("snapshot_id").(string), + CloneID: d.Get("clone_id").(string), + RequestedCapacityMin: int64(capacityMin), + RequestedCapacityMax: int64(capacityMax), + RequestedCapabilities: capabilities, + RequestedTopologies: topologyRequest, + Secrets: helper.ToMapStringString(d.Get("secrets")), + Parameters: helper.ToMapStringString(d.Get("parameters")), + } + + // Unpack the mount_options if we have any and configure the volume struct. + mountOpts, ok := d.GetOk("mount_options") + if ok { + mountOptsList, ok := mountOpts.([]interface{}) + if !ok || len(mountOptsList) != 1 { + return errors.New("failed to unpack mount_options configuration block") + } + + mountOptsMap, ok := mountOptsList[0].(map[string]interface{}) + if !ok { + return errors.New("failed to unpack mount_options configuration block") + } + volume.MountOptions = &api.CSIMountOptions{} + + if val, ok := mountOptsMap["fs_type"].(string); ok { + volume.MountOptions.FSType = val + } + if mountFlagsList, ok := mountOptsMap["mount_flags"].([]interface{}); ok { + volume.MountOptions.MountFlags = []string{} + for _, rawflag := range mountFlagsList { + volume.MountOptions.MountFlags = append(volume.MountOptions.MountFlags, rawflag.(string)) + } + } + } + + // Create the volume. + log.Printf("[DEBUG] creating CSI volume %q in namespace %q", volume.ID, volume.Namespace) + opts := &api.WriteOptions{ + Namespace: d.Get("namespace").(string), + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + _, _, err = client.CSIVolumes().Create(volume, opts) + if err != nil { + return fmt.Errorf("error creating CSI volume: %s", err) + } + + log.Printf("[DEBUG] CSI volume %q created in namespace %q", volume.ID, volume.Namespace) + d.SetId(volume.ID) + + return resourceCSIVolumeRead(d, meta) // populate other computed attributes +} + +func resourceCSIVolumeDelete(d *schema.ResourceData, meta interface{}) error { + providerConfig := meta.(ProviderConfig) + client := providerConfig.client + + id := d.Id() + log.Printf("[DEBUG] deleting CSI volume: %q", id) + opts := &api.WriteOptions{ + Namespace: d.Get("namespace").(string), + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + err := client.CSIVolumes().Delete(id, opts) + if err != nil { + return fmt.Errorf("error deleting CSI volume: %s", err) + } + + return nil +} diff --git a/nomad/resource_csi_volume_registration.go b/nomad/resource_csi_volume_registration.go new file mode 100644 index 00000000..f47de162 --- /dev/null +++ b/nomad/resource_csi_volume_registration.go @@ -0,0 +1,556 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package nomad + +import ( + "bytes" + "errors" + "fmt" + "hash/crc32" + "log" + "strings" + + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/validation" + "github.com/hashicorp/terraform-provider-nomad/nomad/helper" +) + +func resourceCSIVolumeRegistration() *schema.Resource { + return &schema.Resource{ + Create: resourceCSIVolumeRegistrationCreate, + Update: resourceCSIVolumeRegistrationCreate, + Delete: resourceCSIVolumeRegistrationDelete, + Read: resourceCSIVolumeRead, + + Schema: map[string]*schema.Schema{ + // the following cannot be updated without destroying: + // - Namespace/ID + // - PluginID + // - ExternalID + + "namespace": { + ForceNew: true, + Description: "The namespace in which to create the volume.", + Optional: true, + Default: "default", + Type: schema.TypeString, + }, + + "volume_id": { + ForceNew: true, + Description: "The unique ID of the volume, how jobs will refer to the volume.", + Required: true, + Type: schema.TypeString, + }, + + "name": { + Description: "The display name of the volume.", + Required: true, + Type: schema.TypeString, + }, + + "plugin_id": { + ForceNew: true, + Description: "The ID of the CSI plugin that manages this volume.", + Required: true, + Type: schema.TypeString, + }, + + "external_id": { + ForceNew: true, + Description: "The ID of the physical volume from the storage provider.", + Required: true, + Type: schema.TypeString, + }, + + "capability": { + Description: "Capabilities intended to be used in a job. At least one capability must be provided.", + Optional: true, + Type: schema.TypeSet, + MinItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "access_mode": { + Description: "Defines whether a volume should be available concurrently.", + Type: schema.TypeString, + Required: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringInSlice([]string{ + "single-node-reader-only", + "single-node-writer", + "multi-node-reader-only", + "multi-node-single-writer", + "multi-node-multi-writer", + }, false), + }, + }, + "attachment_mode": { + Description: "The storage API that will be used by the volume.", + Required: true, + Type: schema.TypeString, + Elem: &schema.Schema{ + Type: schema.TypeString, + ValidateFunc: validation.StringInSlice([]string{ + "block-device", + "file-system", + }, false), + }, + }, + }, + }, + Set: func(v interface{}) int { + var buf bytes.Buffer + m := v.(map[string]interface{}) + buf.WriteString(fmt.Sprintf("%s-", m["access_mode"].(string))) + buf.WriteString(fmt.Sprintf("%s-", m["attachment_mode"].(string))) + + i := int(crc32.ChecksumIEEE(buf.Bytes())) + if i >= 0 { + return i + } + if -i >= 0 { + return -i + } + // i == MinInt + return 0 + }, + }, + + "mount_options": { + Description: "Options for mounting 'block-device' volumes without a pre-formatted file system.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "fs_type": { + Description: "The file system type.", + Type: schema.TypeString, + Optional: true, + }, + "mount_flags": { + Description: "The flags passed to mount.", + Type: schema.TypeList, + Elem: &schema.Schema{Type: schema.TypeString}, + Optional: true, + }, + }, + }, + }, + + "secrets": { + Description: "An optional key-value map of strings used as credentials for publishing and unpublishing volumes.", + Optional: true, + Type: schema.TypeMap, + Sensitive: true, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "parameters": { + Description: "An optional key-value map of strings passed directly to the CSI plugin to configure the volume.", + Optional: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "topology_request": { + Description: "Specify locations (region, zone, rack, etc.) where the provisioned volume is accessible from.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "required": { + Description: "Required topologies indicate that the volume must be created in a location accessible from all the listed topologies.", + Optional: true, + Type: schema.TypeList, + MaxItems: 1, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "topology": { + Description: "Defines the location for the volume.", + Required: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "segments": { + Description: "Define attributes for the topology request.", + Required: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + }, + }, + }, + // Volume registration does not support preferred topologies. + // https://developer.hashicorp.com/nomad/docs/other-specifications/volume/topology_request#preferred + }, + }, + }, + + "context": { + Description: "An optional key-value map of strings passed directly to the CSI plugin to validate the volume.", + Optional: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + + "deregister_on_destroy": { + Description: "If true, the volume will be deregistered on destroy.", + Optional: true, + Default: true, + Type: schema.TypeBool, + }, + + "controller_required": { + Computed: true, + Type: schema.TypeBool, + }, + + "controllers_expected": { + Computed: true, + Type: schema.TypeInt, + }, + + "controllers_healthy": { + Computed: true, + Type: schema.TypeInt, + }, + + "plugin_provider": { + Computed: true, + Type: schema.TypeString, + }, + + "plugin_provider_version": { + Computed: true, + Type: schema.TypeString, + }, + + "nodes_healthy": { + Computed: true, + Type: schema.TypeInt, + }, + + "nodes_expected": { + Computed: true, + Type: schema.TypeInt, + }, + + "schedulable": { + Computed: true, + Type: schema.TypeBool, + }, + "topologies": { + Computed: true, + Type: schema.TypeList, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "segments": { + Computed: true, + Type: schema.TypeMap, + Elem: &schema.Schema{ + Type: schema.TypeString, + }, + }, + }, + }, + }, + }, + } +} + +func resourceCSIVolumeRegistrationCreate(d *schema.ResourceData, meta interface{}) error { + providerConfig := meta.(ProviderConfig) + client := providerConfig.client + + capabilities, err := parseCSIVolumeCapabilities(d.Get("capability")) + if err != nil { + return fmt.Errorf("failed to unpack capabilities: %v", err) + } + + topologyRequest, err := parseCSIVolumeTopologyRequest(d.Get("topology_request")) + if err != nil { + return fmt.Errorf("failed to unpack topology request: %v", err) + } + + volume := &api.CSIVolume{ + ID: d.Get("volume_id").(string), + Name: d.Get("name").(string), + ExternalID: d.Get("external_id").(string), + RequestedCapabilities: capabilities, + RequestedTopologies: topologyRequest, + Secrets: helper.ToMapStringString(d.Get("secrets")), + Parameters: helper.ToMapStringString(d.Get("parameters")), + Context: helper.ToMapStringString(d.Get("context")), + PluginID: d.Get("plugin_id").(string), + + // COMPAT(1.5.0) + // Maintain backwards compatibility. + AccessMode: capabilities[0].AccessMode, + AttachmentMode: capabilities[0].AttachmentMode, + } + + // Unpack the mount_options if we have any and configure the volume struct. + mountOpts, ok := d.GetOk("mount_options") + if ok { + mountOptsList, ok := mountOpts.([]interface{}) + if !ok || len(mountOptsList) != 1 { + return errors.New("failed to unpack mount_options configuration block") + } + + mountOptsMap, ok := mountOptsList[0].(map[string]interface{}) + if !ok { + return errors.New("failed to unpack mount_options configuration block") + } + volume.MountOptions = &api.CSIMountOptions{} + + if val, ok := mountOptsMap["fs_type"].(string); ok { + volume.MountOptions.FSType = val + } + rawMountFlags := mountOptsMap["mount_flags"].([]interface{}) + volume.MountOptions.MountFlags = make([]string, len(rawMountFlags)) + for index, value := range rawMountFlags { + if val, ok := value.(string); ok { + volume.MountOptions.MountFlags[index] = val + } + } + } + + // Register the volume + log.Printf("[DEBUG] registering CSI volume %q in namespace %q", volume.ID, volume.Namespace) + opts := &api.WriteOptions{ + Namespace: d.Get("namespace").(string), + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + _, err = client.CSIVolumes().Register(volume, opts) + if err != nil { + return fmt.Errorf("error registering CSI volume: %s", err) + } + + log.Printf("[DEBUG] CSI volume %q registered in namespace %q", volume.ID, volume.Namespace) + d.SetId(volume.ID) + + return resourceCSIVolumeRead(d, meta) // populate other computed attributes +} + +func resourceCSIVolumeRegistrationDelete(d *schema.ResourceData, meta interface{}) error { + providerConfig := meta.(ProviderConfig) + client := providerConfig.client + + // If deregistration is disabled, then do nothing + deregister_on_destroy := d.Get("deregister_on_destroy").(bool) + if !deregister_on_destroy { + log.Printf( + "[WARN] volume %q will not deregister since "+ + "'deregister_on_destroy' is %t", d.Id(), deregister_on_destroy) + return nil + } + + id := d.Id() + log.Printf("[DEBUG] deregistering CSI volume: %q", id) + opts := &api.WriteOptions{ + Namespace: d.Get("namespace").(string), + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + err := client.CSIVolumes().Deregister(id, true, opts) + if err != nil { + return fmt.Errorf("error deregistering CSI volume: %s", err) + } + + return nil +} + +func resourceCSIVolumeRead(d *schema.ResourceData, meta interface{}) error { + providerConfig := meta.(ProviderConfig) + client := providerConfig.client + + id := d.Id() + opts := &api.QueryOptions{ + Namespace: d.Get("namespace").(string), + } + if opts.Namespace == "" { + opts.Namespace = "default" + } + log.Printf("[DEBUG] reading information for CSI volume %q in namespace %q", id, opts.Namespace) + volume, _, err := client.CSIVolumes().Info(id, opts) + if err != nil { + // As of Nomad 0.4.1, the API client returns an error for 404 + // rather than a nil result, so we must check this way. + if strings.Contains(err.Error(), "404") { + log.Printf("[DEBUG] CSI volume %q does not exist, so removing", id) + d.SetId("") + return nil + } + + return fmt.Errorf("error checking for CSI volume: %s", err) + } + log.Printf("[DEBUG] found CSI volume %q in namespace %q", volume.Name, volume.Namespace) + + d.Set("name", volume.Name) + d.Set("controller_required", volume.ControllerRequired) + d.Set("controllers_expected", volume.ControllersExpected) + d.Set("controllers_healthy", volume.ControllersHealthy) + d.Set("controllers_healthy", volume.ControllersHealthy) + d.Set("plugin_provider", volume.Provider) + d.Set("plugin_provider_version", volume.ProviderVersion) + d.Set("nodes_healthy", volume.NodesHealthy) + d.Set("nodes_expected", volume.NodesExpected) + d.Set("schedulable", volume.Schedulable) + d.Set("topologies", flattenCSIVolumeTopologies(volume.Topologies)) + // The Nomad API redacts `mount_options` and `secrets`, so we don't update them + // with the response payload; they will remain as is. + + return nil +} + +func parseCSIVolumeCapabilities(i interface{}) ([]*api.CSIVolumeCapability, error) { + capabilities := []*api.CSIVolumeCapability{} + + capabilitySet, ok := i.(*schema.Set) + if !ok { + return nil, fmt.Errorf("invalid type %T, expected *schema.Set", i) + } + + for _, capabilitySetItem := range capabilitySet.List() { + capabilityMap, ok := capabilitySetItem.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T, expected map[string]interface{}", capabilitySetItem) + } + + c := &api.CSIVolumeCapability{} + for k, v := range capabilityMap { + switch k { + case "access_mode": + c.AccessMode = api.CSIVolumeAccessMode(v.(string)) + case "attachment_mode": + c.AttachmentMode = api.CSIVolumeAttachmentMode(v.(string)) + } + } + capabilities = append(capabilities, c) + } + + return capabilities, nil +} + +// parseVolumeTopologyRequest parses a Terraform state representation of volume +// topology request into its Nomad API representation. +func parseCSIVolumeTopologyRequest(i interface{}) (*api.CSITopologyRequest, error) { + req := &api.CSITopologyRequest{} + var err error + + topologyRequestList, ok := i.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T for topology_request, expected []interface{}", i) + } + + if len(topologyRequestList) == 0 { + return nil, nil + } + + topologyMap, ok := topologyRequestList[0].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T for topology_request item, expected map[string]interface{}", topologyRequestList[0]) + } + + if required, ok := topologyMap["required"]; ok { + req.Required, err = parseCSIVolumeTopologies("required", required) + if err != nil { + return nil, fmt.Errorf("failed to parse required CSI topology: %v", err) + } + } + + if preferred, ok := topologyMap["preferred"]; ok { + req.Preferred, err = parseCSIVolumeTopologies("preferred", preferred) + if err != nil { + return nil, fmt.Errorf("failed to parse preferred CSI topology: %v", err) + } + } + + return req, nil +} + +// parseVolumeTopologis parses a Terraform state representation of volume +// topology into its Nomad API representation. +func parseCSIVolumeTopologies(prefix string, i interface{}) ([]*api.CSITopology, error) { + var topologies []*api.CSITopology + + topologiesList, ok := i.([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T for %s.topology, expected []interface{}", i, prefix) + } + + if len(topologiesList) == 0 { + return topologies, nil + } + + topologiesListMap, ok := topologiesList[0].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T for %s.topology, expected map[string]interface{}", topologiesList[0], prefix) + } + + topologiesListMapList, ok := topologiesListMap["topology"].([]interface{}) + if !ok { + return nil, fmt.Errorf("invalid type %T for %s.topology, expected []interface{}", topologiesListMap["topology"], prefix) + } + + for j, topologyItem := range topologiesListMapList { + topologyItemMap, ok := topologyItem.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf( + "invalid type %T for %s.topology.%d, expected map[string]interface{}", + topologyItem, prefix, j) + } + segmentsMap, ok := topologyItemMap["segments"].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf( + "invalid type %T for %s.topology.%d.segments, expected map[string]interface{}", + topologyItemMap["segments"], prefix, j) + } + + segmentsMapString := make(map[string]string, len(segmentsMap)) + for k, v := range segmentsMap { + segmentsMapString[k] = v.(string) + } + topologies = append(topologies, &api.CSITopology{ + Segments: segmentsMapString, + }) + } + + return topologies, nil +} + +// flattenVolumeTopologies turns a list of Nomad API CSITopology structs into +// the flat representation used by Terraform. +func flattenCSIVolumeTopologies(topologies []*api.CSITopology) []interface{} { + topologiesList := []interface{}{} + + for _, topo := range topologies { + if topo == nil { + continue + } + topoItem := make(map[string]interface{}) + topoItem["segments"] = topo.Segments + topologiesList = append(topologiesList, topoItem) + } + + return topologiesList +} diff --git a/nomad/resource_csi_volume_test.go b/nomad/resource_csi_volume_test.go new file mode 100644 index 00000000..d4739d64 --- /dev/null +++ b/nomad/resource_csi_volume_test.go @@ -0,0 +1,229 @@ +// Copyright (c) HashiCorp, Inc. +// SPDX-License-Identifier: MPL-2.0 + +package nomad + +import ( + "errors" + "fmt" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/hashicorp/nomad/api" + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/resource" + "github.com/hashicorp/terraform-plugin-sdk/v2/terraform" +) + +// Testing this resource requires access to a Nomad cluster with CSI plugins +// running. You can follow the instructions in the URL below to get a test +// environment. +// +// https://github.com/hashicorp/nomad/tree/main/demo/csi/hostpath + +func TestResourceCSIVolume_basic(t *testing.T) { + resource.Test(t, resource.TestCase{ + Providers: testProviders, + PreCheck: func() { + testAccPreCheck(t) + testCheckCSIPluginAvailable(t, "hostpath-plugin0") + }, + Steps: []resource.TestStep{ + { + Config: ` +resource "nomad_csi_volume" "test" { + plugin_id = "hostpath-plugin0" + volume_id = "mysql_volume" + name = "mysql_volume" + capacity_min = "10GiB" + capacity_max = "20GiB" + + capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + mount_options { + fs_type = "ext4" + mount_flags = ["ro", "noatime"] + } + + topology_request { + required { + topology { + segments = { + rack = "R1" + "topology.hostpath.csi/node" = "node-0" + } + } + + topology { + segments = { + rack = "R2" + } + } + } + + preferred { + topology { + segments = { + zone = "us-east-1a" + } + } + } + } +} + `, + Check: func(s *terraform.State) error { + resourceState := s.Modules[0].Resources["nomad_csi_volume.test"] + if resourceState == nil { + return errors.New("resource not found in state") + } + + instanceState := resourceState.Primary + if instanceState == nil { + return errors.New("resource has no primary instance") + } + + if instanceState.ID != "mysql_volume" { + return fmt.Errorf("expected ID to be mysql_volume, got %s", instanceState.ID) + } + + expected := map[string]string{ + "namespace": "default", + "name": "mysql_volume", + "plugin_id": "hostpath-plugin0", + "capacity_min": "10GiB", + "capacity_max": "20GiB", + "mount_options.#": "1", + "mount_options.0.mount_flags.#": "2", + "mount_options.0.mount_flags.0": "ro", + "mount_options.0.mount_flags.1": "noatime", + "mount_options.0.fs_type": "ext4", + "topology_request.0.required.0.topology.0.segments.rack": "R1", + "topology_request.0.required.0.topology.0.segments.topology.hostpath.csi/node": "node-0", + "topology_request.0.required.0.topology.1.segments.rack": "R2", + "topology_request.0.preferred.0.topology.0.segments.zone": "us-east-1a", + "capability.#": "1", // capability is a set, so it's hard to infer their indexes. + } + for k, v := range expected { + got := instanceState.Attributes[k] + if got != v { + return fmt.Errorf("expected %s to be %s, got %s", k, v, got) + } + } + + client := testProvider.Meta().(ProviderConfig).client + volume, _, err := client.CSIVolumes().Info(instanceState.ID, nil) + if err != nil { + return fmt.Errorf("failed to read volume %s: %v", instanceState.ID, err) + } + + if volume.Name != expected["name"] { + return fmt.Errorf("expected Name to be %s, got: %s", expected["name"], volume.Name) + } + if volume.Namespace != expected["namespace"] { + return fmt.Errorf("expected Namespace to be %s, got: %s", expected["namespace"], volume.Namespace) + } + if volume.PluginID != expected["plugin_id"] { + return fmt.Errorf("expected PluginID to be %s, got: %s", expected["plugin_id"], volume.PluginID) + } + + expectedCapacity := int64(10 * 1024 * 1024 * 1024) + if volume.Capacity != expectedCapacity { + return fmt.Errorf("expected Capacity to be %d, got: %d", expectedCapacity, volume.Capacity) + } + expectedMinCapacity := int64(10 * 1024 * 1024 * 1024) + if volume.RequestedCapacityMin != expectedMinCapacity { + return fmt.Errorf("expected RequestedCapacityMin to be %d, got: %d", + expectedMinCapacity, volume.RequestedCapacityMin) + } + expectedMaxCapacity := int64(20 * 1024 * 1024 * 1024) + if volume.RequestedCapacityMax != expectedMaxCapacity { + return fmt.Errorf("expected RequestedCapacityMax to be %d, got: %d", + expectedMaxCapacity, volume.RequestedCapacityMax) + } + + expectedMountOptions := &api.CSIMountOptions{ + FSType: "ext4", + // mount flags may contain secrets, so they are not + // returned by the Nomad API, but check if they are at + // least set. + MountFlags: []string{"[REDACTED]"}, + } + if diff := cmp.Diff(expectedMountOptions, volume.MountOptions); diff != "" { + t.Errorf("MountOptions mismatch (-want +got):\n%s", diff) + } + + expectedCapabilities := []*api.CSIVolumeCapability{ + { + AccessMode: api.CSIVolumeAccessModeSingleNodeWriter, + AttachmentMode: api.CSIVolumeAttachmentModeFilesystem, + }, + } + if diff := cmp.Diff(expectedCapabilities, volume.RequestedCapabilities); diff != "" { + t.Errorf("RequestedCapabilities mismatch (-want +got):\n%s", diff) + } + + expectedTopologyRequest := &api.CSITopologyRequest{ + Required: []*api.CSITopology{ + { + Segments: map[string]string{ + "topology.hostpath.csi/node": "node-0", + "rack": "R1", + }, + }, + { + Segments: map[string]string{ + "rack": "R2", + }, + }, + }, + Preferred: []*api.CSITopology{ + { + Segments: map[string]string{ + "zone": "us-east-1a", + }, + }, + }, + } + if diff := cmp.Diff(expectedTopologyRequest, volume.RequestedTopologies); diff != "" { + t.Errorf("RequestedTopologies mismatch (-want +got):\n%s", diff) + } + + expectedTopologies := []*api.CSITopology{ + nil, // not sure why the hostpath plugin returns this nil topology. + { + Segments: map[string]string{ + "topology.hostpath.csi/node": "node-0", + }, + }, + } + if diff := cmp.Diff(expectedTopologies, volume.Topologies); diff != "" { + t.Errorf("Topologies mismatch (-want +got):\n%s", diff) + } + + return nil + }, + }, + }, + + CheckDestroy: func(s *terraform.State) error { + for _, s := range s.Modules[0].Resources { + if s.Type != "nomad_csi_volume" { + continue + } + if s.Primary == nil { + continue + } + client := testProvider.Meta().(ProviderConfig).client + volume, _, err := client.CSIVolumes().Info(s.Primary.ID, nil) + if err != nil && strings.Contains(err.Error(), "404") || volume == nil { + continue + } + return fmt.Errorf("volume %q has not been deleted.", volume.ID) + } + return nil + }, + }) +} diff --git a/nomad/resource_external_volume.go b/nomad/resource_external_volume.go index 20c16d1e..e90b8c84 100644 --- a/nomad/resource_external_volume.go +++ b/nomad/resource_external_volume.go @@ -18,6 +18,8 @@ import ( func resourceExternalVolume() *schema.Resource { return &schema.Resource{ + DeprecationMessage: "nomad_external_volume is deprecated and may be removed in a future release. Use nomad_csi_volume instead.", + Create: resourceExternalVolumeCreate, Update: resourceExternalVolumeCreate, Delete: resourceExternalVolumeDelete, diff --git a/nomad/resource_volume.go b/nomad/resource_volume.go index c3d602ef..95956df6 100644 --- a/nomad/resource_volume.go +++ b/nomad/resource_volume.go @@ -19,6 +19,8 @@ import ( func resourceVolume() *schema.Resource { return &schema.Resource{ + DeprecationMessage: "nomad_volume is deprecated and may be removed in a future release. Use nomad_csi_volume_registration instead.", + Create: resourceVolumeCreate, Update: resourceVolumeCreate, Delete: resourceVolumeDelete, diff --git a/website/docs/r/csi_volume.html.markdown b/website/docs/r/csi_volume.html.markdown new file mode 100644 index 00000000..f5af2b93 --- /dev/null +++ b/website/docs/r/csi_volume.html.markdown @@ -0,0 +1,122 @@ +--- +layout: "nomad" +page_title: "Nomad: nomad_csi_volume" +sidebar_current: "docs-nomad-resource-csi-volume" +description: |- + Manages the lifecycle of creating and deleting CSI volumes. +--- + +# nomad_csi_volume + +Creates and registers a CSI volume in Nomad. + +This can be used to create and register CSI volumes in a Nomad cluster. + +~> **Warning:** this resource will store any sensitive values placed in + `secrets` or `mount_options` in the Terraform's state file. Take care to + [protect your state file](/docs/state/sensitive-data.html). + +## Example Usage + +Creating a volume: + +```hcl +# It can sometimes be helpful to wait for a particular plugin to be available +data "nomad_plugin" "ebs" { + plugin_id = "aws-ebs0" + wait_for_healthy = true +} + +resource "nomad_csi_volume" "mysql_volume" { + depends_on = [data.nomad_plugin.ebs] + + plugin_id = "aws-ebs0" + volume_id = "mysql_volume" + name = "mysql_volume" + capacity_min = "10GiB" + capacity_max = "20GiB" + + capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + mount_options { + fs_type = "ext4" + } + + topology_request { + required { + topology { + segments = { + rack = "R1" + zone = "us-east-1a" + } + } + + topology { + segments = { + rack = "R2" + } + } + } + } +} +``` + +## Argument Reference + +The following arguments are supported: + +- `namespace`: `(string: "default")` - The namespace in which to register the volume. +- `volume_id`: `(string: )` - The unique ID of the volume. +- `name`: `(string: )` - The display name for the volume. +- `plugin_id`: `(string: )` - The ID of the Nomad plugin for registering this volume. +- `snapshot_id`: `(string: )` - The external ID of a snapshot to restore. If ommited, the volume will be created from scratch. Conflicts with `clone_id`. +- `clone_id`: `(string: )` - The external ID of an existing volume to restore. If ommited, the volume will be created from scratch. Conflicts with `snapshot_id`. +- `capacity_min`: `(string: )` - Option to signal a minimum volume size. This may not be supported by all storage providers. +- `capacity_max`: `(string: )` - Option to signal a maximum volume size. This may not be supported by all storage providers. +- `capability`: `(`[`Capability`](#capability-1)`: )` - Options for validating the capability of a volume. +- `topology_request`: `(`[`TopologyRequest`](#topology-request)`: )` - Specify locations (region, zone, rack, etc.) where the provisioned volume is accessible from. +- `mount_options`: `(block: optional)` Options for mounting `block-device` volumes without a pre-formatted file system. + - `fs_type`: `(string: optional)` - The file system type. + - `mount_flags`: `[]string: optional` - The flags passed to `mount`. +- `secrets`: `(map[string]string: optional)` An optional key-value map of strings used as credentials for publishing and unpublishing volumes. +- `parameters`: `(map[string]string: optional)` An optional key-value map of strings passed directly to the CSI plugin to configure the volume. + +### Capability + +- `access_mode`: `(string: )` - Defines whether a volume should be available concurrently. Possible values are: + - `single-node-reader-only` + - `single-node-writer` + - `multi-node-reader-only` + - `multi-node-single-writer` + - `multi-node-multi-writer` +- `attachment_mode`: `(string: )` - The storage API that will be used by the volume. Possible values are: + - `block-device` + - `file-system` + +### Topology Request + +- `required`: `(`[`Topology`](#topology)`: )` - Required topologies indicate that the volume must be created in a location accessible from all the listed topologies. +- `preferred`: `(`[`Topology`](#topology)`: )` - Preferred topologies indicate that the volume should be created in a location accessible from some of the listed topologies. + +### Topology + +- `topology`: `(List of segments: )` - Defines the location for the volume. + - `segments`: `(map[string]string)` - Define the attributes for the topology request. + +In addition to the above arguments, the following attributes are exported and +can be referenced: + +- `access_mode`: `(string)` +- `attachment_mode`: `(string)` +- `controller_required`: `(boolean)` +- `controllers_expected`: `(integer)` +- `controllers_healthy`: `(integer)` +- `plugin_provider`: `(string)` +- `plugin_provider_version`: `(string)` +- `nodes_healthy`: `(integer)` +- `nodes_expected`: `(integer)` +- `schedulable`: `(boolean)` +- `topologies`: `(List of topologies)` diff --git a/website/docs/r/csi_volume_registration.html.markdown b/website/docs/r/csi_volume_registration.html.markdown new file mode 100644 index 00000000..abce658e --- /dev/null +++ b/website/docs/r/csi_volume_registration.html.markdown @@ -0,0 +1,121 @@ +--- +layout: "nomad" +page_title: "Nomad: nomad_csi_volume_registration" +sidebar_current: "docs-nomad-resource-volume-registration" +description: |- + Manages the lifecycle of registering and deregistering CSI volumes. +--- + +# nomad_csi_volume_registration + +Manages the registration of a CSI volume in Nomad + +This can be used to register and deregister CSI volumes in a Nomad cluster. The +volume must already exist to be registered. Use the `nomad_csi_volume` +resource to create a new volume. + +~> **Warning:** this resource will store any sensitive values placed in + `secrets` or `mount_options` in the Terraform's state file. Take care to + [protect your state file](/docs/state/sensitive-data.html). + +## Example Usage + +Registering a volume: + +```hcl +# It can sometimes be helpful to wait for a particular plugin to be available +data "nomad_plugin" "ebs" { + plugin_id = "aws-ebs0" + wait_for_healthy = true +} + +resource "nomad_volume" "mysql_volume" { + depends_on = [data.nomad_plugin.ebs] + + plugin_id = "aws-ebs0" + volume_id = "mysql_volume" + name = "mysql_volume" + external_id = module.hashistack.ebs_test_volume_id + + capability { + access_mode = "single-node-writer" + attachment_mode = "file-system" + } + + mount_options { + fs_type = "ext4" + } + + topology_request { + required { + topology { + segments = { + rack = "R1" + zone = "us-east-1a" + } + } + + topology { + segments = { + rack = "R2" + } + } + } + } +} +``` + +## Argument Reference + +The following arguments are supported: + +- `namespace`: `(string: "default")` - The namespace in which to register the volume. +- `volume_id`: `(string: )` - The unique ID of the volume. +- `name`: `(string: )` - The display name for the volume. +- `plugin_id`: `(string: )` - The ID of the Nomad plugin for registering this volume. +- `external_id`: `(string: )` - The ID of the physical volume from the storage provider. +- `capability`: `(`[`Capability`](#capability-1)`: )` - Options for validating the capability of a volume. +- `topology_request`: `(`[`TopologyRequest`](#topology-request)`: )` - Specify locations (region, zone, rack, etc.) where the provisioned volume is accessible from. +- `mount_options`: `(block: )` Options for mounting `block-device` volumes without a pre-formatted file system. + - `fs_type`: `(string: )` - The file system type. + - `mount_flags`: `([]string: )` - The flags passed to `mount`. +- `secrets`: `(map[string]string: )` - An optional key-value map of strings used as credentials for publishing and unpublishing volumes. +- `parameters`: `(map[string]string: )` - An optional key-value map of strings passed directly to the CSI plugin to configure the volume. +- `context`: `(map[string]string: )` - An optional key-value map of strings passed directly to the CSI plugin to validate the volume. +- `deregister_on_destroy`: `(boolean: false)` - If true, the volume will be deregistered on destroy. + +### Capability + +- `access_mode`: `(string: )` - Defines whether a volume should be available concurrently. Possible values are: + - `single-node-reader-only` + - `single-node-writer` + - `multi-node-reader-only` + - `multi-node-single-writer` + - `multi-node-multi-writer` +- `attachment_mode`: `(string: )` - The storage API that will be used by the volume. Possible values are: + - `block-device` + - `file-system` + +### Topology Request + +- `required`: `(`[`Topology`](#topology)`: )` - Required topologies indicate that the volume must be created in a location accessible from all the listed topologies. + +### Topology + +- `topology`: `(List of segments: )` - Defines the location for the volume. + - `segments`: `(map[string]string)` - Define the attributes for the topology request. + +In addition to the above arguments, the following attributes are exported and +can be referenced: + +- `access_mode`: `(string)` +- `attachment_mode`: `(string)` +- `controller_required`: `(boolean)` +- `controllers_expected`: `(integer)` +- `controllers_healthy`: `(integer)` +- `plugin_provider`: `(string)` +- `plugin_provider_version`: `(string)` +- `nodes_healthy`: `(integer)` +- `nodes_expected`: `(integer)` +- `schedulable`: `(boolean)` +- `topologies`: `(List of topologies)` diff --git a/website/docs/r/external_volume.html.markdown b/website/docs/r/external_volume.html.markdown index 90e7cb79..51096fc4 100644 --- a/website/docs/r/external_volume.html.markdown +++ b/website/docs/r/external_volume.html.markdown @@ -8,6 +8,9 @@ description: |- # nomad_external_volume +~> **Deprecated:** This resource has been deprecated and may be removed in a +future release. Use `nomad_csi_volume` instead. + Creates and registers an external volume in Nomad. This can be used to create and register external volumes in a Nomad cluster. diff --git a/website/docs/r/volume.html.markdown b/website/docs/r/volume.html.markdown index c1e38580..69ecb709 100644 --- a/website/docs/r/volume.html.markdown +++ b/website/docs/r/volume.html.markdown @@ -8,6 +8,9 @@ description: |- # nomad_volume +~> **Deprecated:** This resource has been deprecated and may be removed in a +future release. Use `nomad_csi_volume_registration` instead. + Manages an external volume in Nomad. This can be used to register external volumes in a Nomad cluster. diff --git a/website/nomad.erb b/website/nomad.erb index 99a6070e..938691dd 100644 --- a/website/nomad.erb +++ b/website/nomad.erb @@ -76,6 +76,12 @@ > nomad_acl_token + > + nomad_csi_volume + + > + nomad_csi_volume_registration + > nomad_external_volume