Skip to content

Commit

Permalink
Add support for templatized secret references in storage class parame…
Browse files Browse the repository at this point in the history 
…ters

The following storage class parameters are supported:

csiProvisionerSecretName       / csiProvisionerSecretNamespace
csiControllerPublishSecretName / csiControllerPublishSecretNamespace
csiNodeStageSecretName         / csiNodeStageSecretNamespace
csiNodePublishSecretName       / csiNodePublishSecretNamespace
  • Loading branch information
@liggitt
liggitt committed Apr 18, 2018
1 parent f27faff commit 1c41000
Show file tree
Hide file tree
Showing 2 changed files with 286 additions and 53 deletions.
215 changes: 162 additions & 53 deletions pkg/controller/controller.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,7 @@ import (
"context"
"fmt"
"net"
"os"
"strings"
"time"

Expand All @@ -32,6 +33,8 @@ import (

"k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/apimachinery/pkg/util/validation"
"k8s.io/apimachinery/pkg/util/wait"
"k8s.io/client-go/kubernetes"
"k8s.io/client-go/rest"
Expand All @@ -45,8 +48,18 @@ import (
)

const (
secretNameKey = "csiProvisionerSecretName"
secretNamespaceKey = "csiProvisionerSecretNamespace"
provisionerSecretNameKey = "csiProvisionerSecretName"
provisionerSecretNamespaceKey = "csiProvisionerSecretNamespace"

controllerPublishSecretNameKey = "csiControllerPublishSecretName"
controllerPublishSecretNamespaceKey = "csiControllerPublishSecretNamespace"

nodeStageSecretNameKey = "csiNodeStageSecretName"
nodeStageSecretNamespaceKey = "csiNodeStageSecretNamespace"

nodePublishSecretNameKey = "csiNodePublishSecretName"
nodePublishSecretNamespaceKey = "csiNodePublishSecretNamespace"

// Defines parameters for ExponentialBackoff used for executing
// CSI CreateVolume API call, it gives approx 4 minutes for the CSI
// driver to complete a volume creation.
Expand Down Expand Up @@ -271,7 +284,7 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
return nil, err
}

share, err := makeVolumeName(p.volumeNamePrefix, fmt.Sprintf("%s", options.PVC.ObjectMeta.UID))
pvName, err := makeVolumeName(p.volumeNamePrefix, fmt.Sprintf("%s", options.PVC.ObjectMeta.UID))
if err != nil {
return nil, err
}
Expand All @@ -282,7 +295,7 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
// Create a CSI CreateVolumeRequest and Response
req := csi.CreateVolumeRequest{

Name: share,
Name: pvName,
Parameters: options.Parameters,
VolumeCapabilities: []*csi.VolumeCapability{
{
Expand All @@ -296,14 +309,30 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
}
rep := &csi.CreateVolumeResponse{}

secret := v1.SecretReference{}
if options.Parameters != nil {
credentials, err := getCredentialsFromParameters(p.client, options.Parameters)
if err != nil {
return nil, err
}
req.ControllerCreateSecrets = credentials
secret.Name, secret.Namespace, _ = getSecretAndNamespaceFromParameters(options.Parameters)
// Resolve provision secret credentials.
// No PVC is provided when resolving provision/delete secret names, since the PVC may or may not exist at delete time.
provisionerSecretRef, err := getSecretReference(provisionerSecretNameKey, provisionerSecretNamespaceKey, options.Parameters, pvName, nil)
if err != nil {
return nil, err
}
provisionerCredentials, err := getCredentials(p.client, provisionerSecretRef)
if err != nil {
return nil, err
}
req.ControllerCreateSecrets = provisionerCredentials

// Resolve controller publish, node stage, node publish secret references
controllerPublishSecretRef, err := getSecretReference(controllerPublishSecretNameKey, controllerPublishSecretNamespaceKey, options.Parameters, pvName, options.PVC)
if err != nil {
return nil, err
}
nodeStageSecretRef, err := getSecretReference(nodeStageSecretNameKey, nodeStageSecretNamespaceKey, options.Parameters, pvName, options.PVC)
if err != nil {
return nil, err
}
nodePublishSecretRef, err := getSecretReference(nodePublishSecretNameKey, nodePublishSecretNamespaceKey, options.Parameters, pvName, options.PVC)
if err != nil {
return nil, err
}

opts := wait.Backoff{Duration: backoffDuration, Factor: backoffFactor, Steps: backoffSteps}
Expand Down Expand Up @@ -344,25 +373,19 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
delReq := &csi.DeleteVolumeRequest{
VolumeId: rep.GetVolume().GetId(),
}
if options.Parameters != nil {
credentials, err := getCredentialsFromParameters(p.client, options.Parameters)
if err != nil {
return nil, err
}
delReq.ControllerDeleteSecrets = credentials
}
delReq.ControllerDeleteSecrets = provisionerCredentials
ctx, cancel := context.WithTimeout(context.Background(), p.timeout)
defer cancel()
_, err := p.csiClient.DeleteVolume(ctx, delReq)
if err != nil {
capErr = fmt.Errorf("%v. Cleanup of volume %s failed, volume is orphaned: %v", capErr, share, err)
capErr = fmt.Errorf("%v. Cleanup of volume %s failed, volume is orphaned: %v", capErr, pvName, err)
}
return nil, capErr
}
repBytesString := fmt.Sprintf("%v", respCap)
pv := &v1.PersistentVolume{
ObjectMeta: metav1.ObjectMeta{
Name: share,
Name: pvName,
},
Spec: v1.PersistentVolumeSpec{
PersistentVolumeReclaimPolicy: options.PersistentVolumeReclaimPolicy,
Expand All @@ -373,16 +396,17 @@ func (p *csiProvisioner) Provision(options controller.VolumeOptions) (*v1.Persis
// TODO wait for CSI VolumeSource API
PersistentVolumeSource: v1.PersistentVolumeSource{
CSI: &v1.CSIPersistentVolumeSource{
Driver: driverName,
VolumeHandle: p.volumeIdToHandle(rep.Volume.Id),
VolumeAttributes: volumeAttributes,
Driver: driverName,
VolumeHandle: p.volumeIdToHandle(rep.Volume.Id),
VolumeAttributes: volumeAttributes,
ControllerPublishSecretRef: controllerPublishSecretRef,
NodeStageSecretRef: nodeStageSecretRef,
NodePublishSecretRef: nodePublishSecretRef,
},
},
},
}
if len(secret.Name) != 0 {
pv.Spec.PersistentVolumeSource.CSI.NodePublishSecretRef = &secret
}

glog.Infof("successfully created PV %+v", pv.Spec.PersistentVolumeSource)

return pv, nil
Expand All @@ -406,7 +430,13 @@ func (p *csiProvisioner) Delete(volume *v1.PersistentVolume) error {
storageClassName := volume.Spec.StorageClassName
if len(storageClassName) != 0 {
if storageClass, err := p.client.StorageV1().StorageClasses().Get(storageClassName, metav1.GetOptions{}); err == nil {
credentials, err := getCredentialsFromParameters(p.client, storageClass.Parameters)
// Resolve provision secret credentials.
// No PVC is provided when resolving provision/delete secret names, since the PVC may or may not exist at delete time.
provisionerSecretRef, err := getSecretReference(provisionerSecretNameKey, provisionerSecretNamespaceKey, storageClass.Parameters, volume.Name, nil)
if err != nil {
return err
}
credentials, err := getCredentials(p.client, provisionerSecretRef)
if err != nil {
return err
}
Expand All @@ -422,24 +452,6 @@ func (p *csiProvisioner) Delete(volume *v1.PersistentVolume) error {
return err
}

func getSecretAndNamespaceFromParameters(parameters map[string]string) (string, string, bool) {
if secretName, ok := parameters[secretNameKey]; ok {
namespace := "default"
if len(parameters[secretNamespaceKey]) != 0 {
namespace = parameters[secretNamespaceKey]
}
return secretName, namespace, true
}
return "", "", false
}

func getCredentialsFromParameters(k8s kubernetes.Interface, parameters map[string]string) (map[string]string, error) {
if secretName, namespace, found := getSecretAndNamespaceFromParameters(parameters); found {
return getCredentialsFromSecret(k8s, secretName, namespace)
}
return map[string]string{}, nil
}

//TODO use a unique volume handle from and to Id
func (p *csiProvisioner) volumeIdToHandle(id string) string {
return id
Expand All @@ -449,18 +461,115 @@ func (p *csiProvisioner) volumeHandleToId(handle string) string {
return handle
}

func getCredentialsFromSecret(k8s kubernetes.Interface, secretName, nameSpace string) (map[string]string, error) {
credentials := map[string]string{}
if len(secretName) == 0 {
return credentials, nil
// getSecretReference returns a reference to the secret specified in the given nameKey and namespaceKey parameters, or an error if the parameters are not specified correctly.
// if neither the name or namespace parameter are set, a nil reference and no error is returned.
// no lookup of the referenced secret is performed, and the secret may or may not exist.
//
// supported tokens for name resolution:
// - ${pv.name}
// - ${pvc.namespace}
// - ${pvc.name}
// - ${pvc.annotations['ANNOTATION_KEY']} (e.g. ${pvc.annotations['example.com/node-publish-secret-name']})
//
// supported tokens for namespace resolution:
// - ${pv.name}
// - ${pvc.namespace}
//
// an error is returned in the following situations:
// - only one of name or namespace is provided
// - the name or namespace parameter contains a token that cannot be resolved
// - the resolved name is not a valid secret name
// - the resolved namespace is not a valid namespace name
func getSecretReference(nameKey, namespaceKey string, storageClassParams map[string]string, pvName string, pvc *v1.PersistentVolumeClaim) (*v1.SecretReference, error) {
nameTemplate, hasName := storageClassParams[nameKey]
namespaceTemplate, hasNamespace := storageClassParams[namespaceKey]

if !hasName && !hasNamespace {
return nil, nil
}

if len(nameTemplate) == 0 || len(namespaceTemplate) == 0 {
return nil, fmt.Errorf("%s and %s parameters must be specified together", nameKey, namespaceKey)
}

ref := &v1.SecretReference{}

{
// Secret namespace template can make use of the PV name or the PVC namespace.
// Note that neither of those things are under the control of the PVC user.
namespaceParams := map[string]string{"pv.name": pvName}
if pvc != nil {
namespaceParams["pvc.namespace"] = pvc.Namespace
}

resolvedNamespace, err := resolveTemplate(namespaceTemplate, namespaceParams)
if err != nil {
return nil, fmt.Errorf("error resolving %s value %q: %v", namespaceKey, namespaceTemplate, err)
}
if len(validation.IsDNS1123Label(resolvedNamespace)) > 0 {
if namespaceTemplate != resolvedNamespace {
return nil, fmt.Errorf("%s parameter %q resolved to %q which is not a valid namespace name", namespaceKey, namespaceTemplate, resolvedNamespace)
}
return nil, fmt.Errorf("%s parameter %q is not a valid namespace name", namespaceKey, namespaceTemplate)
}
ref.Namespace = resolvedNamespace
}

{
// Secret name template can make use of the PV name, PVC name or namespace, or a PVC annotation.
// Note that PVC name and annotations are under the PVC user's control.
nameParams := map[string]string{"pv.name": pvName}
if pvc != nil {
nameParams["pvc.name"] = pvc.Name
nameParams["pvc.namespace"] = pvc.Namespace
for k, v := range pvc.Annotations {
nameParams["pvc.annotations['"+k+"']"] = v
}
}
resolvedName, err := resolveTemplate(nameTemplate, nameParams)
if err != nil {
return nil, fmt.Errorf("error resolving %s value %q: %v", nameKey, nameTemplate, err)
}
if len(validation.IsDNS1123Subdomain(resolvedName)) > 0 {
if nameTemplate != resolvedName {
return nil, fmt.Errorf("%s parameter %q resolved to %q which is not a valid secret name", nameKey, nameTemplate, resolvedName)
}
return nil, fmt.Errorf("%s parameter %q is not a valid secret name", nameKey, nameTemplate)
}
ref.Name = resolvedName
}

return ref, nil
}

func resolveTemplate(template string, params map[string]string) (string, error) {
missingParams := sets.NewString()
resolved := os.Expand(template, func(k string) string {
v, ok := params[k]
if !ok {
missingParams.Insert(k)
}
return v
})
if missingParams.Len() > 0 {
return "", fmt.Errorf("invalid tokens: %q", missingParams.List())
}
secret, err := k8s.CoreV1().Secrets(nameSpace).Get(secretName, metav1.GetOptions{})
return resolved, nil
}

func getCredentials(k8s kubernetes.Interface, ref *v1.SecretReference) (map[string]string, error) {
if ref == nil {
return nil, nil
}

secret, err := k8s.CoreV1().Secrets(ref.Namespace).Get(ref.Name, metav1.GetOptions{})
if err != nil {
return credentials, fmt.Errorf("failed to find the secret %s in the namespace %s with error: %v\n", secretName, nameSpace, err)
return nil, fmt.Errorf("error getting secret %s in namespace %s: %v", ref.Name, ref.Namespace, err)
}

credentials := map[string]string{}
for key, value := range secret.Data {
credentials[key] = string(value)
}

return credentials, nil
}
Loading

0 comments on commit 1c41000

Please sign in to comment.