Skip to content

Commit

Permalink
k8s: include client certs and node ca in mounted volumes
Browse files Browse the repository at this point in the history
  • Loading branch information
@alenkacz
alenkacz committed Dec 17, 2022
1 parent 5d6e8b0 commit 3f699ef
Show file tree
Hide file tree
Showing 2 changed files with 105 additions and 35 deletions.
67 changes: 48 additions & 19 deletions src/go/k8s/pkg/resources/certmanager/type_helpers.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -29,14 +29,18 @@ import (
)

const (
redpandaCertVolName = "tlscert"
redpandaCAVolName = "tlsca"
adminAPICertVolName = "tlsadmincert"
adminAPICAVolName = "tlsadminca"
pandaProxyCertVolName = "tlspandaproxycert"
pandaProxyCAVolName = "tlspandaproxyca"
schemaRegistryCertVolName = "tlsschemaregistrycert"
schemaRegistryCAVolName = "tlsschemaregistryca"
redpandaCertVolName = "tlscert"
// originally this volume contained only client CA but for pandaproxy and
// schema registry we need to also include the certs. Now the name of the
// volume does not align with its contents but changing this would mean we
// force restart of redpanda when updating to this version
redpandaClientVolName = "tlsca"
adminAPICertVolName = "tlsadmincert"
adminAPIClientCAVolName = "tlsadminca"
pandaProxyCertVolName = "tlspandaproxycert"
pandaProxyClientCAVolName = "tlspandaproxyca"
schemaRegistryCertVolName = "tlsschemaregistrycert"
schemaRegistryClientCAVolName = "tlsschemaregistryca"
)

// Helper functions and types for Listeners
Expand Down Expand Up @@ -132,6 +136,8 @@ type apiCertificates struct {
clientCertificates []resources.Resource
rootResources []resources.Resource
tlsEnabled bool
// true if api is using our own generated self-signed issuer
selfSignedNodeCertificate bool

// CR allows to specify node certificate, if not provided this will be nil
externalNodeCertificate *corev1.ObjectReference
Expand Down Expand Up @@ -245,6 +251,7 @@ func (cc *ClusterCertificates) prepareAPI(
// every time both listeners share the same set of certificates
nodeSecretRef := tlsListeners[0].GetTLS().NodeSecretRef
result.externalNodeCertificate = nodeSecretRef
result.selfSignedNodeCertificate = tlsListeners[0].GetTLS().IssuerRef == nil && nodeSecretRef == nil
if nodeSecretRef == nil || nodeSecretRef.Name == "" {
certName := NewCertName(cc.pandaCluster.Name, nodeCertSuffix)
certsKey := types.NamespacedName{Name: string(certName), Namespace: cc.pandaCluster.Namespace}
Expand Down Expand Up @@ -459,19 +466,21 @@ func (cc *ClusterCertificates) Volumes() (
var mounts []corev1.VolumeMount
mountPoints := resourcetypes.GetTLSMountPoints()

vol, mount := secretVolumesForTLS(cc.kafkaAPI.nodeCertificateName(), cc.kafkaAPI.clientCertificates, redpandaCertVolName, redpandaCAVolName, mountPoints.KafkaAPI.NodeCertMountDir, mountPoints.KafkaAPI.ClientCAMountDir)
// kafka client certs are needed for pandaproxy and schema registry if enabled
shouldIncludeKafkaClientCerts := len(cc.kafkaAPI.clientCertificates) > 0
vol, mount := secretVolumesForTLS(cc.kafkaAPI.nodeCertificateName(), cc.kafkaAPI.clientCertificates, redpandaCertVolName, redpandaClientVolName, mountPoints.KafkaAPI.NodeCertMountDir, mountPoints.KafkaAPI.ClientCAMountDir, cc.kafkaAPI.selfSignedNodeCertificate, shouldIncludeKafkaClientCerts)
vols = append(vols, vol...)
mounts = append(mounts, mount...)

vol, mount = secretVolumesForTLS(cc.adminAPI.nodeCertificateName(), cc.adminAPI.clientCertificates, adminAPICertVolName, adminAPICAVolName, mountPoints.AdminAPI.NodeCertMountDir, mountPoints.AdminAPI.ClientCAMountDir)
vol, mount = secretVolumesForTLS(cc.adminAPI.nodeCertificateName(), cc.adminAPI.clientCertificates, adminAPICertVolName, adminAPIClientCAVolName, mountPoints.AdminAPI.NodeCertMountDir, mountPoints.AdminAPI.ClientCAMountDir, false, false)
vols = append(vols, vol...)
mounts = append(mounts, mount...)

vol, mount = secretVolumesForTLS(cc.pandaProxyAPI.nodeCertificateName(), cc.pandaProxyAPI.clientCertificates, pandaProxyCertVolName, pandaProxyCAVolName, mountPoints.PandaProxyAPI.NodeCertMountDir, mountPoints.PandaProxyAPI.ClientCAMountDir)
vol, mount = secretVolumesForTLS(cc.pandaProxyAPI.nodeCertificateName(), cc.pandaProxyAPI.clientCertificates, pandaProxyCertVolName, pandaProxyClientCAVolName, mountPoints.PandaProxyAPI.NodeCertMountDir, mountPoints.PandaProxyAPI.ClientCAMountDir, false, false)
vols = append(vols, vol...)
mounts = append(mounts, mount...)

vol, mount = secretVolumesForTLS(cc.schemaRegistryAPI.nodeCertificateName(), cc.schemaRegistryAPI.clientCertificates, schemaRegistryCertVolName, schemaRegistryCAVolName, mountPoints.SchemaRegistryAPI.NodeCertMountDir, mountPoints.SchemaRegistryAPI.ClientCAMountDir)
vol, mount = secretVolumesForTLS(cc.schemaRegistryAPI.nodeCertificateName(), cc.schemaRegistryAPI.clientCertificates, schemaRegistryCertVolName, schemaRegistryClientCAVolName, mountPoints.SchemaRegistryAPI.NodeCertMountDir, mountPoints.SchemaRegistryAPI.ClientCAMountDir, false, false)
vols = append(vols, vol...)
mounts = append(mounts, mount...)

Expand All @@ -481,7 +490,8 @@ func (cc *ClusterCertificates) Volumes() (
func secretVolumesForTLS(
nodeCertificate *types.NamespacedName,
clientCertificates []resources.Resource,
volumeName, caVolumeName, mountDir, caMountDir string,
volumeName, clientVolumeName, mountDir, caMountDir string,
shouldIncludeNodeCa, shouldIncludeClientCert bool,
) ([]corev1.Volume, []corev1.VolumeMount) {
var vols []corev1.Volume
var mounts []corev1.VolumeMount
Expand All @@ -490,7 +500,7 @@ func secretVolumesForTLS(
}

// mount node certificate's private key
vols = append(vols, corev1.Volume{
nodeVolume := corev1.Volume{
Name: volumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
Expand All @@ -507,7 +517,16 @@ func secretVolumesForTLS(
},
},
},
})
}

if shouldIncludeNodeCa {
nodeVolume.VolumeSource.Secret.Items = append(nodeVolume.VolumeSource.Secret.Items, corev1.KeyToPath{
Key: cmmetav1.TLSCAKey,
Path: cmmetav1.TLSCAKey,
})
}

vols = append(vols, nodeVolume)
mounts = append(mounts, corev1.VolumeMount{
Name: volumeName,
MountPath: mountDir,
Expand All @@ -516,8 +535,8 @@ func secretVolumesForTLS(
// if mutual TLS is enabled, mount also client cerificate CA to be able to
// verify client certificates
if len(clientCertificates) > 0 {
vols = append(vols, corev1.Volume{
Name: caVolumeName,
clientCertVolume := corev1.Volume{
Name: clientVolumeName,
VolumeSource: corev1.VolumeSource{
Secret: &corev1.SecretVolumeSource{
SecretName: clientCertificates[0].Key().Name,
Expand All @@ -529,9 +548,19 @@ func secretVolumesForTLS(
},
},
},
})
}
if shouldIncludeClientCert {
clientCertVolume.VolumeSource.Secret.Items = append(clientCertVolume.VolumeSource.Secret.Items, corev1.KeyToPath{
Key: corev1.TLSPrivateKeyKey,
Path: corev1.TLSPrivateKeyKey,
}, corev1.KeyToPath{
Key: corev1.TLSCertKey,
Path: corev1.TLSCertKey,
})
}
vols = append(vols, clientCertVolume)
mounts = append(mounts, corev1.VolumeMount{
Name: caVolumeName,
Name: clientVolumeName,
MountPath: caMountDir,
})
}
Expand Down
73 changes: 57 additions & 16 deletions src/go/k8s/pkg/resources/certmanager/type_helpers_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@ func TestClusterCertificates(t *testing.T) {
pandaCluster *v1alpha1.Cluster
expectedNames []string
volumesCount int
verifyVolumes func(vols []corev1.Volume) bool
}{
{"kafka tls disabled", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Expand All @@ -58,7 +59,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{}, 0},
}, []string{}, 0, nil},
{"kafka tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -72,7 +73,15 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1, func(vols []corev1.Volume) bool {
// verify the volume also contains CA in the node tls folder
for _, i := range vols[0].Secret.Items {
if i.Key == cmmetav1.TLSCAKey {
return true
}
}
return false
}},
{"kafka tls with two tls listeners", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -94,7 +103,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1, nil},
{"kafka tls with external node issuer", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -111,7 +120,15 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda"}, 1, func(vols []corev1.Volume) bool {
// verify the volume does not contain CA in the node tls folder when node cert is injected
for _, i := range vols[0].Secret.Items {
if i.Key == cmmetav1.TLSCAKey {
return false
}
}
return true
}},
{"kafka mutual tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -134,7 +151,28 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda", "test-operator-client", "test-user-client", "test-admin-client"}, 2},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda", "test-operator-client", "test-user-client", "test-admin-client"}, 2, func(vols []corev1.Volume) bool {
// verify the ca volume contains also client cert
foundKey := false
foundCrt := false
for _, v := range vols {
if v.Name == "tlsca" {
for _, i := range vols[0].Secret.Items {
if i.Key == corev1.TLSCertKey {
foundCrt = true
}
if i.Key == corev1.TLSPrivateKeyKey {
foundKey = true
}
if foundKey && foundCrt {
break
}
}
break
}
}
return foundCrt && foundKey
}},
{"kafka mutual tls with two tls listeners", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -149,7 +187,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda", "test-operator-client", "test-user-client", "test-admin-client"}, 2},
}, []string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-redpanda", "test-operator-client", "test-user-client", "test-admin-client"}, 2, nil},
{"admin api tls disabled", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -163,7 +201,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{}, 0},
}, []string{}, 0, nil},
{"admin api tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -177,7 +215,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node"}, 1},
}, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node"}, 1, nil},
{"admin api mutual tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -192,7 +230,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node", "test-admin-api-client"}, 2},
}, []string{"test-admin-selfsigned-issuer", "test-admin-root-certificate", "test-admin-root-issuer", "test-admin-api-node", "test-admin-api-client"}, 2, nil},
{"pandaproxy api tls disabled", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -206,7 +244,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{}, 0},
}, []string{}, 0, nil},
{"pandaproxy api tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -220,7 +258,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node"}, 1},
}, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node"}, 1, nil},
{"pandaproxy api mutual tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -235,7 +273,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node", "test-proxy-api-client"}, 2},
}, []string{"test-proxy-selfsigned-issuer", "test-proxy-root-certificate", "test-proxy-root-issuer", "test-proxy-api-node", "test-proxy-api-client"}, 2, nil},
{"schematregistry api tls disabled", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Spec: v1alpha1.ClusterSpec{
Expand All @@ -247,7 +285,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
},
}, []string{}, 0},
}, []string{}, 0, nil},
{
"schematregistry api tls", &v1alpha1.Cluster{
ObjectMeta: v1.ObjectMeta{Name: "test", Namespace: "test"},
Expand All @@ -262,7 +300,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
[]string{"test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-node"},
1,
1, nil,
},
{
"schematregistry api mutual tls", &v1alpha1.Cluster{
Expand All @@ -279,7 +317,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
[]string{"test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-node", "test-schema-registry-client"},
2,
2, nil,
},
{
"kafka and schematregistry with nodesecretref", &v1alpha1.Cluster{
Expand Down Expand Up @@ -312,7 +350,7 @@ func TestClusterCertificates(t *testing.T) {
},
},
[]string{"test-kafka-selfsigned-issuer", "test-kafka-root-certificate", "test-kafka-root-issuer", "test-operator-client", "test-user-client", "test-admin-client", "test-schema-registry-selfsigned-issuer", "test-schema-registry-root-certificate", "test-schema-registry-root-issuer", "test-schema-registry-client"},
4,
4, nil,
},
}
for _, tt := range tests {
Expand Down Expand Up @@ -340,5 +378,8 @@ func TestClusterCertificates(t *testing.T) {
v, vm := cc.Volumes()
require.Equal(t, tt.volumesCount, len(v), fmt.Sprintf("%s: volumes count don't match", tt.name))
require.Equal(t, tt.volumesCount, len(vm), fmt.Sprintf("%s: volume mounts count don't match", tt.name))
if tt.verifyVolumes != nil {
require.True(t, tt.verifyVolumes(v), "failed during volumes verification")
}
}
}

0 comments on commit 3f699ef

Please sign in to comment.