components: added tls managed component (PROJQUAY-2050)

Adds a new managed component, 'tls'. This separates the
'route' component from TLS and allows users to configure
both separately. 'EXTERNAL_TLS_TERMINATION: true' is the
opinionated setting. Managed 'tls' means the default
cluster wildcard cert is used. Unmanaged 'tls' means
the user-provided cert/key pair will be injected into
the 'Route'.

Signed-off-by: Alec Merdler <alecmerdler@gmail.com>

apis/quay/v1/quayregistry_types.go

@@ -45,6 +45,7 @@ const (
 	ComponentRoute                       = "route"
 	ComponentMirror                      = "mirror"
 	ComponentMonitoring                  = "monitoring"
+	ComponentTLS                         = "tls"
 )
 
 var allComponents = []ComponentKind{
@@ -56,6 +57,7 @@ var allComponents = []ComponentKind{
 	ComponentRoute,
 	ComponentMirror,
 	ComponentMonitoring,
+	ComponentTLS,
 }
 
 var requiredComponents = []ComponentKind{
@@ -259,18 +261,19 @@ func EnsureDefaultComponents(ctx *quaycontext.QuayRegistryContext, quay *QuayReg
 	}
 
 	type componentCheck struct {
-		check bool
+		check func() bool
 		msg   string
 	}
 	componentChecks := map[ComponentKind]componentCheck{
-		ComponentRoute:         {ctx.SupportsRoutes, "cannot use `route` component when `Route` API not available"},
-		ComponentObjectStorage: {ctx.SupportsObjectStorage, "cannot use `ObjectStorage` component when `ObjectStorage` API not available"},
-		ComponentMonitoring:    {ctx.SupportsMonitoring, "cannot use `monitoring` component when `Prometheus` API not available"},
+		ComponentRoute:         {func() bool { return ctx.SupportsRoutes }, "cannot use `route` component when `Route` API not available"},
+		ComponentTLS:           {func() bool { return ctx.SupportsRoutes && ctx.TLSCert == nil && ctx.TLSKey == nil }, "cannot use `tls` component when `Route` API not available or TLS cert/key pair is provided"},
+		ComponentObjectStorage: {func() bool { return ctx.SupportsObjectStorage }, "cannot use `ObjectStorage` component when `ObjectStorage` API not available"},
+		ComponentMonitoring:    {func() bool { return ctx.SupportsMonitoring }, "cannot use `monitoring` component when `Prometheus` API not available"},
 	}
 
 	for _, component := range allComponents {
 		componentCheck, checkExists := componentChecks[component]
-		if (checkExists && !componentCheck.check) && ComponentIsManaged(quay.Spec.Components, component) {
+		if (checkExists && !componentCheck.check()) && ComponentIsManaged(quay.Spec.Components, component) {
 			return quay, errors.New(componentCheck.msg)
 		}
 
@@ -285,7 +288,7 @@ func EnsureDefaultComponents(ctx *quaycontext.QuayRegistryContext, quay *QuayReg
 		if !found {
 			updatedQuay.Spec.Components = append(updatedQuay.Spec.Components, Component{
 				Kind:    component,
-				Managed: !checkExists || componentCheck.check,
+				Managed: !checkExists || componentCheck.check(),
 			})
 		}
 	}

apis/quay/v1/quayregistry_types_test.go

@@ -27,6 +27,7 @@ var ensureDefaultComponentsTests = []struct {
 					{Kind: "clair", Managed: true},
 					{Kind: "objectstorage", Managed: true},
 					{Kind: "route", Managed: true},
+					{Kind: "tls", Managed: true},
 					{Kind: "horizontalpodautoscaler", Managed: true},
 					{Kind: "mirror", Managed: true},
 					{Kind: "monitoring", Managed: true},
@@ -44,6 +45,7 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: true},
 			{Kind: "route", Managed: true},
+			{Kind: "tls", Managed: true},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: true},
@@ -60,6 +62,7 @@ var ensureDefaultComponentsTests = []struct {
 					{Kind: "clair", Managed: true},
 					{Kind: "objectstorage", Managed: true},
 					{Kind: "route", Managed: true},
+					{Kind: "tls", Managed: true},
 					{Kind: "horizontalpodautoscaler", Managed: true},
 					{Kind: "mirror", Managed: true},
 				},
@@ -72,6 +75,7 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: true},
 			{Kind: "route", Managed: true},
+			{Kind: "tls", Managed: true},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 		},
@@ -87,6 +91,7 @@ var ensureDefaultComponentsTests = []struct {
 					{Kind: "clair", Managed: true},
 					{Kind: "objectstorage", Managed: true},
 					{Kind: "route", Managed: true},
+					{Kind: "tls", Managed: true},
 					{Kind: "horizontalpodautoscaler", Managed: true},
 					{Kind: "mirror", Managed: true},
 					{Kind: "monitoring", Managed: true},
@@ -104,12 +109,48 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: true},
 			{Kind: "route", Managed: true},
+			{Kind: "tls", Managed: true},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: true},
 		},
 		errors.New("cannot use `route` component when `Route` API not available"),
 	},
+	{
+		"TLSComponentProvidedWithoutRoutes",
+		QuayRegistry{
+			Spec: QuayRegistrySpec{
+				Components: []Component{
+					{Kind: "postgres", Managed: true},
+					{Kind: "redis", Managed: true},
+					{Kind: "clair", Managed: true},
+					{Kind: "objectstorage", Managed: true},
+					{Kind: "route", Managed: false},
+					{Kind: "tls", Managed: true},
+					{Kind: "horizontalpodautoscaler", Managed: true},
+					{Kind: "mirror", Managed: true},
+					{Kind: "monitoring", Managed: true},
+				},
+			},
+		},
+		quaycontext.QuayRegistryContext{
+			SupportsRoutes:        false,
+			SupportsObjectStorage: true,
+			SupportsMonitoring:    true,
+		},
+		[]Component{
+			{Kind: "postgres", Managed: true},
+			{Kind: "redis", Managed: true},
+			{Kind: "clair", Managed: true},
+			{Kind: "objectstorage", Managed: true},
+			{Kind: "route", Managed: false},
+			{Kind: "tls", Managed: true},
+			{Kind: "horizontalpodautoscaler", Managed: true},
+			{Kind: "mirror", Managed: true},
+			{Kind: "monitoring", Managed: true},
+		},
+		errors.New("cannot use `tls` component when `Route` API not available or TLS cert/key pair is provided"),
+	},
 	{
 		"AllComponentsOmitted",
 		QuayRegistry{
@@ -122,6 +163,7 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: false},
 			{Kind: "route", Managed: false},
+			{Kind: "tls", Managed: false},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: false},
@@ -144,6 +186,7 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: false},
 			{Kind: "route", Managed: true},
+			{Kind: "tls", Managed: true},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: true},
@@ -168,6 +211,7 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: false},
 			{Kind: "route", Managed: false},
+			{Kind: "tls", Managed: false},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: false},
@@ -196,6 +240,38 @@ var ensureDefaultComponentsTests = []struct {
 			{Kind: "clair", Managed: true},
 			{Kind: "objectstorage", Managed: false},
 			{Kind: "route", Managed: false},
+			{Kind: "tls", Managed: true},
+			{Kind: "horizontalpodautoscaler", Managed: true},
+			{Kind: "mirror", Managed: true},
+			{Kind: "monitoring", Managed: true},
+		},
+		nil,
+	},
+	{
+		"SomeComponentsProvidedWithTLS",
+		QuayRegistry{
+			Spec: QuayRegistrySpec{
+				Components: []Component{
+					{Kind: "postgres", Managed: false},
+					{Kind: "objectstorage", Managed: false},
+					{Kind: "route", Managed: false},
+				},
+			},
+		},
+		quaycontext.QuayRegistryContext{
+			SupportsRoutes:     true,
+			TLSCert:            []byte("my-own-cert"),
+			TLSKey:             []byte("my-own-key"),
+			SupportsMonitoring: true,
+			ClusterHostname:    "apps.example.com",
+		},
+		[]Component{
+			{Kind: "postgres", Managed: false},
+			{Kind: "redis", Managed: true},
+			{Kind: "clair", Managed: true},
+			{Kind: "objectstorage", Managed: false},
+			{Kind: "route", Managed: false},
+			{Kind: "tls", Managed: false},
 			{Kind: "horizontalpodautoscaler", Managed: true},
 			{Kind: "mirror", Managed: true},
 			{Kind: "monitoring", Managed: true},

controllers/quay/features.go

@@ -1,7 +1,10 @@
 package controllers
 
 import (
+	"bytes"
 	"context"
+	"crypto/tls"
+	"encoding/pem"
 	"fmt"
 	"strings"
 	"time"
@@ -73,35 +76,13 @@ func (r *QuayRegistryReconciler) checkManagedTLS(ctx *quaycontext.QuayRegistryCo
 	providedTLSKey := configBundle["ssl.key"]
 
 	if providedTLSCert != nil && providedTLSKey != nil {
-		r.Log.Info("provided TLS cert/key pair in `configBundleSecret` will be stored in persistent `Secret`")
+		r.Log.Info("provided TLS cert/key pair in `configBundleSecret` will be used")
 		ctx.TLSCert = providedTLSCert
 		ctx.TLSKey = providedTLSKey
 
 		return ctx, quay, nil
-	}
-
-	var secrets corev1.SecretList
-	listOptions := &client.ListOptions{
-		Namespace: quay.GetNamespace(),
-		LabelSelector: labels.SelectorFromSet(map[string]string{
-			kustomize.QuayRegistryNameLabel: quay.GetName(),
-		}),
-	}
-
-	if err := r.List(context.Background(), &secrets, listOptions); err != nil {
-		return ctx, quay, err
-	}
-
-	for _, secret := range secrets.Items {
-		if v1.IsManagedTLSSecretFor(quay, &secret) {
-			ctx.TLSCert = secret.Data["ssl.cert"]
-			ctx.TLSKey = secret.Data["ssl.key"]
-			break
-		}
-	}
-
-	if ctx.TLSCert == nil || ctx.TLSKey == nil {
-		r.Log.Info("existing TLS cert/key pair not found, one will be generated")
+	} else {
+		r.Log.Info("TLS cert/key pair not provided, will use default cluster wildcard cert")
 	}
 
 	return ctx, quay, nil
@@ -169,6 +150,16 @@ func (r *QuayRegistryReconciler) checkRoutesAvailable(ctx *quaycontext.QuayRegis
 
 		r.Log.Info("detected router canonical hostname: " + ctx.ClusterHostname)
 
+		// TODO(alecmerdler): Try to fetch the wildcard cert from the `ConfigMap` at `openshift-config-managed/default-ingress-cert`...
+		clusterWildcardCert, err := getCertificatesPEM(fakeRoute.(*routev1.Route).Spec.Host + ":443")
+		if err != nil {
+			return ctx, quay, err
+		}
+
+		ctx.ClusterWildcardCert = clusterWildcardCert
+
+		r.Log.Info("detected cluster wildcard certificate for " + ctx.ClusterHostname)
+
 		if err := r.Client.Delete(context.Background(), fakeRoute); err != nil {
 			return ctx, quay, err
 		}
@@ -302,3 +293,26 @@ func configEditorCredentialsSecretFrom(objs []client.Object) string {
 
 	return ""
 }
+
+// Taken from https://stackoverflow.com/questions/46735347/how-can-i-fetch-a-certificate-from-a-url
+func getCertificatesPEM(address string) ([]byte, error) {
+	conn, err := tls.Dial("tcp", address, &tls.Config{
+		InsecureSkipVerify: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	defer conn.Close()
+	var b bytes.Buffer
+	for _, cert := range conn.ConnectionState().PeerCertificates {
+		err := pem.Encode(&b, &pem.Block{
+			Type:  "CERTIFICATE",
+			Bytes: cert.Raw,
+		})
+		if err != nil {
+			return nil, err
+		}
+	}
+
+	return b.Bytes(), nil
+}

controllers/quay/quayregistry_controller.go

@@ -276,9 +276,9 @@ func (r *QuayRegistryReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	log.Info("inflating QuayRegistry into Kubernetes objects using Kustomize")
 	deploymentObjects, err := kustomize.Inflate(quayContext, updatedQuay, &configBundle, log)
 	if err != nil {
-		log.Error(err, "could not inflate QuayRegistry into Kubernetes objects")
+		msg := fmt.Sprintf("could not inflate QuayRegistry into Kubernetes objects: %s", err)
 
-		return ctrl.Result{}, nil
+		return r.reconcileWithCondition(&quay, v1.ConditionTypeRolloutBlocked, metav1.ConditionTrue, v1.ConditionReasonComponentCreationFailed, msg)
 	}
 
 	for _, obj := range kustomize.EnsureCreationOrder(deploymentObjects) {

controllers/quay/quayregistry_controller_test.go

@@ -185,7 +185,7 @@ var _ = Describe("Reconciling a QuayRegistry", func() {
 				To(Succeed())
 		})
 
-		It("should generate a self-signed TLS cert/key pair in a new `Secret`", func() {
+		It("should not generate a self-signed TLS cert/key pair in a new `Secret`", func() {
 			// Reconcile again to get past defaulting step
 			result, err = controller.Reconcile(context.Background(), reconcile.Request{NamespacedName: quayRegistryName})
 
@@ -203,8 +203,8 @@ var _ = Describe("Reconciling a QuayRegistry", func() {
 				if v1.IsManagedTLSSecretFor(quayRegistry, &secret) {
 					found = true
 
-					Expect(secret.Data).To(HaveKey("ssl.cert"))
-					Expect(secret.Data).To(HaveKey("ssl.key"))
+					Expect(secret.Data).NotTo(HaveKey("ssl.cert"))
+					Expect(secret.Data).NotTo(HaveKey("ssl.key"))
 				}
 			}
 

controllers/redhatcop/quayecosystem_controller.go

@@ -532,6 +532,10 @@ func (r *QuayEcosystemReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	if canHandleExternalAccess(quayEcosystem) {
 		log.Info("attempting to migrate external access", "type", quayEcosystem.Spec.Quay.ExternalAccess.Type)
 
+		// NOTE: Assume that if using edge `Route`, there is no custom TLS cert/key pair.
+		// Docs state that custom TLS can be provided to an edge `Route`: (https://access.redhat.com/documentation/en-us/red_hat_quay/3.3/html/deploy_red_hat_quay_on_openshift_with_quay_operator/customizing_your_red_hat_quay_cluster#user_provided_certificates)
+		// But this doesn't seem to be implemented: (https://sourcegraph.com/github.com/quay/quay-operator@v1/-/blob/pkg/controller/quayecosystem/resources/routes.go#L64).
+
 		for _, field := range (&hostsettings.HostSettingsFieldGroup{}).Fields() {
 			if field != "SERVER_HOSTNAME" {
 				delete(baseConfig, field)
@@ -709,7 +713,8 @@ func canHandleExternalAccess(q redhatcop.QuayEcosystem) bool {
 	return q.Spec.Quay.ExternalAccess != nil &&
 		q.Spec.Quay.ExternalAccess.Type == redhatcop.RouteExternalAccessType &&
 		q.Spec.Quay.ExternalAccess.TLS != nil &&
-		q.Spec.Quay.ExternalAccess.TLS.Termination == redhatcop.PassthroughTLSTerminationType
+		(q.Spec.Quay.ExternalAccess.TLS.Termination == redhatcop.PassthroughTLSTerminationType ||
+			q.Spec.Quay.ExternalAccess.TLS.Termination == redhatcop.EdgeTLSTerminationType)
 }
 
 func canHandleRedis(q redhatcop.QuayEcosystem) bool {

kustomize/base/config.deployment.yaml

@@ -16,12 +16,8 @@ spec:
     spec:
       volumes:
         - name: config
-          projected:
-            sources:
-            - secret:
-                name: quay-config-secret
-            - secret:
-                name: quay-config-tls
+          secret:
+            secretName: quay-config-secret
         - name: extra-ca-certs
           configMap:
             name: cluster-service-ca

kustomize/base/config.service.yaml

@@ -5,7 +5,7 @@ metadata:
   labels:
     quay-component: quay-config-editor
 spec:
-  type: LoadBalancer
+  type: ClusterIP
   ports:
     - protocol: TCP
       name: http