Skip to content

Commit

Permalink
monitoring: improved coverage (PROJQUAY-3290)
Browse files Browse the repository at this point in the history 
Improved the coverage for Deployment health checks. We now take into
account possible overrides.
  • Loading branch information
@ricardomaraschini
ricardomaraschini committed Mar 18, 2022
1 parent fa9f24e commit f302223
Show file tree
Hide file tree
Showing 14 changed files with 576 additions and 56 deletions.
52 changes: 52 additions & 0 deletions apis/quay/v1/quayregistry_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -692,3 +692,55 @@ func ComponentSupportsOverride(component ComponentKind, override string) bool {
func init() {
SchemeBuilder.Register(&QuayRegistry{}, &QuayRegistryList{})
}

// GetReplicasOverrideForComponent returns the overrides set by the user for the provided
// component. Returns nil if not set.
func GetReplicasOverrideForComponent(quay *QuayRegistry, kind ComponentKind) *int32 {
for _, cmp := range quay.Spec.Components {
if cmp.Kind != kind {
continue
}

if cmp.Overrides == nil {
return nil
}

return cmp.Overrides.Replicas
}
return nil
}

// GetVolumeSizeOverrideForComponent returns the volume size overrides set by the user for the
// provided component. Returns nil if not set.
func GetVolumeSizeOverrideForComponent(
quay *QuayRegistry, kind ComponentKind,
) (qt *resource.Quantity) {
for _, component := range quay.Spec.Components {
if component.Kind != kind {
continue
}

if component.Overrides != nil && component.Overrides.VolumeSize != nil {
qt = component.Overrides.VolumeSize
}
return
}
return
}

// GetEnvOverrideForComponent return the environment variables overrides for the provided
// component, nil is returned if not defined.
func GetEnvOverrideForComponent(quay *QuayRegistry, kind ComponentKind) []corev1.EnvVar {
for _, cmp := range quay.Spec.Components {
if cmp.Kind != kind {
continue
}

if cmp.Overrides == nil {
return nil
}

return cmp.Overrides.Env
}
return nil
}
22 changes: 22 additions & 0 deletions pkg/cmpstatus/base.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,6 +31,11 @@ func (b *Base) Name() string {
func (b *Base) Check(ctx context.Context, reg qv1.QuayRegistry) (qv1.Condition, error) {
var zero qv1.Condition

// users are able to override the number of replicas. if they do override it to zero
// we expect zero replicas to be running.
replicas := qv1.GetReplicasOverrideForComponent(&reg, qv1.ComponentBase)
scaleddown := replicas != nil && *replicas == 0

// we need to check two distinct deployments, the quay app and its config editor.
for _, depsuffix := range []string{"quay-app", "quay-config-editor"} {
depname := fmt.Sprintf("%s-%s", reg.Name, depsuffix)
Expand Down Expand Up @@ -65,6 +70,23 @@ func (b *Base) Check(ctx context.Context, reg qv1.QuayRegistry) (qv1.Condition,
}, nil
}

basedep := fmt.Sprintf("%s-quay-app", reg.Name)
if dep.Name == basedep && scaleddown {
// if user has scaled down base component and we have zero replicas for
// the current deployment we are good to go, move to the next deployment.
if dep.Status.AvailableReplicas == 0 {
continue
}

return qv1.Condition{
Type: qv1.ComponentBaseReady,
Reason: qv1.ConditionReasonComponentNotReady,
Status: metav1.ConditionFalse,
Message: "Base component is being scaled down",
LastUpdateTime: metav1.NewTime(time.Now()),
}, nil
}

cond := b.deploy.check(dep)
if cond.Status != metav1.ConditionTrue {
// if the deployment is in a faulty state bails out immediately.
Expand Down
233 changes: 233 additions & 0 deletions pkg/cmpstatus/base_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ import (
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client/fake"

qv1 "github.com/quay/quay-operator/apis/quay/v1"
Expand Down Expand Up @@ -59,6 +60,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand Down Expand Up @@ -98,6 +100,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand Down Expand Up @@ -137,6 +140,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand All @@ -159,6 +163,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand Down Expand Up @@ -198,6 +203,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand All @@ -220,6 +226,7 @@ func TestBaseCheck(t *testing.T) {
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Expand Down Expand Up @@ -268,6 +275,232 @@ func TestBaseCheck(t *testing.T) {
Message: "Deployment registry-quay-app not owned by QuayRegistry",
},
},
{
name: "base component being scaled down",
quay: qv1.QuayRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "registry",
UID: "uid",
},
Spec: qv1.QuayRegistrySpec{
Components: []qv1.Component{
{
Kind: qv1.ComponentBase,
Overrides: &qv1.Override{
Replicas: pointer.Int32(0),
},
},
},
},
},
objs: []runtime.Object{
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-app",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 2,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
},
cond: qv1.Condition{
Type: qv1.ComponentBaseReady,
Reason: qv1.ConditionReasonComponentNotReady,
Status: metav1.ConditionFalse,
Message: "Base component is being scaled down",
},
},
{
name: "base component scaled down",
quay: qv1.QuayRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "registry",
UID: "uid",
},
Spec: qv1.QuayRegistrySpec{
Components: []qv1.Component{
{
Kind: qv1.ComponentBase,
Overrides: &qv1.Override{
Replicas: pointer.Int32(0),
},
},
},
},
},
objs: []runtime.Object{
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-app",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 0,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-config-editor",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
},
cond: qv1.Condition{
Type: qv1.ComponentBaseReady,
Reason: qv1.ConditionReasonComponentReady,
Status: metav1.ConditionTrue,
Message: "Base component healthy",
},
},
{
name: "zero available replicas quay app",
quay: qv1.QuayRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "registry",
UID: "uid",
},
},
objs: []runtime.Object{
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-app",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 0,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
},
cond: qv1.Condition{
Type: qv1.ComponentBaseReady,
Reason: qv1.ConditionReasonComponentNotReady,
Status: metav1.ConditionFalse,
Message: "Deployment registry-quay-app has zero replicas available",
},
},
{
name: "zero available replicas for config editor",
quay: qv1.QuayRegistry{
ObjectMeta: metav1.ObjectMeta{
Name: "registry",
UID: "uid",
},
},
objs: []runtime.Object{
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-app",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 1,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
&appsv1.Deployment{
ObjectMeta: metav1.ObjectMeta{
Name: "registry-quay-config-editor",
OwnerReferences: []metav1.OwnerReference{
{
Kind: "QuayRegistry",
Name: "registry",
APIVersion: "quay.redhat.com/v1",
UID: "uid",
},
},
},
Status: appsv1.DeploymentStatus{
AvailableReplicas: 0,
Conditions: []appsv1.DeploymentCondition{
{
Type: appsv1.DeploymentAvailable,
Status: corev1.ConditionTrue,
Message: "all good",
},
},
},
},
},
cond: qv1.Condition{
Type: qv1.ComponentBaseReady,
Reason: qv1.ConditionReasonComponentNotReady,
Status: metav1.ConditionFalse,
Message: "Deployment registry-quay-config-editor has zero replicas available",
},
},
} {
t.Run(tt.name, func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 2*time.Second)
Expand Down
Loading

0 comments on commit f302223

Please sign in to comment.