-
Notifications
You must be signed in to change notification settings - Fork 1.1k
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
Add complex scaling logic for custom formula & external scaling via grpc server #4583
Changes from all commits
09e7abe
7c9f1e1
29b6a79
3fb4c31
d96ca2e
7182fb8
7b4e66d
8daa27d
46db72d
247900e
34b9b21
6cdc876
f15b506
216857d
9cfc11d
7e6802a
5bdfb63
e6efd13
c4a26ab
68601ac
afa992c
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -102,6 +102,31 @@ type AdvancedConfig struct { | |
HorizontalPodAutoscalerConfig *HorizontalPodAutoscalerConfig `json:"horizontalPodAutoscalerConfig,omitempty"` | ||
// +optional | ||
RestoreToOriginalReplicaCount bool `json:"restoreToOriginalReplicaCount,omitempty"` | ||
// +optional | ||
ComplexScalingLogic ComplexScalingLogic `json:"complexScalingLogic,omitempty"` | ||
} | ||
|
||
// ComplexScalingLogic describes advanced scaling logic options like formula | ||
// and gRPC server for external calculations | ||
type ComplexScalingLogic struct { | ||
// +optional | ||
ExternalCalculations []ExternalCalculation `json:"externalCalculators,omitempty"` | ||
// +optional | ||
Formula string `json:"formula,omitempty"` | ||
// +optional | ||
Target string `json:"target,omitempty"` | ||
} | ||
|
||
// ExternalCalculation structure describes name and URL of a gRPC server | ||
// that KEDA can connect to with collected metrics and modify them. Each server | ||
// has a timeout and tls certification. If certDir is left empty, it will | ||
// connect with insecure.NewCredentials() | ||
type ExternalCalculation struct { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. this struct as well |
||
Name string `json:"name"` | ||
URL string `json:"url"` | ||
Timeout string `json:"timeout"` | ||
// +optional | ||
CertificateDirectory string `json:"certDir"` | ||
} | ||
|
||
// HorizontalPodAutoscalerConfig specifies horizontal scale config | ||
|
@@ -141,6 +166,10 @@ type ScaledObjectStatus struct { | |
// +optional | ||
ResourceMetricNames []string `json:"resourceMetricNames,omitempty"` | ||
// +optional | ||
CompositeScalerName string `json:"compositeScalerName,omitempty"` | ||
// +optional | ||
ExternalCalculationHealth map[string]HealthStatus `json:"externalCalculationHealth,omitempty"` | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. i created a new structure for grpc servers to have their own health status. Option number 2 could be to add them to already existing health status and prefix them with something like |
||
// +optional | ||
Conditions Conditions `json:"conditions,omitempty"` | ||
// +optional | ||
Health map[string]HealthStatus `json:"health,omitempty"` | ||
|
Original file line number | Diff line number | Diff line change |
---|---|---|
|
@@ -19,7 +19,11 @@ package v1alpha1 | |
import ( | ||
"context" | ||
"encoding/json" | ||
"errors" | ||
"fmt" | ||
"reflect" | ||
"strconv" | ||
"time" | ||
|
||
appsv1 "k8s.io/api/apps/v1" | ||
autoscalingv2 "k8s.io/api/autoscaling/v2" | ||
|
@@ -213,6 +217,16 @@ func verifyScaledObjects(incomingSo *ScaledObject, action string) error { | |
} | ||
} | ||
|
||
// verify ComplexScalingLogic structure if defined in ScaledObject | ||
if incomingSo.Spec.Advanced != nil && !reflect.DeepEqual(incomingSo.Spec.Advanced.ComplexScalingLogic, ComplexScalingLogic{}) { | ||
_, _, err = ValidateComplexScalingLogic(incomingSo, []autoscalingv2.MetricSpec{}) | ||
if err != nil { | ||
scaledobjectlog.Error(err, "error validating ComplexScalingLogic") | ||
prommetrics.RecordScaledObjectValidatingErrors(incomingSo.Namespace, action, "complex-scaling-logic") | ||
|
||
return err | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
|
@@ -297,3 +311,109 @@ func verifyCPUMemoryScalers(incomingSo *ScaledObject, action string) error { | |
} | ||
return nil | ||
} | ||
|
||
// ValidateComplexScalingLogic validates all combinations of given arguments | ||
// and their values | ||
func ValidateComplexScalingLogic(so *ScaledObject, specs []autoscalingv2.MetricSpec) (float64, autoscalingv2.MetricTargetType, error) { | ||
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. im not so sure these validation funcs should be in |
||
csl := so.Spec.Advanced.ComplexScalingLogic | ||
|
||
// if Formula AND ExternalCalculations is empty, return an error | ||
if csl.Formula == "" && len(csl.ExternalCalculations) < 1 { | ||
return -1, autoscalingv2.MetricTargetType(""), fmt.Errorf("error atleast one ComplexScalingLogic function needs to be specified (formula or externalCalculation)") | ||
} | ||
|
||
var num float64 | ||
var metricType autoscalingv2.MetricTargetType | ||
|
||
// validate formula if not empty | ||
if err := validateCSLformula(so); err != nil { | ||
err := errors.Join(fmt.Errorf("error validating formula in ComplexScalingLogic"), err) | ||
return -1, autoscalingv2.MetricTargetType(""), err | ||
} | ||
// validate externalCalculators if not empty | ||
if err := validateCSLexternalCalculations(csl); err != nil { | ||
err := errors.Join(fmt.Errorf("error validating externalCalculator in ComplexScalingLogic"), err) | ||
return -1, autoscalingv2.MetricTargetType(""), err | ||
} | ||
// validate target if not empty | ||
num, metricType, err := validateCSLtarget(csl, specs) | ||
if err != nil { | ||
err := errors.Join(fmt.Errorf("error validating target in ComplexScalingLogic"), err) | ||
return -1, autoscalingv2.MetricTargetType(""), err | ||
} | ||
return num, metricType, nil | ||
} | ||
|
||
func validateCSLformula(so *ScaledObject) error { | ||
csl := so.Spec.Advanced.ComplexScalingLogic | ||
|
||
// if formula is empty, nothing to validate | ||
if csl.Formula == "" { | ||
return nil | ||
} | ||
// formula needs target because it's always transformed to Composite scaler | ||
if csl.Target == "" { | ||
return fmt.Errorf("formula is given but target is empty") | ||
} | ||
|
||
// possible TODO: this could be more soffisticated - only check for names that | ||
// are used in the formula itself. This would require parsing the formula. | ||
for _, trig := range so.Spec.Triggers { | ||
if trig.Name == "" { | ||
return fmt.Errorf("trigger of type '%s' has empty name but csl.Formula is defined", trig.Type) | ||
} | ||
} | ||
if len(csl.ExternalCalculations) > 0 { | ||
if csl.ExternalCalculations[len(csl.ExternalCalculations)-1].Name == "" { | ||
return fmt.Errorf("last externalCalculator has empty name but csl.Formula is defined") | ||
} | ||
} | ||
return nil | ||
} | ||
|
||
func validateCSLexternalCalculations(cls ComplexScalingLogic) error { | ||
// timeout check | ||
for _, ec := range cls.ExternalCalculations { | ||
_, err := strconv.ParseInt(ec.Timeout, 10, 64) | ||
if err != nil { | ||
// expect timeout in time format like 1m10s | ||
_, err = time.ParseDuration(ec.Timeout) | ||
if err != nil { | ||
return fmt.Errorf("%s: error while converting type of timeout for external calculator", err) | ||
} | ||
} | ||
if ec.URL == "" { | ||
return fmt.Errorf("URL is empty for externalCalculator '%s'", ec.Name) | ||
} | ||
} | ||
|
||
return nil | ||
} | ||
|
||
func validateCSLtarget(csl ComplexScalingLogic, specs []autoscalingv2.MetricSpec) (float64, autoscalingv2.MetricTargetType, error) { | ||
if csl.Target == "" { | ||
return -1, "", nil | ||
} | ||
// convert string to float | ||
num, err := strconv.ParseFloat(csl.Target, 64) | ||
if err != nil || num <= 0.0 { | ||
return -1, "", fmt.Errorf("error converting target for complex logic (string->float) to valid target: %w", err) | ||
} | ||
|
||
var metricType autoscalingv2.MetricTargetType | ||
// if target is given, composite scaler for metric collection will be | ||
// passed to HPA config -> all types need to be the same | ||
// make sure all scalers have the same metricTargetType | ||
for _, metric := range specs { | ||
if metric.External == nil { | ||
continue | ||
} | ||
if metricType == "" { | ||
metricType = metric.External.Target.Type | ||
} else if metric.External.Target.Type != metricType { | ||
err := fmt.Errorf("error metric target type not the same for composite scaler: %s & %s", metricType, metric.External.Target.Type) | ||
return -1, "", err | ||
} | ||
} | ||
return num, metricType, nil | ||
} |
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
There was a point from @JorTurFer kedacore/keda-docs#1189 (comment) that this structure could use a better name - something like
modifiers
in yaml file. In such case i think it'd be good to change this one as well