forked from elastic/beats
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
[Ingest Manager] Refuse invalid stream values in configuration (elast…
…ic#19587) * added stream check * changelog * fmt * invalid names * updated config files * updated config files * changes in all config files * config typo * small changes * Update go.sum
- Loading branch information
1 parent
a3ac5aa
commit 8f0ae3c
Showing
13 changed files
with
600 additions
and
10 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
199 changes: 199 additions & 0 deletions
199
x-pack/elastic-agent/pkg/agent/application/filters/stream_checker.go
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,199 @@ | ||
| // Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one | ||
| // or more contributor license agreements. Licensed under the Elastic License; | ||
| // you may not use this file except in compliance with the Elastic License. | ||
|
|
||
| package filters | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "strings" | ||
|
|
||
| "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/errors" | ||
| "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/agent/transpiler" | ||
| "github.com/elastic/beats/v7/x-pack/elastic-agent/pkg/core/logger" | ||
| ) | ||
|
|
||
| // ErrInvalidNamespace is error returned when namespace value provided is invalid. | ||
| var ErrInvalidNamespace = errors.New("provided namespace is invalid", errors.TypeConfig) | ||
|
|
||
| // ErrInvalidDataset is error returned when dataset name value provided is invalid. | ||
| var ErrInvalidDataset = errors.New("provided dataset name is invalid", errors.TypeConfig) | ||
|
|
||
| // ErrInvalidIndex occurs when concatenation of {dataset.type}-{dataset.name}-{dataset.namespace} does not meet index criteria. | ||
| var ErrInvalidIndex = errors.New("provided combination of type, dataset name and namespace is invalid", errors.TypeConfig) | ||
|
|
||
| // StreamChecker checks for invalid values in stream namespace and dataset. | ||
| func StreamChecker(log *logger.Logger, ast *transpiler.AST) error { | ||
| inputsNode, found := transpiler.Lookup(ast, "inputs") | ||
| if !found { | ||
| return nil | ||
| } | ||
|
|
||
| inputsNodeList, ok := inputsNode.Value().(*transpiler.List) | ||
| if !ok { | ||
| return nil | ||
| } | ||
|
|
||
| inputsNodeListCollection, ok := inputsNodeList.Value().([]transpiler.Node) | ||
| if !ok { | ||
| return errors.New("inputs is not a list", errors.TypeConfig) | ||
| } | ||
|
|
||
| for _, inputNode := range inputsNodeListCollection { | ||
| namespace := "default" | ||
| datasetName := "generic" | ||
| // fail only if dataset.namespace or dataset[namespace] is found and invalid | ||
| // not provided values are ok and will be fixed by rules | ||
| if nsNode, found := inputNode.Find("dataset.namespace"); found { | ||
| nsKey, ok := nsNode.(*transpiler.Key) | ||
| if ok { | ||
| newNamespace := nsKey.Value().(transpiler.Node).String() | ||
| if !isValid(newNamespace) { | ||
| return ErrInvalidNamespace | ||
| } | ||
| namespace = newNamespace | ||
| } | ||
| } else { | ||
| dsNode, found := inputNode.Find("dataset") | ||
| if found { | ||
| // got a dataset | ||
| datasetMap, ok := dsNode.Value().(*transpiler.Dict) | ||
| if ok { | ||
| nsNode, found := datasetMap.Find("namespace") | ||
| if found { | ||
| nsKey, ok := nsNode.(*transpiler.Key) | ||
| if ok { | ||
| newNamespace := nsKey.Value().(transpiler.Node).String() | ||
| if !isValid(newNamespace) { | ||
| return ErrInvalidNamespace | ||
| } | ||
| namespace = newNamespace | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // get the type, longest type for now is metrics | ||
| datasetType := "metrics" | ||
| if nsNode, found := inputNode.Find("dataset.type"); found { | ||
| nsKey, ok := nsNode.(*transpiler.Key) | ||
| if ok { | ||
| newDataset := nsKey.Value().(transpiler.Node).String() | ||
| datasetType = newDataset | ||
| } | ||
| } else { | ||
| dsNode, found := inputNode.Find("dataset") | ||
| if found { | ||
| // got a dataset | ||
| datasetMap, ok := dsNode.Value().(*transpiler.Dict) | ||
| if ok { | ||
| nsNode, found := datasetMap.Find("type") | ||
| if found { | ||
| nsKey, ok := nsNode.(*transpiler.Key) | ||
| if ok { | ||
| newDataset := nsKey.Value().(transpiler.Node).String() | ||
| datasetType = newDataset | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| streamsNode, ok := inputNode.Find("streams") | ||
| if ok { | ||
| streamsList, ok := streamsNode.Value().(*transpiler.List) | ||
| if ok { | ||
| streamNodes, ok := streamsList.Value().([]transpiler.Node) | ||
| if !ok { | ||
| return errors.New("streams is not a list", errors.TypeConfig) | ||
| } | ||
|
|
||
| for _, streamNode := range streamNodes { | ||
| streamMap, ok := streamNode.(*transpiler.Dict) | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| // fix this only if in compact form | ||
| if dsNameNode, found := streamMap.Find("dataset.name"); found { | ||
| dsKey, ok := dsNameNode.(*transpiler.Key) | ||
| if ok { | ||
| newDataset := dsKey.Value().(transpiler.Node).String() | ||
| if !isValid(newDataset) { | ||
| return ErrInvalidDataset | ||
| } | ||
| datasetName = newDataset | ||
| } | ||
| } else { | ||
| datasetNode, found := streamMap.Find("dataset") | ||
| if found { | ||
| datasetMap, ok := datasetNode.Value().(*transpiler.Dict) | ||
| if !ok { | ||
| continue | ||
| } | ||
|
|
||
| dsNameNode, found := datasetMap.Find("name") | ||
| if found { | ||
| dsKey, ok := dsNameNode.(*transpiler.Key) | ||
| if ok { | ||
| newDataset := dsKey.Value().(transpiler.Node).String() | ||
| if !isValid(newDataset) { | ||
| return ErrInvalidDataset | ||
| } | ||
| datasetName = newDataset | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
| } | ||
|
|
||
| if indexName := fmt.Sprintf("%s-%s-%s", datasetType, datasetName, namespace); !matchesIndexContraints(indexName) { | ||
| return ErrInvalidIndex | ||
| } | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // The only two requirement are that it has only characters allowed in an Elasticsearch index name | ||
| // and does NOT contain a `-`. | ||
| func isValid(namespace string) bool { | ||
| return matchesIndexContraints(namespace) && !strings.Contains(namespace, "-") | ||
| } | ||
|
|
||
| // The only two requirement are that it has only characters allowed in an Elasticsearch index name | ||
| // Index names must meet the following criteria: | ||
| // Lowercase only | ||
| // Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # | ||
| // Cannot start with -, _, + | ||
| // Cannot be . or .. | ||
| func matchesIndexContraints(namespace string) bool { | ||
| // Cannot be . or .. | ||
| if namespace == "." || namespace == ".." { | ||
| return false | ||
| } | ||
|
|
||
| if len(namespace) <= 0 || len(namespace) > 255 { | ||
| return false | ||
| } | ||
|
|
||
| // Lowercase only | ||
| if strings.ToLower(namespace) != namespace { | ||
| return false | ||
| } | ||
|
|
||
| // Cannot include \, /, *, ?, ", <, >, |, ` ` (space character), ,, # | ||
| if strings.ContainsAny(namespace, "\\/*?\"<>| ,#") { | ||
| return false | ||
| } | ||
|
|
||
| // Cannot start with -, _, + | ||
| if strings.HasPrefix(namespace, "-") || strings.HasPrefix(namespace, "_") || strings.HasPrefix(namespace, "+") { | ||
| return false | ||
| } | ||
|
|
||
| return true | ||
| } |
Oops, something went wrong.