From 3deec1c1b2522037f48ecfae39873f838c0e0c42 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Sat, 9 Apr 2022 22:59:54 -0700 Subject: [PATCH 1/2] Support filter by severity --- CHANGELOG.md | 1 + .../processor/filterconfig/config.go | 12 ++++++++++-- .../processor/filterlog/filterlog.go | 14 ++++++++++++++ .../processor/filterlog/filterlog_test.go | 13 +++++++++++-- processor/attributesprocessor/README.md | 11 ++++++++--- .../attributesprocessor/testdata/config.yaml | 17 +++++++++++++++++ 6 files changed, 61 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0f482a469b27..4c3c01ff049e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ - `datadogexporter`: Add `host_metadata` configuration section to configure host metadata export (#9100) - `cmd/mdatagen`: Update documentation generated for attributes to list enumerated values and show the "value" that will be visible on metrics when it is different from the attribute key in metadata.yaml (#8983) +- `attributesprocessor`: Support filter by severity (#9132) ### 🛑 Breaking changes 🛑 diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index 04c4f1e86a58..8cd57be64ee4 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -94,6 +94,10 @@ type MatchProperties struct { // against. LogBodies []string `mapstructure:"log_bodies"` + // LogSeverities is a list of strings that the LogRecord's severity text field must match + // against. + LogSeverities []string `mapstructure:"log_severities"` + // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. // This field is optional. @@ -122,6 +126,10 @@ func (mp *MatchProperties) ValidateForSpans() error { return errors.New("log_bodies should not be specified for trace spans") } + if len(mp.LogSeverities) > 0 { + return errors.New("log_severities should not be specified for trace spans") + } + if len(mp.Services) == 0 && len(mp.SpanNames) == 0 && len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 { return errors.New(`at least one of "services", "span_names", "attributes", "libraries" or "resources" field must be specified`) @@ -136,8 +144,8 @@ func (mp *MatchProperties) ValidateForLogs() error { return errors.New("neither services nor span_names should be specified for log records") } - if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 { - return errors.New(`at least one of "attributes", "libraries", "resources" or "log_bodies" field must be specified`) + if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && len(mp.LogSeverities) == 0 { + return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`) } return nil diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index 65cd246146b6..3a71b76fd96d 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -38,6 +38,9 @@ type propertiesMatcher struct { // log bodies to compare to. bodyFilters filterset.FilterSet + + // log severities to compare to + severityFilters filterset.FilterSet } // NewMatcher creates a LogRecord Matcher that matches based on the given MatchProperties. @@ -62,10 +65,18 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { return nil, fmt.Errorf("error creating log record body filters: %v", err) } } + var severityFS filterset.FilterSet + if len(mp.LogSeverities) > 0 { + severityFS, err = filterset.CreateFilterSet(mp.LogSeverities, &mp.Config) + if err != nil { + return nil, fmt.Errorf("error creating log record severity filters: %v", err) + } + } return &propertiesMatcher{ PropertiesMatcher: rm, bodyFilters: bodyFS, + severityFilters: severityFS, }, nil } @@ -81,6 +92,9 @@ func (mp *propertiesMatcher) MatchLogRecord(lr pdata.LogRecord, resource pdata.R if lr.Body().Type() == pdata.ValueTypeString && mp.bodyFilters != nil && mp.bodyFilters.Matches(lr.Body().StringVal()) { return true } + if mp.severityFilters != nil && mp.severityFilters.Matches(lr.SeverityText()) { + return true + } return mp.PropertiesMatcher.Match(lr.Attributes(), resource, library) } diff --git a/internal/coreinternal/processor/filterlog/filterlog_test.go b/internal/coreinternal/processor/filterlog/filterlog_test.go index db6f8c243add..99a4546dad64 100644 --- a/internal/coreinternal/processor/filterlog/filterlog_test.go +++ b/internal/coreinternal/processor/filterlog/filterlog_test.go @@ -40,14 +40,15 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { { name: "empty_property", property: filterconfig.MatchProperties{}, - errorString: `at least one of "attributes", "libraries", "resources" or "log_bodies" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`, }, { name: "empty_log_bodies_and_attributes", property: filterconfig.MatchProperties{ LogBodies: []string{}, + LogSeverities: []string{}, }, - errorString: `at least one of "attributes", "libraries", "resources" or "log_bodies" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`, }, { name: "span_properties", @@ -156,11 +157,19 @@ func TestLogRecord_Matching_True(t *testing.T) { LogBodies: []string{"AUTH.*"}, }, }, + { + name: "log_severity_regexp_match", + properties: &filterconfig.MatchProperties{ + Config: *createConfig(filterset.Regexp), + LogSeverities: []string{"debug.*"}, + }, + }, } lr := pdata.NewLogRecord() lr.Attributes().InsertString("abc", "def") lr.Body().SetStringVal("AUTHENTICATION FAILED") + lr.SetSeverityText("debug") for _, tc := range testcases { t.Run(tc.name, func(t *testing.T) { diff --git a/processor/attributesprocessor/README.md b/processor/attributesprocessor/README.md index 74715aed1dd7..1b14ecf581e3 100644 --- a/processor/attributesprocessor/README.md +++ b/processor/attributesprocessor/README.md @@ -166,13 +166,14 @@ if the input data should be included or excluded from the processor. To configur this option, under `include` and/or `exclude` at least `match_type` and one of the following is required: - For spans, one of `services`, `span_names`, `attributes`, `resources`, or `libraries` must be specified -with a non-empty value for a valid configuration. The `log_bodies`, `expressions`, `resource_attributes` and + +with a non-empty value for a valid configuration. The `log_bodies`, `log_severities`, `expressions`, `resource_attributes` and `metric_names` fields are invalid. -- For logs, one of `log_bodies`, `attributes`, `resources`, or `libraries` must be specified with a +- For logs, one of `log_bodies`, `log_severities`, `attributes`, `resources`, or `libraries` must be specified with a non-empty value for a valid configuration. The `span_names`, `metric_names`, `expressions`, `resource_attributes`, and `services` fields are invalid. - For metrics, one of `metric_names`, `resources` must be specified -with a valid non-empty value for a valid configuration. The `span_names`, `log_bodies` and +with a valid non-empty value for a valid configuration. The `span_names`, `log_bodies`, `log_severities` and `services` fields are invalid. @@ -219,6 +220,10 @@ attributes: # This is an optional field. log_bodies: [, ..., ] + # The log severity text must match at least one of the items. + # This is an optional field. + log_severities: [, ..., ] + # The metric name must match at least one of the items. # This is an optional field. metric_names: [, ..., ] diff --git a/processor/attributesprocessor/testdata/config.yaml b/processor/attributesprocessor/testdata/config.yaml index 2c68eb918de6..a3c11646b06b 100644 --- a/processor/attributesprocessor/testdata/config.yaml +++ b/processor/attributesprocessor/testdata/config.yaml @@ -325,6 +325,23 @@ processors: - key: token action: delete + # The following demonstrates how to process logs that have a severity text that match regexp + # patterns. This processor will remove "token" attribute and will obfuscate "password" + # attribute in spans where severity matches "debug". + attributes/log_severity_regexp: + # Specifies the span properties that must exist for the processor to be applied. + include: + # match_type defines that "services" is an array of regexp-es. + match_type: regexp + # The log severity "debug.*" pattern. + log_severities: [ "debug.*" ] + actions: + - key: password + action: update + value: "obfuscated" + - key: token + action: delete + receivers: nop: From 742853ca0bf7ea934e79d4f4e3d8256fa5b4bf07 Mon Sep 17 00:00:00 2001 From: Antoine Toulme Date: Mon, 11 Apr 2022 14:30:30 -0700 Subject: [PATCH 2/2] code review changes - use specific language for severity *text* filter --- .../processor/filterconfig/config.go | 12 +++++------ .../processor/filterlog/filterlog.go | 20 +++++++++---------- .../processor/filterlog/filterlog_test.go | 14 ++++++------- processor/attributesprocessor/README.md | 9 ++++----- .../attributesprocessor/testdata/config.yaml | 4 ++-- 5 files changed, 29 insertions(+), 30 deletions(-) diff --git a/internal/coreinternal/processor/filterconfig/config.go b/internal/coreinternal/processor/filterconfig/config.go index 8cd57be64ee4..abdefe5b28a5 100644 --- a/internal/coreinternal/processor/filterconfig/config.go +++ b/internal/coreinternal/processor/filterconfig/config.go @@ -94,9 +94,9 @@ type MatchProperties struct { // against. LogBodies []string `mapstructure:"log_bodies"` - // LogSeverities is a list of strings that the LogRecord's severity text field must match + // LogSeverityTexts is a list of strings that the LogRecord's severity text field must match // against. - LogSeverities []string `mapstructure:"log_severities"` + LogSeverityTexts []string `mapstructure:"log_severity_texts"` // MetricNames is a list of strings to match metric name against. // A match occurs if metric name matches at least one item in the list. @@ -126,8 +126,8 @@ func (mp *MatchProperties) ValidateForSpans() error { return errors.New("log_bodies should not be specified for trace spans") } - if len(mp.LogSeverities) > 0 { - return errors.New("log_severities should not be specified for trace spans") + if len(mp.LogSeverityTexts) > 0 { + return errors.New("log_severity_texts should not be specified for trace spans") } if len(mp.Services) == 0 && len(mp.SpanNames) == 0 && len(mp.Attributes) == 0 && @@ -144,8 +144,8 @@ func (mp *MatchProperties) ValidateForLogs() error { return errors.New("neither services nor span_names should be specified for log records") } - if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && len(mp.LogSeverities) == 0 { - return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`) + if len(mp.Attributes) == 0 && len(mp.Libraries) == 0 && len(mp.Resources) == 0 && len(mp.LogBodies) == 0 && len(mp.LogSeverityTexts) == 0 { + return errors.New(`at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`) } return nil diff --git a/internal/coreinternal/processor/filterlog/filterlog.go b/internal/coreinternal/processor/filterlog/filterlog.go index 3a71b76fd96d..8ae7812f25ed 100644 --- a/internal/coreinternal/processor/filterlog/filterlog.go +++ b/internal/coreinternal/processor/filterlog/filterlog.go @@ -39,8 +39,8 @@ type propertiesMatcher struct { // log bodies to compare to. bodyFilters filterset.FilterSet - // log severities to compare to - severityFilters filterset.FilterSet + // log severity texts to compare to + severityTextFilters filterset.FilterSet } // NewMatcher creates a LogRecord Matcher that matches based on the given MatchProperties. @@ -65,18 +65,18 @@ func NewMatcher(mp *filterconfig.MatchProperties) (Matcher, error) { return nil, fmt.Errorf("error creating log record body filters: %v", err) } } - var severityFS filterset.FilterSet - if len(mp.LogSeverities) > 0 { - severityFS, err = filterset.CreateFilterSet(mp.LogSeverities, &mp.Config) + var severitytextFS filterset.FilterSet + if len(mp.LogSeverityTexts) > 0 { + severitytextFS, err = filterset.CreateFilterSet(mp.LogSeverityTexts, &mp.Config) if err != nil { - return nil, fmt.Errorf("error creating log record severity filters: %v", err) + return nil, fmt.Errorf("error creating log record severity text filters: %v", err) } } return &propertiesMatcher{ - PropertiesMatcher: rm, - bodyFilters: bodyFS, - severityFilters: severityFS, + PropertiesMatcher: rm, + bodyFilters: bodyFS, + severityTextFilters: severitytextFS, }, nil } @@ -92,7 +92,7 @@ func (mp *propertiesMatcher) MatchLogRecord(lr pdata.LogRecord, resource pdata.R if lr.Body().Type() == pdata.ValueTypeString && mp.bodyFilters != nil && mp.bodyFilters.Matches(lr.Body().StringVal()) { return true } - if mp.severityFilters != nil && mp.severityFilters.Matches(lr.SeverityText()) { + if mp.severityTextFilters != nil && mp.severityTextFilters.Matches(lr.SeverityText()) { return true } diff --git a/internal/coreinternal/processor/filterlog/filterlog_test.go b/internal/coreinternal/processor/filterlog/filterlog_test.go index 99a4546dad64..30f7f05a5546 100644 --- a/internal/coreinternal/processor/filterlog/filterlog_test.go +++ b/internal/coreinternal/processor/filterlog/filterlog_test.go @@ -40,15 +40,15 @@ func TestLogRecord_validateMatchesConfiguration_InvalidConfig(t *testing.T) { { name: "empty_property", property: filterconfig.MatchProperties{}, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`, }, { name: "empty_log_bodies_and_attributes", property: filterconfig.MatchProperties{ - LogBodies: []string{}, - LogSeverities: []string{}, + LogBodies: []string{}, + LogSeverityTexts: []string{}, }, - errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severities" field must be specified`, + errorString: `at least one of "attributes", "libraries", "resources", "log_bodies" or "log_severity_texts" field must be specified`, }, { name: "span_properties", @@ -158,10 +158,10 @@ func TestLogRecord_Matching_True(t *testing.T) { }, }, { - name: "log_severity_regexp_match", + name: "log_severity_text_regexp_match", properties: &filterconfig.MatchProperties{ - Config: *createConfig(filterset.Regexp), - LogSeverities: []string{"debug.*"}, + Config: *createConfig(filterset.Regexp), + LogSeverityTexts: []string{"debug.*"}, }, }, } diff --git a/processor/attributesprocessor/README.md b/processor/attributesprocessor/README.md index 1b14ecf581e3..1d9ef756316d 100644 --- a/processor/attributesprocessor/README.md +++ b/processor/attributesprocessor/README.md @@ -166,14 +166,13 @@ if the input data should be included or excluded from the processor. To configur this option, under `include` and/or `exclude` at least `match_type` and one of the following is required: - For spans, one of `services`, `span_names`, `attributes`, `resources`, or `libraries` must be specified - -with a non-empty value for a valid configuration. The `log_bodies`, `log_severities`, `expressions`, `resource_attributes` and +with a non-empty value for a valid configuration. The `log_bodies`, `log_severity_texts`, `expressions`, `resource_attributes` and `metric_names` fields are invalid. -- For logs, one of `log_bodies`, `log_severities`, `attributes`, `resources`, or `libraries` must be specified with a +- For logs, one of `log_bodies`, `log_severity_texts`, `attributes`, `resources`, or `libraries` must be specified with a non-empty value for a valid configuration. The `span_names`, `metric_names`, `expressions`, `resource_attributes`, and `services` fields are invalid. - For metrics, one of `metric_names`, `resources` must be specified -with a valid non-empty value for a valid configuration. The `span_names`, `log_bodies`, `log_severities` and +with a valid non-empty value for a valid configuration. The `span_names`, `log_bodies`, `log_severity_texts` and `services` fields are invalid. @@ -222,7 +221,7 @@ attributes: # The log severity text must match at least one of the items. # This is an optional field. - log_severities: [, ..., ] + log_severity_texts: [, ..., ] # The metric name must match at least one of the items. # This is an optional field. diff --git a/processor/attributesprocessor/testdata/config.yaml b/processor/attributesprocessor/testdata/config.yaml index a3c11646b06b..2dfeee9bc0f1 100644 --- a/processor/attributesprocessor/testdata/config.yaml +++ b/processor/attributesprocessor/testdata/config.yaml @@ -333,8 +333,8 @@ processors: include: # match_type defines that "services" is an array of regexp-es. match_type: regexp - # The log severity "debug.*" pattern. - log_severities: [ "debug.*" ] + # The log severity text "debug.*" pattern. + log_severity_texts: [ "debug.*" ] actions: - key: password action: update