Skip to content

Commit

Permalink
Fix/eval null (#172)
Browse files Browse the repository at this point in the history 
* [fix] Handle situations where a TRANSFORMS REGEX uses _VAL

Handle more complex syntax of regex to capture only valid named capture groups

* Fix test case

The purpose of this test is to validate we don't make a bad search

* Don't test for null() eval fields

When generating tests if a field is EVAL-field=null() don't check for field
  • Loading branch information
Ryan Faircloth committed Aug 23, 2020
1 parent 6b625e6 commit 6738310
Showing 1 changed file with 30 additions and 29 deletions.
59 changes: 30 additions & 29 deletions pytest_splunk_addon/standard_lib/addon_parser/props_parser.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,14 @@
Provides props.conf parsing mechanism
"""
import logging
import re
import re
from itertools import product
from . import convert_to_fields, Field
from . import TransformsParser

LOGGER = logging.getLogger("pytest-splunk-addon")


class PropsParser(object):
"""
Parses props.conf and extracts the fields.
Expand All @@ -20,7 +21,7 @@ class PropsParser(object):
"""

def __init__(self, splunk_app_path, app):
self.app = app
self.app = app
self.splunk_app_path = splunk_app_path
self._props = None
self.transforms_parser = TransformsParser(self.splunk_app_path, self.app)
Expand All @@ -46,9 +47,7 @@ def get_props_fields(self):
for stanza_type, stanza_name, stanza in self.get_props_stanzas():
for classname in stanza.options:
LOGGER.info(
"Parsing parameter=%s of stanza=%s",
classname,
stanza_name,
"Parsing parameter=%s of stanza=%s", classname, stanza_name,
)
props_property = stanza.options[classname]
if not re.match("REPORT", classname, re.IGNORECASE):
Expand All @@ -61,17 +60,19 @@ def get_props_fields(self):
"stanza": stanza_name,
"stanza_type": stanza_type,
"classname": classname,
"fields": field_list
"fields": field_list,
}
else:
for transform_stanza, fields in self.get_report_fields(props_property):
for transform_stanza, fields in self.get_report_fields(
props_property
):
field_list = list(fields)
if field_list:
yield {
"stanza": stanza_name,
"stanza_type": stanza_type,
"classname": f"{classname}::{transform_stanza}",
"fields": field_list
"fields": field_list,
}

def get_props_method(self, class_name):
Expand All @@ -88,7 +89,7 @@ def get_props_method(self, class_name):
"EXTRACT": self.get_extract_fields,
"EVAL": self.get_eval_fields,
"FIELDALIAS": self.get_fieldalias_fields,
"LOOKUP": self.get_lookup_fields
"LOOKUP": self.get_lookup_fields,
}
for each_type in method_mapping:
if re.match(each_type, class_name, re.IGNORECASE):
Expand Down Expand Up @@ -156,7 +157,6 @@ def get_list_of_sources(source):
yield template.format(*each_permutation)
LOGGER.debug("Found %d combinations", count)


def get_sourcetype_assignments(self, props_property):
"""
Get the sourcetype assigned for the source
Expand All @@ -177,10 +177,9 @@ def get_sourcetype_assignments(self, props_property):
Yields:
the sourcetype field with possible value
"""
yield Field({
"name": props_property.name,
"expected_values": [props_property.value]
})
yield Field(
{"name": props_property.name, "expected_values": [props_property.value]}
)

@convert_to_fields
def get_extract_fields(self, props_property):
Expand Down Expand Up @@ -226,7 +225,6 @@ def get_extract_fields(self, props_property):
yield extract_source_key.group(1)
fields_group.insert(0, extract_source_key.group(1))


@convert_to_fields
def get_eval_fields(self, props_property):
"""
Expand All @@ -248,8 +246,8 @@ def get_eval_fields(self, props_property):
generator of fields
"""
regex = r"EVAL-(?P<FIELD>.*)"
yield from re.findall(regex, props_property.name, re.IGNORECASE)

if not props_property.value == "null()":
yield from re.findall(regex, props_property.name, re.IGNORECASE)

@convert_to_fields
def get_fieldalias_fields(self, props_property):
Expand Down Expand Up @@ -292,7 +290,6 @@ def get_fieldalias_fields(self, props_property):
# Convert list of tuples into list
return list(set([item for t in fields_tuples for item in t]))


def get_report_fields(self, props_property):
"""
Returns the fields parsed from REPORT
Expand All @@ -313,13 +310,14 @@ def get_report_fields(self, props_property):
generator of (transform_stanza ,fields) parsed from transforms.conf
"""

transforms_itr = (each_stanza.strip() for each_stanza in props_property.value.split(","))
transforms_itr = (
each_stanza.strip() for each_stanza in props_property.value.split(",")
)
for transforms_section in transforms_itr:
yield (
transforms_section,
self.transforms_parser.get_transform_fields(transforms_section)
)

transforms_section,
self.transforms_parser.get_transform_fields(transforms_section),
)

@convert_to_fields
def get_lookup_fields(self, props_property):
Expand All @@ -338,19 +336,23 @@ def get_lookup_fields(self, props_property):
List of lookup fields
"""
parsed_fields = self.parse_lookup_str(props_property.value)
lookup_field_list = parsed_fields["input_fields"] + parsed_fields["output_fields"]
lookup_field_list = (
parsed_fields["input_fields"] + parsed_fields["output_fields"]
)

# If the OUTPUT or OUTPUTNEW argument is never used, then get the fields from the csv file
if not parsed_fields["output_fields"]:
LOGGER.info("OUTPUT fields not found classname=%s. Parsing the lookup csv file",
props_property.name
LOGGER.info(
"OUTPUT fields not found classname=%s. Parsing the lookup csv file",
props_property.name,
)
lookup_field_list += list(
self.transforms_parser.get_lookup_csv_fields(parsed_fields["lookup_stanza"])
self.transforms_parser.get_lookup_csv_fields(
parsed_fields["lookup_stanza"]
)
)
return list(set(lookup_field_list))


def parse_lookup_str(self, lookup_str):
"""
Get list of lookup fields by parsing the lookup string.
Expand Down Expand Up @@ -410,4 +412,3 @@ def parse_lookup_str(self, lookup_str):
"lookup_stanza": lookup_stanza,
}


0 comments on commit 6738310

Please sign in to comment.