diff --git a/CHANGELOG.md b/CHANGELOG.md index c94f9dd34..3e531725d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -20,6 +20,7 @@ CHANGELOG #### Parsers - `intelmq.bots.parsers.shadowserver.config`: - Parse and harmonize field `end_time` as date in Feeds "Drone-Brute-Force" and "Amplification-DDoS-Victim" (PR#1833 by Mikk Margus Möll). + - Add conversion function `convert_date_utc` which assumes UTC and sanitizes the data to datetime (by Sebastian Wagner, fixes #1848). - `intelmq.bots.parsers.shadowserver.parser_json`: - Use the overwrite parameter for optionally overwriting the "feed.name" field (by Sebastian Wagner). - `intelmq.bots.parsers.microsoft.parser_ctip`: diff --git a/intelmq/bots/parsers/shadowserver/config.py b/intelmq/bots/parsers/shadowserver/config.py index 4571e28ae..9564f6c5b 100644 --- a/intelmq/bots/parsers/shadowserver/config.py +++ b/intelmq/bots/parsers/shadowserver/config.py @@ -214,6 +214,14 @@ def convert_date(value): return harmonization.DateTime.sanitize(value) +def convert_date_utc(value): + """ + Parses a datetime from the value and assumes UTC by appending the TZ to the value. + Not the same as add_UTC_to_timestamp, as convert_date_utc also does the sanitiation + """ + return harmonization.DateTime.sanitize(value + '+00:00') + + # https://www.shadowserver.org/wiki/pmwiki.php/Services/Open-DB2 open_db2_discovery_service = { 'required_fields': [ @@ -1769,7 +1777,7 @@ def convert_date(value): ('extra.destination.sector', 'dest_sector', validate_to_none), ('extra.', 'public_source', validate_to_none), ('extra.', 'start_time', validate_to_none), - ('extra.', 'end_time', convert_date), + ('extra.', 'end_time', convert_date_utc), ('extra.', 'client_version', validate_to_none), ('destination.account', 'username', validate_to_none), ('extra.', 'password', validate_to_none), @@ -2003,7 +2011,7 @@ def convert_date(value): ('extra.', 'bytes', convert_int), ('extra.', 'sensor_geo', validate_to_none), ('extra.', 'sector', validate_to_none), - ('extra.', 'end_time', convert_date), + ('extra.', 'end_time', convert_date_utc), ('extra.', 'public_source', validate_to_none), ], 'constant_fields': {