Merge pull request #8 from logicmonitor/DEV-118714-implement-functionality-to-exclude-metadata-from-lm-logs-logstash-plugin

Dev 118714 implement functionality to exclude metadata from lm logs logstash plugin

Author: choudharypooja

README.md

@@ -33,6 +33,7 @@ output {
 | keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
 | timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
 | timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
+| include_metadata  | If false, the metadata fields will not be sent to LM Logs  | true |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 

lib/logstash/outputs/lmlogs.rb

@@ -93,6 +93,9 @@ class InvalidHTTPConfigError < StandardError; end
   # Username to use for HTTP auth.
   config :access_id, :validate => :string, :required => true
 
+  # Include/Exclude metadata from sending to LM Logs
+  config :include_metadata, :validate => :boolean, :default => true
+
   # Password to use for HTTP auth
   config :access_key, :validate => :password, :required => true
 
@@ -258,12 +261,15 @@ def multi_receive(events)
       documents = []
       chunk.each do |event|
         event_json = JSON.parse(event.to_json)
-        lmlogs_event = event_json
-        lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
-        lmlogs_event["event"].delete("original") # remove redundant log field
+        lmlogs_event = {}
 
-        lmlogs_event["message"] = event.get(@message_key).to_s
+        if @include_metadata
+         lmlogs_event = event_json
+         lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+         lmlogs_event["event"].delete("original") # remove redundant log field
+        end
 
+        lmlogs_event["message"] = event.get(@message_key).to_s
         lmlogs_event["_lm.resourceId"] = {}
         lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
 

logstash-output-lmlogs.gemspec

@@ -1,6 +1,6 @@
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.1.0'
+  s.version         = '1.2.0'
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"