Merge pull request #13 from logicmonitor/Add-metadata-limit-and-change-user-agent

Add user agent as lm-logs-logstash/{plugin-version}.
Do not send event metadata by default.
Add config include_metadata_keys.

Author: siddharthck

CHANGELOG.md

@@ -7,8 +7,12 @@
 ## 1.1.0
 - Disable mfa for gem push
 ## 1.2.0
-- Add metadata by default to every log 
+- Add metadata by default to every log
 ## 1.2.1
 - Fix ensuring metadata exists before deleting the original while forwarding to lm-logs
 ## 1.3.0
-- Add bearer token support for authentication with Logicmonitor
+- Add bearer token support for authentication with Logicmonitor
+## 1.3.1
+- Fix user agent populated in headers
+## 2.0.0
+- Dont send event metadata by default. Add config include_metadata_keys for including custom metadata

README.md

@@ -1,6 +1,6 @@
 [![Gem Version](https://badge.fury.io/rb/logstash-output-lmlogs.svg)](https://badge.fury.io/rb/logstash-output-lmlogs)
 
-This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com) 
+This plugin sends Logstash events to the [Logicmonitor Logs](https://www.logicmonitor.com)
 
 # Getting started
 
@@ -28,27 +28,28 @@ You would need either `access_id` and `access_id` both or `bearer_token` for aut
 
 | Option | Description| Default |
 | --- | --- | --- |
-| batch_size | Event batch size to send to LM Logs.| 100 | 
+| batch_size | Event batch size to send to LM Logs.| 100 |
 | message_key | Key that will be used by the plugin as the system key | "message" |
 | lm_property | Key that will be used by LM to match resource based on property | "system.hostname" |
 | keep_timestamp | If false, LM Logs will use the ingestion timestamp as the event timestamp | true |
 | timestamp_is_key  | If true, LM Logs will use a specified key as the event timestamp | false |
 | timestamp_key | If timestamp_is_key is set, LM Logs will use this key in the event as the timestamp | "logtimestamp"  |
-| include_metadata  | If false, the metadata fields will not be sent to LM Logs  | true |
+| include_metadata  | If true, all metadata fields will be sent to LM Logs  | false |
+| include_metadata_keys  | Array of json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson. If config `include_metadata` is set to true, all metadata will be sent regardless of this config. | [] |
 
 See the [source code](lib/logstash/outputs/lmlogs.rb) for the full list of options
 
 The syntax for `message_key` and `source_key` values are available in the [Logstash Event API Documentation](https://www.elastic.co/guide/en/logstash/current/event-api.html)
 
-## Known issues 
+## Known issues
  - Installation of the plugin fails on Logstash 6.2.1.
- 
- 
+
+
  ## Contributing
- 
+
  Bug reports and pull requests are welcome. This project is intended to
  be a safe, welcoming space for collaboration.
- 
+
  ## Development
- 
+
 We use docker to build the plugin. You can build it by running  `docker-compose run jruby gem build logstash-output-lmlogs.gemspec `

lib/logstash/outputs/lmlogs.rb

@@ -8,6 +8,7 @@
 require 'base64'
 require 'openssl'
 require 'manticore'
+require_relative "version"
 
 # An example output that does nothing.
 class LogStash::Outputs::LMLogs < LogStash::Outputs::Base
@@ -94,14 +95,17 @@ class InvalidHTTPConfigError < StandardError; end
   config :access_id, :validate => :string, :required => false, :default => nil
 
   # Include/Exclude metadata from sending to LM Logs
-  config :include_metadata, :validate => :boolean, :default => true
+  config :include_metadata, :validate => :boolean, :default => false
 
   # Password to use for HTTP auth
   config :access_key, :validate => :password, :required => false, :default => nil
 
   # Use bearer token instead of access key/id for authentication.
   config :bearer_token, :validate => :password, :required => false, :default => nil
 
+  # json keys for which plugin looks for these keys and adds as event meatadata. A dot "." can be used to add nested subjson.
+  config :include_metadata_keys, :validate => :array, :required => false, :default => []
+
   @@MAX_PAYLOAD_SIZE = 8*1024*1024
 
   # For developer debugging.
@@ -116,6 +120,14 @@ def register
     logger.info("Max Payload Size: ",
                 :size => @@MAX_PAYLOAD_SIZE)
     configure_auth
+
+    @final_metadata_keys = Hash.new
+    if @include_metadata_keys.any?
+      include_metadata_keys.each do | nested_key |
+        @final_metadata_keys[nested_key] = nested_key.to_s.split('.')
+      end
+    end
+
   end # def register
 
   def client_config
@@ -164,12 +176,12 @@ def configure_auth
     if @access_id == nil || @access_key.value == nil
       @logger.info "Access Id or access key null. Using bearer token for authentication."
       @use_bearer_instead_of_lmv1 = true
-    end  
-    if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil 
+    end
+    if @use_bearer_instead_of_lmv1 && @bearer_token.value == nil
       @logger.error "Bearer token not specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor."
       raise LogStash::ConfigurationError, 'No valid authentication specified. Either access_id and access_key both or bearer_token must be specified for authentication with Logicmonitor.'
     end
-  end  
+  end
   def generate_auth_string(body)
     if @use_bearer_instead_of_lmv1
       return "Bearer #{@bearer_token.value}"
@@ -196,7 +208,7 @@ def send_batch(events)
         :body => body,
         :headers => {
                 "Content-Type" => "application/json",
-                "User-Agent" => "LM Logs Logstash Plugin",
+                "User-Agent" => "lm-logs-logstash/" + LmLogsLogstashPlugin::VERSION,
                 "Authorization" => "#{auth_string}"
         }
     })
@@ -265,34 +277,48 @@ def multi_receive(events)
     events.each_slice(@batch_size) do |chunk|
       documents = []
       chunk.each do |event|
-        event_json = JSON.parse(event.to_json)
-        lmlogs_event = {}
-
-        if @include_metadata
-         lmlogs_event = event_json
-         lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
-         if lmlogs_event.dig("event", "original") != nil
-            lmlogs_event["event"].delete("original") # remove redundant log field
-         end
-        end
 
-        lmlogs_event["message"] = event.get(@message_key).to_s
-        lmlogs_event["_lm.resourceId"] = {}
-        lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
+        documents = isValidPayloadSize(documents, processEvent(event), @@MAX_PAYLOAD_SIZE)
+      end
+      send_batch(documents)
+    end
+  end
 
-        if @keep_timestamp
-          lmlogs_event["timestamp"] = event.get("@timestamp")
-        end
 
-        if @timestamp_is_key
-          lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
+  def processEvent(event)
+    event_json = JSON.parse(event.to_json)
+    lmlogs_event = {}
+
+    if @include_metadata
+      lmlogs_event = event_json
+      lmlogs_event.delete("@timestamp")  # remove redundant timestamp field
+      if lmlogs_event.dig("event", "original") != nil
+        lmlogs_event["event"].delete("original") # remove redundant log field
+      end
+    elsif @final_metadata_keys
+      @final_metadata_keys.each do | key, value |
+        nestedVal = event_json
+        value.each { |x| nestedVal = nestedVal[x] }
+        if nestedVal != nil
+          lmlogs_event[key] = nestedVal
         end
+      end
+    end
 
-        documents = isValidPayloadSize(documents,lmlogs_event,@@MAX_PAYLOAD_SIZE)
+    lmlogs_event["message"] = event.get(@message_key).to_s
+    lmlogs_event["_lm.resourceId"] = {}
+    lmlogs_event["_lm.resourceId"]["#{@lm_property}"] = event.get(@property_key.to_s)
 
-      end
-      send_batch(documents)
+    if @keep_timestamp
+      lmlogs_event["timestamp"] = event.get("@timestamp")
+    end
+
+    if @timestamp_is_key
+      lmlogs_event["timestamp"] = event.get(@timestamp_key.to_s)
     end
+
+    return lmlogs_event
+
   end
 
   def log_failure(message, opts)

lib/logstash/outputs/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module LmLogsLogstashPlugin
+  VERSION = '2.0.0'
+end

logstash-output-lmlogs.gemspec

@@ -1,6 +1,14 @@
+
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+
+require "logstash/outputs/version.rb"
+
+
+
 Gem::Specification.new do |s|
   s.name = 'logstash-output-lmlogs'
-  s.version         = '1.3.0'
+  s.version         = LmLogsLogstashPlugin::VERSION
   s.licenses = ['Apache-2.0']
   s.summary = "Logstash output plugin for LM Logs"
   s.description     = "This gem is a Logstash plugin required to be installed on top of the Logstash core pipeline using $LS_HOME/bin/logstash-plugin install gemname. This gem is not a stand-alone program"
@@ -25,4 +33,5 @@ Gem::Specification.new do |s|
   s.add_runtime_dependency 'manticore', '>= 0.5.2', '< 1.0.0'
 
   s.add_development_dependency 'logstash-devutils'
+  s.add_development_dependency 'hashdiff', '>= 1.0.0'
 end

spec/outputs/metadata_spec.rb

@@ -0,0 +1,86 @@
+# encoding: utf-8
+require "logstash/devutils/rspec/spec_helper"
+require "logstash/outputs/lmlogs"
+require "logstash/event"
+require "hashdiff"
+
+
+describe LogStash::Outputs::LMLogs do
+
+
+    let(:logstash_event) {LogStash::Event.new("message" => "hello this is log 1",
+    "host" => "host1",
+    "nested1" => {"nested2" => {"nested3" => "value"},
+                  "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                  "nested2b" => {"nested3b" => "value"}
+                      },
+    "nested1_" => "value",
+    "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                  "nested_ignored" => "somevalue"
+                }
+    )}
+    let(:sample_lm_logs_event){{"message" => "hello this is log 1", "_lm.resourceId" => {"test.property" => "host1"}, "timestamp" => "2021-03-22T04:28:55.907121106Z"}}
+    let(:include_metadata_keys) {["host", "nested1.nested2.nested3", "nested1.nested2a", "nested.nested2" ]}
+
+    def create_output_plugin_with_conf(conf)
+        return LogStash::Outputs::LMLogs.new(conf)
+    end
+
+    def check_same_hash(h1,h2)
+
+    end
+
+    it "default behaviour" do
+      puts "default behaviour"
+      plugin = create_output_plugin_with_conf({
+          "portal_name" => "localhost",
+          "access_id" => "abcd",
+          "access_key" => "abcd",
+          "lm_property" => "system.hostname",
+          "property_key" => "host"
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+
+    it "with include_metadata set to true" do
+      puts "with include_metadata set to true"
+      plugin = create_output_plugin_with_conf({
+        "portal_name" => "localhost",
+        "access_id" => "abcd",
+        "access_key" => "abcd",
+        "lm_property" => "system.hostname",
+        "property_key" => "host",
+        "include_metadata" => true
+      })
+      constructed_event = plugin.processEvent(logstash_event)
+      expected_event = {
+        "message" => "hello this is log 1",
+        "timestamp" => logstash_event.timestamp,
+        "@version" => "1",
+        "_lm.resourceId" => {"system.hostname" => "host1"},
+        "host" => "host1",
+        "nested1" => {"nested2" => {"nested3" => "value"},
+                      "nested2a" => {"nested3a" => {"nested4" => "valueA"}},
+                      "nested2b" => {"nested3b" => "value"}
+                      },
+        "nested1_" => "value",
+        "nested" => {"nested2" => {"nested3" => "value",
+                                "nested3b" => "value"},
+                    "nested_ignored" => "somevalue"
+                  }
+      }
+      puts " actual : #{constructed_event} \n expected : #{expected_event}"
+      puts " hash diff : #{Hashdiff.diff(constructed_event,expected_event)}"
+      expect(Hashdiff.diff(constructed_event,expected_event)).to eq([])
+    end
+end