New features for rate limiting (#19)

* Time and Size based rate limit, gzip compression and psutil for system properties

* addressed review comment

* resonseInterface to responseInterface

* updated the testcases

* added try-except for json.dumps and introduced constants for size limiting

* Addressed comments regarding linting in code using Flake8

Author: lmajay

docs/source/conf.py

@@ -29,7 +29,7 @@
 # -- Project information -----------------------------------------------------
 _version = {}
 with open("../../logicmonitor_data_sdk/version.py") as fp:
-  exec(fp.read(), _version)
+    exec(fp.read(), _version)
 
 project = 'LogicMonitor DATA SDK Python'
 copyright = '2021, Logicmonitor'

example/cpu_metrics.py

@@ -12,17 +12,16 @@
 import os
 import sys
 import time
+import psutil
 
-import psutil as psutil
-
-sys.path.append("..")
 import logicmonitor_data_sdk
-from logicmonitor_data_sdk.api.response_interface import ResonseInterface
+from logicmonitor_data_sdk.api.response_interface import ResponseInterface
 from logicmonitor_data_sdk.models import Resource, DataSource, DataPoint, \
-  DataSourceInstance
+    DataSourceInstance
 
 from logicmonitor_data_sdk.api.metrics import Metrics
 
+sys.path.append("..")
 logger = logging.getLogger('lmdata.api')
 logger.setLevel(logging.INFO)
 
@@ -31,39 +30,39 @@
 configuration.debug = False
 
 
-class MyResponse(ResonseInterface):
-  """
-  Sample callback to handle the response from the REST endpoints
-  """
+class MyResponse(ResponseInterface):
+    """
+    Sample callback to handle the response from the REST endpoints
+    """
 
-  def success_callback(self, request, response, status, request_id):
-    logger.info("%s: %s: %s", response, status, request_id)
+    def success_callback(self, request, response, status, request_id):
+        logger.info("%s: %s: %s", response, status, request_id)
 
-  def error_callback(self, request, response, status, request_id, reason):
-    logger.error("%s: %s: %s %s", response, status, reason, request_id)
+    def error_callback(self, request, response, status, request_id, reason):
+        logger.error("%s: %s: %s %s", response, status, reason, request_id)
 
 
 def MetricRequest():
-  """
-  Main function to get the CPU values using `psutil` and send to Metrics REST endpoint
-  """
-  device_name = os.uname()[1]
-  resource = Resource(ids={'system.displayname': device_name}, name=device_name,
-                      create=True)
-  datasource = DataSource(name="CPU")
-  instance = DataSourceInstance(name='cpu-1')
-  datapoint = DataPoint(name="cpu_utilization")
-  metric_api = Metrics(batch=True, interval=10, response_callback=MyResponse())
-  while True:
-    values = {str(int(time.time())): str(psutil.cpu_percent())}
+    """
+    Main function to get the CPU values using `psutil` and send to Metrics REST endpoint
+    """
+    device_name = os.uname()[1]
+    resource = Resource(ids={'system.displayname': device_name}, name=device_name,
+                        create=True)
+    datasource = DataSource(name="CPU")
+    instance = DataSourceInstance(name='cpu-1')
+    datapoint = DataPoint(name="cpu_utilization")
+    metric_api = Metrics(batch=True, interval=10, response_callback=MyResponse())
+    while True:
+        values = {str(int(time.time())): str(psutil.cpu_percent())}
 
-    metric_api.send_metrics(resource=resource,
-                            datasource=datasource,
-                            instance=instance,
-                            datapoint=datapoint,
-                            values=values)
-    time.sleep(10)
+        metric_api.send_metrics(resource=resource,
+                                datasource=datasource,
+                                instance=instance,
+                                datapoint=datapoint,
+                                values=values)
+        time.sleep(10)
 
 
 if __name__ == "__main__":
-  MetricRequest()
+    MetricRequest()

example/disk_metrics.py

@@ -16,9 +16,9 @@
 
 import logicmonitor_data_sdk
 from logicmonitor_data_sdk.api.metrics import Metrics
-from logicmonitor_data_sdk.api.response_interface import ResonseInterface
+from logicmonitor_data_sdk.api.response_interface import ResponseInterface
 from logicmonitor_data_sdk.models import Resource, DataSource, DataPoint, \
-  DataSourceInstance
+    DataSourceInstance
 
 logger = logging.getLogger('lmdata.api')
 logger.setLevel(logging.INFO)
@@ -28,50 +28,50 @@
 configuration.debug = False
 
 
-class MyResponse(ResonseInterface):
-  """
-  Sample callback to handle the response from the REST endpoints
-  """
+class MyResponse(ResponseInterface):
+    """
+    Sample callback to handle the response from the REST endpoints
+    """
 
-  def success_callback(self, request, response, status, request_id):
-    logger.info("%s: %s: %s", response, status, request_id)
+    def success_callback(self, request, response, status, request_id):
+        logger.info("%s: %s: %s", response, status, request_id)
 
-  def error_callback(self, request, response, status, request_id, reason):
-    logger.error("%s: %s: %s %s", response, status, reason, request_id)
+    def error_callback(self, request, response, status, request_id, reason):
+        logger.error("%s: %s: %s %s", response, status, reason, request_id)
 
 
 def MetricRequest():
-  """
-  Main function to get the CPU values using `psutil` and send to Metrics REST endpoint
-  """
-  device_name = os.uname()[1]
-  resource = Resource(ids={'system.displayname': device_name}, name=device_name,
-                      create=True)
-  datasource = DataSource(name="DiskUsingSDK")
-  datapoints = ['total', 'used', 'free']
-  metric_api = Metrics(batch=True, interval=30, response_callback=MyResponse())
-  while True:
-    partitions = psutil.disk_partitions()
-    for p in partitions:
-      # Using the device as instance name. We can use the mountpoint as well.
+    """
+    Main function to get the CPU values using `psutil` and send to Metrics REST endpoint
+    """
+    device_name = os.uname()[1]
+    resource = Resource(ids={'system.displayname': device_name}, name=device_name,
+                        create=True)
+    datasource = DataSource(name="DiskUsingSDK")
+    datapoints = ['total', 'used', 'free']
+    metric_api = Metrics(batch=True, interval=30, response_callback=MyResponse())
+    while True:
+        partitions = psutil.disk_partitions()
+        for p in partitions:
+            # Using the device as instance name. We can use the mountpoint as well.
 
-      instance_name = p.device
-      usage = psutil.disk_usage(instance_name)._asdict()
+            instance_name = p.device
+            usage = psutil.disk_usage(instance_name)._asdict()
 
-      # Create the instance object for every device. Name should not have the
-      # special characters so replacing it with the '-'.
-      instance = DataSourceInstance(name=instance_name.replace('/', '-'),
-                                    display_name=instance_name)
-      for one_datapoint in datapoints:
-        datapoint = DataPoint(name=one_datapoint)
-        values = {str(int(time.time())): str(usage[one_datapoint])}
-        metric_api.send_metrics(resource=resource,
-                                datasource=datasource,
-                                instance=instance,
-                                datapoint=datapoint,
-                                values=values)
-    time.sleep(5)
+            # Create the instance object for every device. Name should not have the
+            # special characters so replacing it with the '-'.
+            instance = DataSourceInstance(name=instance_name.replace('/', '-'),
+                                          display_name=instance_name)
+            for one_datapoint in datapoints:
+                datapoint = DataPoint(name=one_datapoint)
+                values = {str(int(time.time())): str(usage[one_datapoint])}
+                metric_api.send_metrics(resource=resource,
+                                        datasource=datasource,
+                                        instance=instance,
+                                        datapoint=datapoint,
+                                        values=values)
+        time.sleep(5)
 
 
 if __name__ == "__main__":
-  MetricRequest()
+    MetricRequest()

example/log_non_batch.py

@@ -9,27 +9,41 @@
 """
 
 # Sample Program to send Logs to LogicMonitor Platform
-#
-import os
 import logicmonitor_data_sdk
-
 from logicmonitor_data_sdk.api.logs import Logs
 from logicmonitor_data_sdk.models import Resource
+from logicmonitor_data_sdk.api.response_interface import ResponseInterface
+from example import system_properties
+
+
+class MyResponse(ResponseInterface):
+    """
+    Sample callback to handle the response from the REST endpoints
+    """
+
+    def success_callback(self, request, response, status, request_id):
+        # logging.info("%s: %s: %s", response, status, request_id)
+        print(response, status, request_id)
+
+    def error_callback(self, request, response, status, request_id, reason):
+        # logging.error("%s: %s: %s %s", response, status, reason, request_id)
+        print(response, status, reason, request_id)
+
 
 # Initialize LM SDK and provide required authentication parameters
 # On LM Portal, create 'API Token' for the user to get access Id and access Key
-configuration = logicmonitor_data_sdk.Configuration( company='your_company',
-                                                          id='API access id',
-                                                         key='API access key')
+configuration = logicmonitor_data_sdk.Configuration(company='your_company',
+                                                    id='API access id',
+                                                    key='API access key')
 
 # The resource which is already present on LM Platform. Use a unique property to match
-# the resource and send log for that.                                                    
-resource = Resource(ids={"system.hostname": 'your_system'})
+# the resource and send log for that.
+resource = Resource(ids={"system.hostname": 'your_system'}, properties=system_properties.get_system_info())
 
-#Create an api handle for sending the logs 
+# Create an api handle for sending the logs
 # "batch" would club logs for 8MB size or 30 Sec - whichever is earlier. Its default is "True".
-log_api = Logs(batch = False)
+log_api = Logs(batch=False)
 
-return_value = log_api.send_logs(resource=resource, msg= "this is sample log")
+return_value = log_api.send_logs(resource=resource, msg="this is sample log")
 
 print(return_value)

example/simple_example.py

@@ -8,64 +8,63 @@
 =======
 """
 
-import time
-from random import seed, random
-
-import logicmonitor_data_sdk
-
 # LogicMonitor metric data model is as below
 #
-#Company
+# Company
 #  |--- Resource (like device/service. Ex: VM)
 #  |--- Data Source   (Ex. CPU)
 #         |--- Instance (of a Data Source on a resource. Ex. CPU-1)
 #               |--- Data Point (the metric which is being monitored. Ex. %Used)
 #                     |- <Time> : <Metric Value>
 #                     |- <Time> : <Metric Value>
 #                     |...
-#
+import time
+from random import random
+
+import logicmonitor_data_sdk
 from logicmonitor_data_sdk.api.metrics import Metrics
-from logicmonitor_data_sdk.api.response_interface import ResonseInterface
+from logicmonitor_data_sdk.api.response_interface import ResponseInterface
 from logicmonitor_data_sdk.api_client import ApiClient
-from logicmonitor_data_sdk.models import DataSource, \
-  Resource, DataSourceInstance, DataPoint
+from logicmonitor_data_sdk.models import DataSource, Resource, DataSourceInstance, DataPoint
+from example import system_properties
 
 # Configure SDK with Account and access information
 # On your LogicMonitor portal, create API token (LMv1) for user and get
 # Access Id and Access Key
 configuration = logicmonitor_data_sdk.Configuration(company='your_company',
                                                     id='API_ACCESS_ID',
                                                     key='API_ACCESS_KEY')
-class MyResponse(ResonseInterface):
+
+
+class MyResponse(ResponseInterface):
     """
     Sample callback to handle the response from the REST endpoints
     """
 
     def success_callback(self, request, response, status, request_id):
-        #logging.info("%s: %s: %s", response, status, request_id)
+        # logging.info("%s: %s: %s", response, status, request_id)
         print(response, status, request_id)
 
-
     def error_callback(self, request, response, status, request_id, reason):
-        #logging.error("%s: %s: %s %s", response, status, reason, request_id)
+        # logging.error("%s: %s: %s %s", response, status, reason, request_id)
         print(response, status, reason, request_id)
 
 
 # Create api handle for Metrics use case (we also support Logs)
 api_client = ApiClient(configuration=configuration)
-metric_api = Metrics(batch=False,interval=10,response_callback=MyResponse(),api_client=api_client)
+metric_api = Metrics(batch=False, interval=10, response_callback=MyResponse(), api_client=api_client)
 return_val = metric_api.send_metrics(
-               resource=Resource(
-                   ids={"system.hostname": "SampleDevice"},  #Core Properties of the Resource
-                   create=True,                              #Auto-create resource if does not exist
-                   name="SampleDevice",                      #Name of the resource
-                   properties={"using.sdk": "true"}),        #Additional Properties [Optional]
-               datasource=DataSource(
-                   name="SampleDS"),                         #Name of data source is must. Rest optional
-               instance=DataSourceInstance(
-                   name="SampleInstance"),                   #Name of instance is must. Rest optional
-               datapoint=DataPoint(
-                  name="SampleDataPoint"),                  #The metric
-               values={str(int(time.time())): str(random())} #Values at specific time(s)
+    resource=Resource(
+        ids={"system.hostname": "SampleDevice"},  # Core Properties of the Resource
+        create=True,  # Auto-create resource if does not exist
+        name="SampleDevice",  # Name of the resource
+        properties=system_properties.get_system_info()),  # Additional Properties [Optional]
+    datasource=DataSource(
+        name="SampleDS"),  # Name of data source is must. Rest optional
+    instance=DataSourceInstance(
+        name="SampleInstance"),  # Name of instance is must. Rest optional
+    datapoint=DataPoint(
+        name="SampleDataPoint"),  # The metric
+    values={str(int(time.time())): str(random())}  # Values at specific time(s)
 )
-print("Return Value = ",return_val)
+print("Return Value = ", return_val)

example/system_properties.py

@@ -0,0 +1,26 @@
+import psutil
+
+
+def sanitize_for_serialization(obj):
+    dict_obj = {}
+    fields = obj._fields
+    for i in fields:
+        dict_obj[i] = str(obj.__getattribute__(i))
+    return dict_obj
+
+
+def get_system_info():
+    swap_memory = sanitize_for_serialization(psutil.swap_memory())
+    virtual_memory = sanitize_for_serialization(psutil.virtual_memory())
+    system = {
+        "using.sdk": "true",
+        'boot_time': str(psutil.boot_time()),
+        'cpu_count': str(psutil.cpu_count()),
+        'swap_memory_total': swap_memory["total"],
+        'swap_memory_used': swap_memory["used"],
+        'swap_memory_free': swap_memory["free"],
+        'virtual_memory_total': virtual_memory["total"],
+        'virtual_memory_available': virtual_memory["available"],
+        'virtual_memory_used': virtual_memory["used"]
+    }
+    return system

example/test_property.py

@@ -11,10 +11,10 @@
 import logging
 import sys
 
-sys.path.append("..")
 import logicmonitor_data_sdk
 from logicmonitor_data_sdk.api.metrics import Metrics
 
+sys.path.append("..")
 logger = logging.getLogger('lmdata.api')
 logger.setLevel(logging.INFO)
 
@@ -35,7 +35,7 @@
 
 response = metric_api.update_instance_property(
     resource_ids={'system.deviceId': '233267'}, datasource='dsname_1',
-    instancename='instance_1', instance_properties={
+    instance_name='instance_1', instance_properties={
       'ins.property': 'values'},
 )
-logger.info(response)
+logger.info(response)