Merge pull request #25 from DevoInc/release-next

Version 1.5.0 to Master

Author: Worvast

.gitignore

@@ -6,6 +6,7 @@ environment.env
 tests_results/*
 !tests_results/.gitkeep
 !local_server_files/*
+check_list.txt
 
 .idea
 # Byte-compiled / optimized / DLL files

CHANGELOG.md

@@ -4,6 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
+## [1.5.0]
+#### Changed
+ * Modify Configuration loads functions
+ * Now Sender send, send_raw, flush_buffer and fill_buffer return the number of lines sent
+
+#### Fixed
+ * Problems when loads configuration from default sites
+ * Problems with local_server
+ 
 ## [1.4.0]
 #### Added
  * Allow the parameters user, application name and comment to queries

README.md

@@ -1,7 +1,7 @@
 
 [![master Build Status](https://travis-ci.com/DevoInc/python-sdk.svg?branch=master)](https://travis-ci.com/DevoInc/python-sdk) [![LICENSE](https://img.shields.io/dub/l/vibe-d.svg)](https://github.com/DevoInc/python-sdk/blob/master/LICENSE)
 
-[![wheel](https://img.shields.io/badge/wheel-yes-brightgreen.svg)](https://pypi.org/project/devo-sdk/) [![version](https://img.shields.io/badge/version-1.3.0-blue.svg)](https://pypi.org/project/devo-sdk/) [![python](https://img.shields.io/badge/python-2.7%20%7C%203.3%20%7C%203.4%20%7C%203.5%20%7C%203.6%20%7C%203.7-blue.svg)](https://pypi.org/project/devo-sdk/)
+[![wheel](https://img.shields.io/badge/wheel-yes-brightgreen.svg)](https://pypi.org/project/devo-sdk/) [![version](https://img.shields.io/badge/version-1.5.0-blue.svg)](https://pypi.org/project/devo-sdk/) [![python](https://img.shields.io/badge/python-2.7%20%7C%203.3%20%7C%203.4%20%7C%203.5%20%7C%203.6%20%7C%203.7-blue.svg)](https://pypi.org/project/devo-sdk/)
 
 
 # Devo Python SDK

devo/__version__.py

@@ -1,6 +1,6 @@
 __description__ = 'Devo Python Library.'
 __url__ = 'http://www.devo.com'
-__version__ = "1.4.0"
+__version__ = "1.5.0"
 __author__ = 'Devo'
 __author_email__ = 'support@devo.com'
 __license__ = 'MIT'

devo/api/scripts/client_cli.py

@@ -22,10 +22,12 @@ def cli():
 
 
 @cli.command()
-@click.option('--config', '-c', type=click.Path(),
-              help='JSON/Yaml File with configuration,'
-              ' you can put all options here',
-              default="~/.devo.json")
+@click.option('--config', '-c', type=click.Path(exists=True),
+              help='Optional JSON/Yaml File with configuration info.')
+@click.option('--env', '-e', help='Use env vars for configuration',
+              default=False)
+@click.option('--default', '-d', help='Use default file for configuration',
+              default=False)
 @click.option('--url', '-u', help='Endpoint for the api.')
 @click.option('--user', '-user', help='User for the api.')
 @click.option('--app_name', '-app_name', help='Application name for the api.')
@@ -46,10 +48,10 @@ def cli():
                                'stdout')
 @click.option('--format', '-f', default="json/simple/compact",
               help='The output format. Default is json/simple/compact')
-@click.option('--from',
+@click.option('--from', default=None,
               help='From date, and time for the query (YYYY-MM-DD hh:mm:ss). '
                    'For valid formats see lt-common README')
-@click.option('--to',
+@click.option('--to', default=None,
               help='To date, and time for the query (YYYY-MM-DD hh:mm:ss). '
                    'For valid formats see lt-common README')
 def query(**kwargs):
@@ -132,38 +134,30 @@ def configure(args):
     :return: Client API Object and Config values in array
     """
     config = Configuration()
-    if args.get('config') != "~/.devo.json":
-        config.load_config(args.get('config'), 'api')
-
-    config.mix(dict(args))
+    try:
+        if args.get('config'):
+            config.load_config(args.get('config'), 'api')
 
-    if "key" not in args.keys() and "api" not in args.keys() \
-            and "token" not in args.keys():
-        config.set("key", os.environ.get('DEVO_API_KEY', None))
-        config.set("secret", os.environ.get('DEVO_API_SECRET', None))
-        if "url" not in args.keys():
+        if args.get('env'):
+            config.set("key", os.environ.get('DEVO_API_KEY', None))
+            config.set("secret", os.environ.get('DEVO_API_SECRET', None))
             config.set("url", os.environ.get('DEVO_API_URL', None))
+            config.set("user", os.environ.get('DEVO_API_USER', None))
+            config.set("comment", os.environ.get('DEVO_API_COMMENT', None))
 
-    if "user" not in args.keys():
-        config.set("user", os.environ.get('DEVO_API_USER', None))    
-    if "comment" not in args.keys():
-        config.set("comment", os.environ.get('DEVO_API_COMMENT', None))
-
-    if not config.keys("key") and not config.keys("api") \
-            and not config.keys("token") \
-            and os.path.exists("~/.devo.json"):
-        config.load_default_json('api')
-
-    config.keys('from')
-    config.keys('to')
+        if args.get('default'):
+            config.load_default_config(section="api")
+    finally:
+        config.mix(dict(args))
+        conf = config.get()
 
     # Try to compose the api
     api = None
     try:
-        api = Client.from_config(config.get())
+        api = Client.from_config(conf)
     except DevoClientException as error:
         print_error(str(error), show_help=True)
-    return api, config.get()
+    return api, conf
 
 
 def print_error(error, show_help=False, stop=True):

devo/common/generic/configuration.py

@@ -2,11 +2,14 @@
 """ Util for load generic config file in devo standars"""
 import json
 import sys
+import os
+
 
 class ConfigurationException(Exception):
     """ Default Configuration Exception """
     pass
 
+
 class Configuration(object):
     """
     Main class for load config files, and extract config objects
@@ -71,7 +74,17 @@ def load_config(self, path, section=None):
 
         raise ConfigurationException("Configuration file type unknown or not supportted: %s" %path)
 
-    def load_default_config(self, ext="yml", section=None):
+    @staticmethod
+    def __search_default_config_file():
+        return "json" if os.path.exists(os.path.expanduser("~/.devo.json")) \
+            else "yaml" if os.path.exists(os.path.expanduser("~/.devo.yaml")) \
+            else "yml" if os.path.exists(os.path.expanduser("~/.devo.yml")) \
+            else None
+
+    def load_default_config(self, ext=None, section=None):
+        if not ext:
+            ext = self.__search_default_config_file()
+
         if ext is "yml" or ext is "yaml":
             return self.load_yaml("~/.devo.%s" % ext, section)
         elif ext is "json":
@@ -151,7 +164,7 @@ def set(self, key_list, value):
         if value:
             if not isinstance(key_list, list):
                 key_list = [key_list]
-            self.set_key_chain(self, key_list, value)
+            self.cfg = self.set_key_chain(self.cfg, key_list, value)
 
     @staticmethod
     def set_key_chain(aux_dict, key_list, value):

devo/sender/data.py

@@ -20,7 +20,6 @@
 
 class DevoSenderException(Exception):
     """ Default Devo Sender Exception """
-    pass
 
 
 class SenderConfigSSL:
@@ -121,7 +120,6 @@ def __init__(self, config=None, **kwargs):
         logger = kwargs.get('logger', None)
         tag = kwargs.get('tag', None)
 
-
         logging.Handler.__init__(self)
         self.logger = self.__set_logger(verbose_level) if logger is None \
             else logger
@@ -134,6 +132,7 @@ def __init__(self, config=None, **kwargs):
         self.max_zip_buffer = 19500
         self.compression_level = -1
         self.zip_buffer = b''
+        self.zip_events = 0
         self.composed_msg = b''
         self.facility = facility
 
@@ -210,8 +209,8 @@ def __connect_ssl(self):
         except socket.error as error:
             self.close()
             raise DevoSenderException(
-                "Devo-Sender|SSL conn establishment socket error: %s" % str(error))
-
+                "Devo-Sender|SSL conn establishment socket error: %s" %
+                str(error))
 
     def info(self, msg):
         """
@@ -261,12 +260,27 @@ def __encode_record(record):
 
         return record
 
+    def __send_oc(self, record):
+        msg_size = len(record)
+        sent = 0
+        total = int(msg_size / 4096)
+        if msg_size % 4096 > 0:
+            total += 1
+        for iteration in range(0, total):
+            sent += self.socket.send(
+                record[int(iteration * 4096):
+                       int((iteration + 1) * 4096)])
+        if sent == 0:
+            raise DevoSenderException("Devo-Sender|Send error")
+            return False
+        return True
+
     def send_raw(self, record, multiline=False, zip=False):
         """
         Send raw messages to the collector
 
         >>>con.send_raw('<14>Jan  1 00:00:00 MacBook-Pro-de-X.local'
-        ...             'my.app.devo_sender.test: txt Prueba')
+        ...             'my.app.devo_sender.test: txt test')
 
         """
         try:
@@ -277,30 +291,23 @@ def send_raw(self, record, multiline=False, zip=False):
                 try:
                     if not multiline and not zip:
                         sent = self.socket.send(self.__encode_record(record))
+                        return 1
                     else:
                         if multiline:
                             record = self.__encode_record(record)
 
-                        msg_size = len(record)
-                        sent = 0
-                        total = int(msg_size / 4096)
-                        if msg_size % 4096 > 0:
-                            total += 1
-                        for iteration in range(0, total):
-                            sent += self.socket.send(
-                                record[int(iteration * 4096):
-                                       int((iteration + 1) * 4096)])
-
-                        if sent == 0:
-                            raise DevoSenderException("Devo-Sender|Send error")
-
-                    if self._sender_config.debug:
-                        self.logger.debug('sent|%d|size|%d|msg|%s' %
-                                          (sent, len(record), record))
+                        if self.__send_oc(record):
+                            return 1
+                        return 0
                 except socket.error:
                     self.close()
                     raise DevoSenderException(
                         "Devo-Sender|Socket error: %s" % str(socket.error))
+                finally:
+                    if self._sender_config.debug:
+                        self.logger.debug('sent|%d|size|%d|msg|%s' %
+                                          (sent, len(record), record))
+
         except Exception as error:
             raise error
 
@@ -352,14 +359,14 @@ def send(self, tag, msg, **kwargs):
         :param kwargs: date -> String Date format '%Y-%m-%d %H%M%S'
 
 
-        >>>con.send(tag='my.app.devo_sender.test', msg='prueba mensaje')
+        >>>con.send(tag='my.app.devo_sender.test', msg='test of msg')
         See Also:
             send_raw
         """
         if isinstance(msg, bytes):
-            self.send_bytes(tag, msg, **kwargs)
+            return self.send_bytes(tag, msg, **kwargs)
         else:
-            self.send_str(tag, msg, **kwargs)
+            return self.send_str(tag, msg, **kwargs)
 
     def send_str(self, tag, msg, **kwargs):
         """
@@ -369,24 +376,24 @@ def send_str(self, tag, msg, **kwargs):
             msg += "\n"
 
         msg = COMPOSE % (self.compose_mem(tag, **kwargs), msg)
-        self.send_raw(msg, multiline=kwargs.get('multiline', False))
+        return self.send_raw(msg, multiline=kwargs.get('multiline', False))
 
     def send_bytes(self, tag, msg, **kwargs):
         """
         Send function when bytes, sure py3x. Can be zipped
         """
         msg = COMPOSE_BYTES % (self.compose_mem(tag, bytes=True, **kwargs), msg)
         if kwargs.get('zip', False):
-            self.fill_buffer(msg)
+            return self.fill_buffer(msg)
         else:
             if msg[-1:] != b"\n":
                 msg += b"\n"
 
-            self.send_raw(msg, multiline=kwargs.get('multiline', False))
+            return self.send_raw(msg, multiline=kwargs.get('multiline', False))
 
     def fill_buffer(self, msg):
         """
-        Internal method for fill buffer for be zipped and sended
+        Internal method for fill buffer for be zipped and sent
         :param msg: bytes
         :return: None
         """
@@ -395,19 +402,29 @@ def fill_buffer(self, msg):
 
         self.zip_buffer += msg
         if len(self.zip_buffer) > self.max_zip_buffer:
-            self.flush_buffer()
+            return self.flush_buffer()
+
+        self.zip_events += 1
+        return 0
 
     def flush_buffer(self):
         """
-        Method for flush-send buffer, its zipped and sended now
+        Method for flush-send buffer, its zipped and sent now
         :return: None
         """
         if self.zip_buffer:
-            compressor = zlib.compressobj(self.compression_level,
-                                          zlib.DEFLATED, 31)
-            record = compressor.compress(self.zip_buffer) + compressor.flush()
-            self.send_raw(record, zip=True)
-            self.zip_buffer = b''
+            try:
+                compressor = zlib.compressobj(self.compression_level,
+                                              zlib.DEFLATED, 31)
+                record = compressor.compress(self.zip_buffer) + compressor.flush()
+                if self.send_raw(record, zip=True):
+                    return self.zip_events
+                return 0
+            except Exception as error:
+                raise error
+            finally:
+                self.zip_buffer = b''
+                self.zip_events = 0
 
     def set_logger_tag(self, tag):
         """

devo/sender/lookup.py

@@ -9,7 +9,7 @@
 
 class Lookup:
     """ Main class Lookup for create and send the object from some sources """
-    # Type of the header sended
+    # Type of the header sent
     # - EVENT_START for START header
     # - EVENT_END for END header
     EVENT_START = 'START'