Fix/sender exception (#177)

Closes DevoSenderException is throwing an unexpected exception when transforming to string #175
DevoClientException refactoring
DevoSenderException refactoring
Snyk integration for checking dependencies and static code security
pipdeptree dependency open from 2.5.0 to >=2.5.0
Fixed message shown when configuration file (JSON or YAML) is not correct

---------

Co-authored-by: Felipe Conde Benavides <felipe.conde@devo.com>
Co-authored-by: Alejandro Dotor de Pradas <alejandro.dotordepradas@devo.com>

Author: 3 people

CHANGELOG.md

@@ -4,11 +4,15 @@ All notable changes to this project will be documented in this file.
 The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/)
 and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.html).
 
-## [5.0.7] - Unreleased
+## [5.1.0] - 2023-03-07
 ### Fixed
+ * Fixed [exception handling issue](https://github.com/DevoInc/python-sdk/issues/175)
+in `devo/sender/data.py`
  * Fixed message shown when configuration file (JSON or YAML) is not correct
 ### Added
+ * Client side exception management refactoring for sending and querying data
  * Snyk integration for checking dependencies and static code security
+ * `pipdeptree` dependency open from ==2.5.0 to >=2.5.0
 
 ## [5.0.6] - 2023-02-21
 ### Chorus

CONTRIBUTING.md

@@ -26,7 +26,7 @@ reporting a new bug.
 To be able to make a fork and the corresponding MR you have to accept Devo's CLA
 The process to modify one package or script is the next:
 
-1. Create your fork from release-next
+1. Create your fork from `master` git branch.
 2. Add to the `CHANGELOG.md`, in 
 [`Unreleased`](#How_can_I_minimize_the_effort_required?) the tasks 
 that you are going to take or are carrying out to be able to review at a quick 

devo/__version__.py

@@ -1,6 +1,6 @@
 __description__ = 'Devo Python Library.'
 __url__ = 'http://www.devo.com'
-__version__ = "5.0.7"
+__version__ = "5.1.0"
 __author__ = 'Devo'
 __author_email__ = 'support@devo.com'
 __license__ = 'MIT'

devo/api/client.py

@@ -1,20 +1,22 @@
 # -*- coding: utf-8 -*-
 """Main class for pull data from Devo API (Client)."""
-import hmac
+import calendar
 import hashlib
+import hmac
+import json
 import logging
 import os
 import re
 import time
-import json
+from datetime import datetime, timedelta
+
+import pytz
 import requests
+from requests import JSONDecodeError
+
 from devo.common import default_from, default_to
 from .processors import processors, proc_json, \
     json_compact_simple_names, proc_json_compact_simple_to_jobj
-import calendar
-from datetime import datetime, timedelta
-import pytz
-
 
 CLIENT_DEFAULT_APP_NAME = 'python-sdk-app'
 CLIENT_DEFAULT_USER = 'python-sdk-user'
@@ -37,15 +39,20 @@
     "no_endpoint": "Endpoint 'address' not found",
     "to_but_no_from": "If you use end dates for the query 'to' it is "
                       "necessary to use start date 'from'",
-    "binary_format_requires_output": "Binary format like `msgpack` and `xls` requires output parameter",
+    "binary_format_requires_output": "Binary format like `msgpack` and `xls` requires output"
+                                     " parameter",
     "wrong_processor": "processor must be lambda/function or one of the defaults API processors.",
     "default_keepalive_only": "Mode '%s' always uses default KeepAlive Token",
     "keepalive_not_supported": "Mode '%s' does not support KeepAlive Token",
     "stream_mode_not_supported": "Mode '%s' does not support stream mode",
-    "future_queries_not_supported": "Modes 'xls' and 'msgpack' does not support future queries because KeepAlive"
-                                    " tokens are not available for those resonses type",
+    "future_queries_not_supported": "Modes 'xls' and 'msgpack' does not support future queries"
+                                    " because KeepAlive tokens are not available for those "
+                                    "resonses type",
     "missing_api_key": "You need a API Key and API secret to make this",
-    "data_query_error": "Error while receiving query data: %s "
+    "data_query_error": "Error while receiving query data: %s ",
+    "connection_error": "Failed to establish a new connection",
+    "other_errors": "Error while invoking query",
+    "error_no_detail": "Error code %d while invoking query"
 }
 
 DEFAULT_KEEPALIVE_TOKEN = '\n'
@@ -54,72 +61,83 @@
 
 
 class DevoClientException(Exception):
-    """ Default Devo Client Exception """
-
-    def __init__(self, message, status=None, code=None, cause=None):
-        if isinstance(message, dict):
-            self.status = message.get('status', status)
-            self.cause = message.get('cause', cause)
-            self.message = message.get('msg',
-                                       message if isinstance(message, str)
-                                       else json.dumps(message))
-            self.cid = message.get('cid', None)
-            self.code = message.get('code', code)
-            self.timestamp = message.get('timestamp',
-                                         time.time_ns() // 1000000)
-        else:
-            self.message = message
-            self.status = status
-            self.cause = cause
-            self.cid = None
-            self.code = code
-            self.timestamp = time.time_ns() // 1000000
-        super().__init__(message)
-
-    def __str__(self):
-        return self.message + ((": " + self.cause) if self.cause else '')
-
-
-def raise_exception(error_data, status=None):
-    if isinstance(error_data, requests.models.Response):
-        raise DevoClientException(
-            _format_error(error_data.json(), status=error_data.status_code))
-
-    elif isinstance(error_data, str):
-        if not status:
-            raise DevoClientException(
-                _format_error({"object": error_data}, status=None))
-        raise DevoClientException(
-            _format_error({"object": error_data}, status=status))
-    elif isinstance(error_data, BaseException):
-        raise DevoClientException(_format_error(error_data, status=None))\
-            from error_data
-    else:
-        raise DevoClientException(_format_error(error_data, status=None))
-
-
-def _format_error(error, status):
-    if isinstance(error, dict):
-        response = {
-            "msg": error.get("msg", "Error Launching Query"),
-            "cause": error.get("object") or error.get("context") or error
-        }
-        # 'object' may be a list
-        if isinstance(response["cause"], list):
-            response["cause"] = ": ".join(response["cause"])
-        if status:
-            response['status'] = status
-        elif 'status' in error:
-            response['status'] = error['status']
-        for item in ['code', 'cid', 'timestamp']:
-            if item in error:
-                response[item] = error[item]
-        return response
-    else:
-        return {
-            "msg": str(error),
-            "cause": str(error)
-        }
+    """ Default Devo Client Exception for functionalities
+     related to querying data to the platform"""
+
+    def __init__(self, message: str):
+        """
+        Creates an exception related to query data functionality
+
+        :param message: Message describing the exception. It will be
+         also used as `args` attribute in `Exception`class
+        """
+        self.message = message
+        """Message describing exception"""
+        super().__init__(self.message)
+
+
+class DevoClientRequestException(DevoClientException):
+    """ Devo Client Exception that is raised whenever a query data request
+    is performed and processed but an error is found on server side"""
+
+    def __init__(self, response: requests.models.Response):
+        """
+        Creates an exception related bad request of data queries
+
+        :param response: A `requests.models.Response` model standing
+        for the `request` library response for the query data request.
+        It will be also used as `args` attribute in `Exception`class
+        """
+        self.status = response.status_code
+        try:
+            error_response = response.json()
+            self.message = error_response.get("msg",
+                                              error_response.get("error", "Error Launching Query"))
+            """Message describing exception"""
+            if 'code' in error_response:
+                self.code = error_response['code']
+                """Error code `int` as returned by server"""
+            if 'error' in error_response:
+                self.cause = error_response.get("error")
+                """Cause of error or detailed description as returned by server"""
+                if 'code' in error_response['error']:
+                    self.code = error_response['error']['code']
+                if 'message' in error_response['error']:
+                    self.message = error_response['error']['message']
+            elif 'object' in error_response:
+                self.message = ": ".join(error_response["object"])
+            else:
+                self.cause = error_response
+            if 'cid' in error_response:
+                self.cid = error_response['cid']
+                """Unique request identifier as assigned by server"""
+            self.timestamp = error_response.get('timestamp', time.time_ns() // 1000000)
+            """Timestamp of the error if returned by server, autogenerated if not"""
+        except JSONDecodeError as exc:
+            self.message = ERROR_MSGS["error_no_detail"] % self.status
+        super().__init__(self.message)
+
+
+class DevoClientDataResponseException(DevoClientException):
+    """ Devo Client Exception that is raised after a successful streamed request
+    whenever an error is found during the processing of an event"""
+
+    def __init__(self, message: str, code: int, cause: str):
+        """
+        Creates an exception related to wrong processing of an event of a successful request
+
+        :param message: Message describing the exception. It will be
+         also used as `args` attribute in `Exception`class
+        :param code: Error code `int` as returned by server
+        :param cause: Cause of error or detailed description as returned by server
+        """
+        self.message = message
+        """Message describing exception"""
+        self.code = code
+        """Error code `int` as returned by server"""
+        self.cause = cause
+        """Cause of error or detailed description as returned by server"""
+        super().__init__(self.message)
 
 
 class ClientConfig:
@@ -169,12 +187,12 @@ def set_processor(self, processor=None):
             try:
                 self.processor = processors()[self.proc]()
             except KeyError:
-                raise_exception(f"Processor {self.proc} not found")
+                raise DevoClientException(f"Processor {self.proc} not found")
         elif isinstance(processor, (type(lambda x: 0))):
             self.proc = "CUSTOM"
             self.processor = processor
         else:
-            raise_exception(ERROR_MSGS["wrong_processor"])
+            raise DevoClientException(ERROR_MSGS["wrong_processor"])
         return True
 
     def set_user(self, user=CLIENT_DEFAULT_USER):
@@ -269,7 +287,7 @@ def __init__(self, address=None, auth=None, config=None,
 
         self.auth = auth
         if not address:
-            raise raise_exception(ERROR_MSGS['no_endpoint'])
+            raise DevoClientException(ERROR_MSGS['no_endpoint'])
 
         self.address = self.__get_address_parts(address)
 
@@ -418,7 +436,7 @@ def query(self, query=None, query_id=None, dates=None,
                 toDate = self._toDate_parser(fromDate, default_to(dates['to']))
 
                 if toDate > default_to("now()"):
-                    raise raise_exception(ERROR_MSGS["future_queries_not_supported"])
+                    raise DevoClientException(ERROR_MSGS["future_queries_not_supported"])
 
             self.config.stream = False
 
@@ -470,8 +488,8 @@ def _return_string_stream(self, payload):
             first = next(response)
         except StopIteration:
             return None  # The query did not return any result
-        except TypeError:
-            raise_exception(response)
+        except TypeError as error:
+            raise DevoClientException(ERROR_MSGS["other_errors"]) from error
 
         if self._is_correct_response(first):
             if self.config.proc == SIMPLECOMPACT_TO_OBJ:
@@ -548,7 +566,7 @@ def _make_request(self, payload):
             try:
                 response = self.__request(payload)
                 if response.status_code != 200:
-                    raise DevoClientException(response)
+                    raise DevoClientRequestException(response)
 
                 if self.config.stream:
                     if (self.config.response in ["msgpack", "xls"]):
@@ -560,15 +578,12 @@ def _make_request(self, payload):
             except requests.exceptions.ConnectionError as error:
                 tries += 1
                 if tries > self.retries:
-                    return raise_exception(error)
-                time.sleep(self.retry_delay * (2 ** (tries-1)))
+                    raise DevoClientException(ERROR_MSGS["connection_error"]) from error
+                time.sleep(self.retry_delay * (2 ** (tries - 1)))
             except DevoClientException as error:
-                if isinstance(error, DevoClientException):
-                    raise_exception(error.args[0])
-                else:
-                    raise_exception(error)
+                raise
             except Exception as error:
-                return raise_exception(error)
+                raise DevoClientException(ERROR_MSGS["other_errors"]) from error
 
     def __request(self, payload):
         """
@@ -758,19 +773,18 @@ def _call_jobs(self, address):
                                         verify=self.verify,
                                         timeout=self.timeout)
             except ConnectionError as error:
-                raise_exception({"status": 404, "msg": error})
+                raise DevoClientException(ERROR_MSGS["connection_error"]) from error
 
             if response:
                 if response.status_code != 200 or \
                         "error" in response.text[0:15].lower():
-                    raise_exception(response.text)
-                    return None
+                    raise DevoClientRequestException(response)
                 try:
                     return json.loads(response.text)
                 except json.decoder.JSONDecodeError:
                     return response.text
             tries += 1
-            time.sleep(self.retry_delay * (2 ** (tries-1)))
+            time.sleep(self.retry_delay * (2 ** (tries - 1)))
         return {}
 
     @staticmethod
@@ -893,7 +907,7 @@ def _error_handler(self, content):
                 error = match.group(0)
                 code = int(match.group(1))
                 message = match.group(2).strip()
-                raise DevoClientException(
+                raise DevoClientDataResponseException(
                     ERROR_MSGS["data_query_error"]
                     % message, code=code, cause=error)
             else: