fix: pysnmp debug mode (#1069)

* fix: pysnmp debug mode [run-int-tests] Signed-off-by: Ilya Kheifets <ikheifets@splunk.com> * fix: make debug mode more configurable [run-int-tests] Signed-off-by: Ilya Kheifets <ikheifets@splunk.com> * fix: add debug logs for traps Signed-off-by: Ilya Kheifets <ikheifets@splunk.com> * fix: linting tips [run-int-tests] Signed-off-by: Ilya Kheifets <ikheifets@splunk.com> --------- Signed-off-by: Ilya Kheifets <ikheifets@splunk.com>
splunk · Sep 17, 2024 · d4770e2 · d4770e2
1 parent abee2ee
commit d4770e2
Show file tree

Hide file tree

Showing 28 changed files with 108 additions and 8 deletions.
diff --git a/charts/splunk-connect-for-snmp/templates/traps/deployment.yaml b/charts/splunk-connect-for-snmp/templates/traps/deployment.yaml
@@ -59,6 +59,8 @@ spec:
               value: "http://{{ printf "%s-%s" .Release.Name "mibserver" }}/standard.txt"
             - name: LOG_LEVEL
               value: {{ .Values.traps.logLevel | default "INFO" }}
+            - name: PYSNMP_DEBUG
+              value: {{ .Values.pysnmpDebug | default "" | quote }}
             {{- if .Values.splunk.protocol }}
             - name: SPLUNK_HEC_SCHEME
               value: {{ .Values.splunk.protocol | default "https" | quote }}

diff --git a/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl b/charts/splunk-connect-for-snmp/templates/worker/_helpers.tpl
@@ -111,6 +111,8 @@ Common labels
   value: {{ .Values.worker.udpConnectionTimeout | default "3" | quote }}
 - name: MAX_OID_TO_PROCESS
   value: {{ .Values.poller.maxOidToProcess | default "70" | quote }}
+- name: PYSNMP_DEBUG
+  value: {{ .Values.pysnmpDebug | default "" | quote }}
 - name: PROFILES_RELOAD_DELAY
   value: {{ .Values.worker.profilesReloadDelay | default "60" | quote }}
 - name: MIB_SOURCES

diff --git a/charts/splunk-connect-for-snmp/values.schema.json b/charts/splunk-connect-for-snmp/values.schema.json
@@ -37,6 +37,9 @@
     "imagePullSecrets": {
       "type": "array"
     },
+    "pysnmpDebug": {
+      "type": "string"
+    },
     "UI": {
       "type": "object",
       "additionalProperties": false,

diff --git a/docker_compose/docker-compose-traps.yaml b/docker_compose/docker-compose-traps.yaml
@@ -25,6 +25,7 @@ services:
     - SPLUNK_HEC_INSECURESSL=${SPLUNK_HEC_INSECURESSL:-false}
     - SPLUNK_HEC_PATH=${SPLUNK_HEC_PATH:-/services/collector}
     - SNMP_V3_SECURITY_ENGINE_ID=${SNMP_V3_SECURITY_ENGINE_ID:-80003a8c04}
+    - PYSNMP_DEBUG=${PYSNMP_DEBUG}
     image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
     networks:
       - my_network

diff --git a/docker_compose/docker-compose-worker-poller.yaml b/docker_compose/docker-compose-worker-poller.yaml
@@ -42,6 +42,7 @@ services:
     - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60}
     - WORKER_CONCURRENCY=${WORKER_POLLER_CONCURRENCY:-2}
     - PREFETCH_COUNT=${PREFETCH_POLLER_COUNT:-1}
+    - PYSNMP_DEBUG=${PYSNMP_DEBUG}
     image: ${SC4SNMP_IMAGE}:${SC4SNMP_TAG:-latest}
     networks:
       - my_network

diff --git a/docker_compose/docker-compose-worker-sender.yaml b/docker_compose/docker-compose-worker-sender.yaml
@@ -41,6 +41,7 @@ services:
       - PROFILES_RELOAD_DELAY=${PROFILES_RELOAD_DELAY:-60}
       - WORKER_CONCURRENCY=${WORKER_SENDER_CONCURRENCY:-2}
       - PREFETCH_COUNT=${PREFETCH_SENDER_COUNT:-1}
+      - PYSNMP_DEBUG=${PYSNMP_DEBUG}
     depends_on:
       - redis
       - mongo

diff --git a/docker_compose/docker-compose-worker-trap.yaml b/docker_compose/docker-compose-worker-trap.yaml
@@ -44,6 +44,7 @@ services:
       - RESOLVE_TRAP_ADDRESS=${RESOLVE_TRAP_ADDRESS:-false}
       - MAX_DNS_CACHE_SIZE_TRAPS=${MAX_DNS_CACHE_SIZE_TRAPS:-500}
       - TTL_DNS_CACHE_TRAPS=${TTL_DNS_CACHE_TRAPS:-1800}
+      - PYSNMP_DEBUG=${PYSNMP_DEBUG}
     depends_on:
       - redis
       - mongo

diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/rendered/manifests/tests/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/...tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...s/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...s/tests_autoscaling_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...caling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/...caling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...caling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...caling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...oscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...oscaling_enabled_deprecated/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -65,6 +65,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/...manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...d/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...d/manifests/tests_enable_ui/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...ifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/...ifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...ifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...ifests/tests_only_polling/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...anifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...anifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/.../manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/.../manifests/tests_only_traps/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...ests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml b/...ests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/poller/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...ests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml b/...ests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/sender/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/...ifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml b/...ifests/tests_probes_enabled/splunk-connect-for-snmp/templates/worker/trap/deployment.yaml
@@ -66,6 +66,8 @@ spec:
               value: "3"
             - name: MAX_OID_TO_PROCESS
               value: "70"
+            - name: PYSNMP_DEBUG
+              value: ""
             - name: PROFILES_RELOAD_DELAY
               value: "60"
             - name: MIB_SOURCES

diff --git a/splunk_connect_for_snmp/snmp/manager.py b/splunk_connect_for_snmp/snmp/manager.py
@@ -63,6 +63,7 @@
 PROFILES_RELOAD_DELAY = int(os.getenv("PROFILES_RELOAD_DELAY", "60"))
 UDP_CONNECTION_TIMEOUT = int(os.getenv("UDP_CONNECTION_TIMEOUT", 3))
 MAX_OID_TO_PROCESS = int(os.getenv("MAX_OID_TO_PROCESS", 70))
+PYSNMP_DEBUG = os.getenv("PYSNMP_DEBUG", "")
 
 DEFAULT_STANDARD_MIBS = [
     "HOST-RESOURCES-MIB",
@@ -72,8 +73,29 @@
     "TCP-MIB",
     "UDP-MIB",
 ]
+
 logger = get_task_logger(__name__)
 
+if PYSNMP_DEBUG:
+    # Usage: PYSNMP_DEBUG=dsp,msgproc,io
+
+    # List of available debug flags:
+    # io, dsp, msgproc, secmod, mibbuild, mibview, mibinstrum, acl, proxy, app, all
+
+    from pysnmp import debug
+
+    debug_flags = list(debug.flagMap.keys())
+    enabled_debug_flags = [
+        debug_flag.strip()
+        for debug_flag in PYSNMP_DEBUG.split(",")
+        if debug_flag.strip() in debug_flags
+    ]
+
+    if enabled_debug_flags:
+        debug.setLogger(
+            debug.Debug(*enabled_debug_flags, options={"loggerName": logger})
+        )
+
 
 def return_address_and_port(target):
     if ":" in target:

diff --git a/splunk_connect_for_snmp/traps.py b/splunk_connect_for_snmp/traps.py
@@ -27,6 +27,7 @@
 
 import asyncio
 import os
+import sys
 from typing import Any, Dict
 
 import yaml
@@ -48,11 +49,39 @@
 SECURITY_ENGINE_ID_LIST = os.getenv("SNMP_V3_SECURITY_ENGINE_ID", "80003a8c04").split(
     ","
 )
+
 LOG_LEVEL = os.getenv("LOG_LEVEL", "INFO")
+PYSNMP_DEBUG = os.getenv("PYSNMP_DEBUG", "")
 
-logging.basicConfig(
-    level=getattr(logging, LOG_LEVEL), format="%(asctime)s %(levelname)s %(message)s"
-)
+logger = logging.getLogger(__name__)
+
+formatter = logging.Formatter("%(asctime)s %(levelname)s %(message)s")
+handler = logging.StreamHandler(sys.stdout)
+handler.setFormatter(formatter)
+handler.setLevel(getattr(logging, LOG_LEVEL))
+
+logger.addHandler(handler)
+
+
+if PYSNMP_DEBUG:
+    # Usage: PYSNMP_DEBUG=dsp,msgproc,io
+
+    # List of available debug flags:
+    # io, dsp, msgproc, secmod, mibbuild, mibview, mibinstrum, acl, proxy, app, all
+
+    from pysnmp import debug
+
+    debug_flags = list(debug.flagMap.keys())
+    enabled_debug_flags = [
+        debug_flag.strip()
+        for debug_flag in PYSNMP_DEBUG.split(",")
+        if debug_flag.strip() in debug_flags
+    ]
+
+    if enabled_debug_flags:
+        debug.setLogger(
+            debug.Debug(*enabled_debug_flags, options={"loggerName": logger})
+        )
 
 # //using rabbitmq as the message broker
 app = Celery("sc4snmp_traps")
@@ -68,7 +97,7 @@
 def cb_fun(
     snmp_engine, state_reference, context_engine_id, context_name, varbinds, cb_ctx
 ):
-    logging.debug(
+    logger.debug(
         'Notification from ContextEngineId "%s", ContextName "%s"'
         % (context_engine_id.prettyPrint(), context_name.prettyPrint())
     )
@@ -94,7 +123,7 @@ def cb_fun(
 
 # Callback function for logging traps authentication errors
 def authentication_observer_cb_fun(snmp_engine, execpoint, variables, contexts):
-    logging.error(
+    logger.error(
         f"Security Model failure for device {variables.get('transportAddress', None)}: "
         f"{variables.get('statusInformation', {}).get('errorIndication', None)}"
     )
@@ -154,13 +183,13 @@ def main():
             priv_key = get_secret_value(location, "privKey", required=False)
 
             auth_protocol = get_secret_value(location, "authProtocol", required=False)
-            logging.debug(f"authProtocol: {auth_protocol}")
+            logger.debug(f"authProtocol: {auth_protocol}")
             auth_protocol = AuthProtocolMap.get(auth_protocol.upper(), "NONE")
 
             priv_protocol = get_secret_value(
                 location, "privProtocol", required=False, default="NONE"
             )
-            logging.debug(f"privProtocol: {priv_protocol}")
+            logger.debug(f"privProtocol: {priv_protocol}")
             priv_protocol = PrivProtocolMap.get(priv_protocol.upper(), "NONE")
 
             for security_engine_id in SECURITY_ENGINE_ID_LIST:
@@ -173,7 +202,7 @@ def main():
                     privKey=priv_key,
                     securityEngineId=v2c.OctetString(hexValue=security_engine_id),
                 )
-                logging.debug(
+                logger.debug(
                     f"V3 users: {username} auth {auth_protocol} authkey {len(auth_key)*'*'} privprotocol {priv_protocol} "
                     f"privkey {len(priv_key)*'*'} securityEngineId {len(security_engine_id)*'*'}"
                 )