Add custom message for pre/post sanity check results (#14755)

Expand custom message feature to find potentially risky tests and modify existing custom message to be expandable for future use.

Description of PR
Expand custom message feature to find potentially risky tests and modify existing custom message to be expandable for future use.

Approach
What is the motivation for this PR?
To enhance debugging by adding extra information into customMsg we already have, and also make it expandable so others can add debugging information to be utlised in ADE

co-authorized by: jianquanye@microsoft.com

tests/common/plugins/sanity_check/__init__.py

@@ -17,6 +17,15 @@
 logger = logging.getLogger(__name__)
 
 SUPPORTED_CHECKS = checks.CHECK_ITEMS
+DUT_CHEK_LIST = ['core_dump_check_pass', 'config_db_check_pass']
+CACHE_LIST = ['core_dump_check_pass', 'config_db_check_pass',
+              'pre_sanity_check_failed', 'post_sanity_check_failed',
+              'pre_sanity_recovered', 'post_sanity_recovered']
+
+
+def reset_cache_list(request):
+    for item in CACHE_LIST:
+        request.config.cache.set(item, None)
 
 
 def pytest_sessionfinish(session, exitstatus):
@@ -28,6 +37,8 @@ def pytest_sessionfinish(session, exitstatus):
         session.config.cache.set("pre_sanity_check_failed", None)
     if post_sanity_failed:
         session.config.cache.set("post_sanity_check_failed", None)
+    for key in CACHE_LIST:
+        session.config.cache.set(key, None)
 
     if pre_sanity_failed and not post_sanity_failed:
         session.exitstatus = constants.PRE_SANITY_CHECK_FAILED_RC
@@ -119,6 +130,49 @@ def do_checks(request, check_items, *args, **kwargs):
     return check_results
 
 
+@pytest.fixture(scope="module", autouse=True)
+def log_custom_msg(request):
+    yield
+    module_name = request.node.name
+    items = request.session.items
+    for item in items:
+        if item.module.__name__ + ".py" == module_name.split("/")[-1]:
+            customMsgDict = {}
+            dutChekResults = {}
+            for key in DUT_CHEK_LIST:
+                if request.config.cache.get(key, None) is False:
+                    dutChekResults[key] = False
+            if dutChekResults:
+                customMsgDict['DutChekResult'] = dutChekResults
+
+            # Check pre_sanity_checks results
+            preSanityCheckResults = {}
+            if request.config.cache.get("pre_sanity_check_failed", None):
+                preSanityCheckResults['pre_sanity_check_failed'] = True
+            # pre_sanity_recovered should be None in healthy case, record either True/False
+            if request.config.cache.get("pre_sanity_recovered", None) is not None:
+                preSanityCheckResults['pre_sanity_recovered'] = request.config.cache.get("pre_sanity_recovered", None)
+            if preSanityCheckResults:
+                customMsgDict['PreSanityCheckResults'] = preSanityCheckResults
+
+            # Check post_sanity_checks results
+            postSanityCheckResults = {}
+            if request.config.cache.get("post_sanity_check_failed", None):
+                postSanityCheckResults['post_sanity_check_failed'] = True
+            # post_sanity_recovered should be None in healthy case, record either True/False
+            if request.config.cache.get("post_sanity_recovered", None) is not None:
+                preSanityCheckResults['post_sanity_recovered'] = request.config.cache.get("post_sanity_recovered", None)
+            if postSanityCheckResults:
+                customMsgDict['PostSanityCheckResults'] = postSanityCheckResults
+
+            # if we have any custom message to log, append it to user_properties
+            if customMsgDict:
+                logger.debug("customMsgDict: {}".format(customMsgDict))
+                item.user_properties.append(('CustomMsg', json.dumps(customMsgDict)))
+
+    reset_cache_list(request)
+
+
 @pytest.fixture(scope="module")
 def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinfo):
     logger.info("Prepare sanity check")
@@ -269,8 +323,10 @@ def sanity_check_full(localhost, duthosts, request, fanouthosts, nbrhosts, tbinf
 def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, localhost, nbrhosts, check_items,
                                     recover_method, request, tbinfo, sanity_check_stage: str):
     cache_key = "pre_sanity_check_failed"
+    recovery_cache_key = "pre_sanity_recovered"
     if sanity_check_stage == STAGE_POST_TEST:
         cache_key = "post_sanity_check_failed"
+        recovery_cache_key = "post_sanity_recovered"
 
     try:
         dut_failed_results = defaultdict(list)
@@ -299,6 +355,7 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local
 
     except BaseException as e:
         request.config.cache.set(cache_key, True)
+        request.config.cache.set(recovery_cache_key, False)
 
         logger.error(f"Recovery of sanity check failed with exception: {repr(e)}")
         pt_assert(
@@ -313,13 +370,17 @@ def recover_on_sanity_check_failure(duthosts, failed_results, fanouthosts, local
     new_failed_results = [result for result in new_check_results if result['failed']]
     if new_failed_results:
         request.config.cache.set(cache_key, True)
+        request.config.cache.set(recovery_cache_key, False)
         pt_assert(False,
                   f"!!!!!!!!!!!!!!!! {sanity_check_stage} sanity check after recovery failed: !!!!!!!!!!!!!!!!\n"
                   f"{json.dumps(new_failed_results, indent=4, default=fallback_serializer)}")
+    # Record recovery success
+    request.config.cache.set(recovery_cache_key, True)
 
 
+# make sure teardown of log_custom_msg happens after sanity_check
 @pytest.fixture(scope="module", autouse=True)
-def sanity_check(request):
+def sanity_check(request, log_custom_msg):
     if request.config.option.skip_sanity:
         logger.info("Skip sanity check according to command line argument")
         yield

tests/conftest.py

@@ -2343,13 +2343,8 @@ def _remove_entry(table_name, key_name, config):
             logger.info("Core dump and config check passed for {}".format(module_name))
 
     if check_result:
-        items = request.session.items
-        for item in items:
-            if item.module.__name__ + ".py" == module_name.split("/")[-1]:
-                item.user_properties.append(('CustomMsg', json.dumps({'DutChekResult': {
-                    'core_dump_check_pass': core_dump_check_pass,
-                    'config_db_check_pass': config_db_check_pass
-                }})))
+        request.config.cache.set("core_dump_check_pass", core_dump_check_pass)
+        request.config.cache.set("config_db_check_pass", config_db_check_pass)
 
 
 @pytest.fixture(scope="function")