Skip to content

Commit

Permalink
[pytest] Skip testing the format of alerting message from Monit on 20…
Browse files Browse the repository at this point in the history
…2012 image (#3559)

What is the motivation for this PR?
Since Supervisord will replace Monit to do the monitoring of critical processes, this test needs skip the testbeds which were installed with 202012 or newer image version. At the same time, this test needs handle the error if the command sudo monit status 'lldp|lldpmgrd' returns the non-zero exit code.

I met the following error message when this PR (sonic-net/sonic-buildimage#7676) was tested on virtual testbed.

monit/test_monit_status.py::test_monit_status[vlab-03] PASSED            [ 50%]
monit/test_monit_status.py::test_monit_reporting_message[vlab-03] 
-------------------------------- live log call ---------------------------------
02:11:26 utilities.wait_until                     L0068 ERROR  | Exception caught while checking check_monit_last_output: IndexError('list index out of range',)
02:12:26 utilities.wait_until                     L0068 ERROR  | Exception caught while checking check_monit_last_output: IndexError('list index out of range',)
02:13:27 utilities.wait_until                     L0068 ERROR  | Exception caught while checking check_monit_last_output: IndexError('list index out of range',)
FAILED   

How did you do it?
I used the pytest_require(...) to skip the testbed which were installed 202012 or newer image version.

How did you verify/test it?
I verified this change on the testbed str-msn2700-03.

Any platform specific information?
N/A

Supported testbed topology if it's a new test case?
N/A
  • Loading branch information
yozhao101 committed Jun 1, 2021
1 parent e27602e commit 5634dd0
Showing 1 changed file with 66 additions and 7 deletions.
73 changes: 66 additions & 7 deletions tests/monit/test_monit_status.py
Original file line number Diff line number Diff line change
@@ -1,12 +1,14 @@
"""
Test the running status of Monit service
Test the running status and format of alerting message of Monit service.
"""
import logging

import pytest

from pkg_resources import parse_version
from tests.common.utilities import wait_until
from tests.common.helpers.assertions import pytest_assert
from tests.common.helpers.assertions import pytest_require

logger = logging.getLogger(__name__)

Expand All @@ -15,8 +17,19 @@
pytest.mark.disable_loganalyzer
]


@pytest.fixture
def disable_lldp(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
"""Stops `lldpmgrd` process at setup stage and restarts it at teardwon.
Args:
duthosts: The fixture returns list of DuTs.
enum_rand_one_per_hwsku_frontend_hostname: The fixture randomly pick up
a frontend DuT from testbed.
Returns:
None.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
duthost.command("docker exec lldp supervisorctl stop lldpmgrd")
if duthost.is_multi_asic:
Expand All @@ -26,23 +39,69 @@ def disable_lldp(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
if duthost.is_multi_asic:
duthost.command("docker exec lldp0 supervisorctl start lldpmgrd")


def check_monit_last_output(duthost):
monit_status_result = duthost.shell("sudo monit status \'lldp|lldpmgrd\'", module_ignore_errors=True)['stdout_lines']
indices = [i for i, s in enumerate(monit_status_result) if 'last output' in s]
monit_last_output = monit_status_result[indices[0]]
if duthost.is_multi_asic:
return "/usr/bin/lldpmgrd' is not running in host and in namespace asic0" in monit_last_output
"""Checks whether alerting message appears as output of command 'monit status' if
process `lldpmgrd` was stopped.
Args:
duthost: An AnsibleHost object of DuT.
Returns:
None.
"""
monit_status_result = duthost.shell("sudo monit status 'lldp|lldpmgrd'", module_ignore_errors=True)
exit_code = monit_status_result["rc"]
pytest_assert(exit_code == 0, "Failed to get Monit status of process 'lldpmgrd'!")

indices = [i for i, s in enumerate(monit_status_result["stdout_lines"]) if 'last output' in s]
if len(indices) > 0:
monit_last_output = monit_status_result["stdout_lines"][indices[0]]
if duthost.is_multi_asic:
return "/usr/bin/lldpmgrd' is not running in host and in namespace asic0" in monit_last_output
else:
return "/usr/bin/lldpmgrd' is not running in host" in monit_last_output
else:
return "/usr/bin/lldpmgrd' is not running in host" in monit_last_output
pytes.fail("Failed to get Monit last output of process 'lldpmgrd'!")


def test_monit_status(duthosts, enum_rand_one_per_hwsku_frontend_hostname):
"""Checks whether the Monit service was running or not.
Args:
duthosts: The fixture returns list of DuTs.
enum_rand_one_per_hwsku_frontend_hostname: The fixture randomly picks up
a frontend DuT from testbed.
Returns:
None.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]
monit_status_result = duthost.shell("sudo monit status", module_ignore_errors=True)

exit_code = monit_status_result["rc"]
pytest_assert(exit_code == 0, "Monit is either not running or not configured correctly")


def test_monit_reporting_message(duthosts, enum_rand_one_per_hwsku_frontend_hostname, disable_lldp):
"""Checks whether the format of alerting message from Monit is correct or not.
202012 and newer image version will be skipped for testing since Supervisord
replaced Monit to do the monitoring critical processes.
Args:
duthosts: The fixture returns list of DuTs.
enum_rand_one_per_hwsku_frontend_hostname: The fixture randomly pick up
a frontend DuT from testbed.
disable_lldp: The fixture function stops `lldpmgrd` process before testing
and restarts `lldpmgrd` process at teardown.
Returns:
None.
"""
duthost = duthosts[enum_rand_one_per_hwsku_frontend_hostname]

pytest_require("201811" in duthost.os_version or "201911" in duthost.os_version,
"Test is not supported for 202012 and newer image versions!")

if not wait_until(180, 60, check_monit_last_output, duthost):
pytest.fail("Expected Monit reporting message not found")

0 comments on commit 5634dd0

Please sign in to comment.