Skip to content

Commit

Permalink
[platform_tests/counterpoll]: Support multi-asic for counterpoll wate…
Browse files Browse the repository at this point in the history 
…rmark tests

* Update counterpoll watermark platform test to support multi-asic DUTs

BugLink: #14753
Signed-off-by: Liam Kearney <liamkearney@microsoft.com>
  • Loading branch information
@liamkearney-msft
liamkearney-msft committed Sep 27, 2024
1 parent 3e8708d commit 03188f5
Show file tree
Hide file tree
Showing 2 changed files with 90 additions and 64 deletions.
13 changes: 13 additions & 0 deletions tests/common/devices/sonic_asic.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -163,6 +163,19 @@ def run_sonic_db_cli_cmd(self, sonic_db_cmd):
cmd = "{} {}".format(self.sonic_db_cli, sonic_db_cmd)
return self.sonichost.command(cmd, verbose=False)

def sonic_db_get_keys(self, db_id, pattern):
"""
Get all keys for a given pattern in given redis database for a particular asic
:param db_id: ID of redis database
:param pattern: Redis key pattern
:return: A list of key name in string
"""
cmd = '{} KEYS \"{}\"'.format(db_id, pattern)
logger.debug('Getting keys from asic{} redis by command: {}'.format(self.asic_index, cmd))
output = self.run_sonic_db_cli_cmd(cmd)
content = output['stdout'].strip()
return content.split('\n') if content else None

def run_redis_cli_cmd(self, redis_cmd):
if self.namespace != DEFAULT_NAMESPACE:
redis_cli = "/usr/bin/redis-cli"
Expand Down
141 changes: 77 additions & 64 deletions tests/platform_tests/counterpoll/test_counterpoll_watermark.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,6 @@
from tests.common.config_reload import config_reload
from tests.common.fixtures.duthost_utils import backup_and_restore_config_db # noqa F401
from tests.common.helpers.assertions import pytest_assert
from tests.common.helpers.sonic_db import redis_get_keys
from tests.common.utilities import get_inventory_files, get_host_visible_vars
from tests.common.utilities import skip_release, wait_until
from tests.common.reboot import reboot
Expand Down Expand Up @@ -61,8 +60,13 @@ def dut_vars(duthosts, enum_rand_one_per_hwsku_hostname, request):
yield dut_vars


def get_keys_on_asics(duthost, db_id, key):
return {asic.asic_index: asic.sonic_db_get_keys(db_id, key) for asic in duthost.asics}


def check_counters_populated(duthost, key):
return bool(redis_get_keys(duthost, 'COUNTERS_DB', key))
keys = get_keys_on_asics(duthost, "COUNTERS_DB", key)
return bool(keys.values())


def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_per_hwsku_hostname, dut_vars,
Expand Down Expand Up @@ -132,51 +136,54 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_
for map_to_verify in maps_to_verify:
map_prefix = map_to_verify['prefix']
maps = map_to_verify[MAPS]
map_output = redis_get_keys(duthost, 'COUNTERS_DB', MAPS_LONG_PREFIX.format(map_prefix))
map = []
failed = ""
for map_entry in maps:
map.append(map_entry)
msg = "no {} maps found in COUNTERS_DB".format(map_prefix)
pytest_assert(map_output, msg)
for line in map_output:
try:
map.remove(line)
except ValueError:
failed = "MAP {} was not found in {} MAPS list".format(line, map_prefix)
pytest_assert("" == failed, failed)
pytest_assert(len(map) == 0, "{} maps mismatch, one or more queue was not found in redis COUNTERS_DB"
.format(map_prefix))
map_outputs = get_keys_on_asics(duthost, 'COUNTERS_DB', MAPS_LONG_PREFIX.format(map_prefix))
for asic_idx, map_output in map_outputs.items():
map = []
failed = ""
for map_entry in maps:
map.append(map_entry)
msg = "no {} maps found in COUNTERS_DB on asic{}".format(map_prefix, asic_idx)
pytest_assert(map_output, msg)
for line in map_output:
try:
map.remove(line)
except ValueError:
failed = "MAP {} was not found in {} MAPS list".format(line, map_prefix)
pytest_assert("" == failed, failed)
pytest_assert(len(map) == 0,
"{} maps mismatch, one or more queue was not found in redis COUNTERS_DB on asic{}"
.format(map_prefix, asic_idx))

failed_list = []
with allure.step("Verifying {} STATS in FLEX_COUNTER_DB on {}...".format(tested_counterpoll, duthost.hostname)):
stats_output = redis_get_keys(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix))
counted = 0
# build expected counterpoll stats vs unexpected
expected_types = []
unexpected_types = []
for counterpoll, v in list(RELEVANT_MAPS.items()):
types_to_check = v[CounterpollConstants.TYPE]
if counterpoll in tested_counterpoll:
for type in types_to_check:
expected_types.append(FLEX_COUNTER_PREFIX + type)
else:
for type in types_to_check:
unexpected_types.append(FLEX_COUNTER_PREFIX + type)
logging.info("expected types for for counterpoll {}:\n{}".format(tested_counterpoll, expected_types))
logging.info("unexpected types for for counterpoll {}:\n{}".format(tested_counterpoll, unexpected_types))
for line in stats_output:
for expected in expected_types:
if expected in line:
counted += 1
for unexpected in unexpected_types:
if unexpected in line:
failed_list.append("found for {} unexpected stat counter in FLEX_COUNTER_DB: {}"
.format(tested_counterpoll, line))
logging.info("counted {} {} STATs type in FLEX_COUNTER_DB on {}..."
.format(counted, tested_counterpoll, duthost.hostname))
pytest_assert(len(failed_list) == 0, failed_list)
pytest_assert(counted > 0, "counted {} for {}".format(counted, tested_counterpoll))
stats_outputs = get_keys_on_asics(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix))
for asic_idx, stats_output in stats_outputs.items():
counted = 0
# build expected counterpoll stats vs unexpected
expected_types = []
unexpected_types = []
for counterpoll, v in list(RELEVANT_MAPS.items()):
types_to_check = v[CounterpollConstants.TYPE]
if counterpoll in tested_counterpoll:
for type in types_to_check:
expected_types.append(FLEX_COUNTER_PREFIX + type)
else:
for type in types_to_check:
unexpected_types.append(FLEX_COUNTER_PREFIX + type)
logging.info("expected types for for counterpoll {}:\n{}".format(tested_counterpoll, expected_types))
logging.info("unexpected types for for counterpoll {}:\n{}".format(tested_counterpoll, unexpected_types))
for line in stats_output:
for expected in expected_types:
if expected in line:
counted += 1
for unexpected in unexpected_types:
if unexpected in line:
failed_list.append("found for {} unexpected stat counter in FLEX_COUNTER_DB on asic{}: {}"
.format(tested_counterpoll, asic_idx, line))
logging.info("counted {} {} STATs type in FLEX_COUNTER_DB on {} asic{}..."
.format(counted, tested_counterpoll, duthost.hostname, asic_idx))
pytest_assert(len(failed_list) == 0, failed_list)
pytest_assert(counted > 0, "counted {} for {}".format(counted, tested_counterpoll))

# for watermark only, also count stats with actual values in COUNTERS_DB
if CounterpollConstants.WATERMARK in tested_counterpoll:
Expand All @@ -193,19 +200,24 @@ def test_counterpoll_queue_watermark_pg_drop(duthosts, localhost, enum_rand_one_
# count FLEXCOUNTER_DB countrpolls and put in results dict key per countrpoll
with allure.step("check all counterpolls {} results on {} ...".format(RELEVANT_COUNTERPOLLS, duthost.hostname)):
for counterpoll in RELEVANT_COUNTERPOLLS:
counted_dict[counterpoll] = 0
counted_dict[counterpoll] = {}
for asic in duthost.asics:
counted_dict[counterpoll][asic.asic_index] = 0
for map_prefix in MAPS_PREFIX_FOR_ALL_COUNTERPOLLS:
stats_output = redis_get_keys(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix))
for line in stats_output:
for counterpoll, v in list(RELEVANT_MAPS.items()):
types_to_check = v[CounterpollConstants.TYPE]
for type in types_to_check:
if type in line:
counted_dict[counterpoll] += 1
stats_outputs = get_keys_on_asics(duthost, 'FLEX_COUNTER_DB', '*{}*'.format(map_prefix))
for asic_idx, stats_output in stats_outputs.items():
for line in stats_output:
for counterpoll, v in list(RELEVANT_MAPS.items()):
types_to_check = v[CounterpollConstants.TYPE]
for type in types_to_check:
if type in line:
counted_dict[counterpoll][asic_idx] += 1
logging.info("counted_dict {}".format(counted_dict))
# verify each queue/watermark/pg-drop counterpoll has stats in FLEX_COUNTER_DB
for counterpoll in RELEVANT_COUNTERPOLLS:
pytest_assert(counted_dict[counterpoll] > 0)
for asic in duthost.asics:
pytest_assert(counted_dict[counterpoll][asic.asic_index] > 0,
"No stats in FLEX_COUNTER_DB for {} asic{}".format(duthost.hostname, asic.asic_index))
# for watermark only, also count stats with actual values in COUNTERS_DB
if CounterpollConstants.WATERMARK in tested_counterpoll:
with allure.step("counting watermark STATS in FLEX_COUNTER_DB on {}...".format(duthost.hostname)):
Expand Down Expand Up @@ -236,14 +248,15 @@ def verify_counterpoll_status(duthost, counterpoll_list, expected):


def count_watermark_stats_in_counters_db(duthost):
watermark_stats_output = redis_get_keys(duthost, 'COUNTERS_DB', '*{}*'
.format(CounterpollConstants.WATERMARK.upper()))
watermark_stats = {}
for watermark_type in WATERMARK_COUNTERS_DB_STATS_TYPE:
watermark_stats[watermark_type] = 0
for line in watermark_stats_output:
if watermark_type in line:
watermark_stats[watermark_type] += 1
logging.info("watermark_stats {}".format(watermark_stats))
for k, v in list(watermark_stats.items()):
pytest_assert(v > 0, "watermark_stats {} in COUNTERS_DB: {}, expected > 0".format(k, v))
watermark_stats_output = get_keys_on_asics(duthost, 'COUNTERS_DB', '*{}*'
.format(CounterpollConstants.WATERMARK.upper()))
for asic_idx, output in watermark_stats_output.items():
watermark_stats = {}
for watermark_type in WATERMARK_COUNTERS_DB_STATS_TYPE:
watermark_stats[watermark_type] = 0
for line in output:
if watermark_type in line:
watermark_stats[watermark_type] += 1
logging.info("watermark_stats on {} {}".format(asic_idx, watermark_stats))
for k, v in list(watermark_stats.items()):
pytest_assert(v > 0, "watermark_stats {} in COUNTERS_DB on asic{}: {}, expected > 0".format(k, asic_idx, v))

0 comments on commit 03188f5

Please sign in to comment.