[dhcp-relay] make DHCP relay an extension (#6531)

- Why I did it
Make DHCP relay docker an extension. DHCP relay now carries dhcp relay commands CLI plugin and has a complete manifest.
It is installed as extension if INCLUDE_DHCP_REALY is set to y.

DEPENDS on #5939

- How I did it
Modify DHCP relay docker makefile and dockerfile. Make changes to sonic_debian_extension.j2 to install sonic packages.
I moved DHCP related CLI tests from sonic-utilities to DHCP relay docker.
This PR introduces a way to write a plugin as part of docker image and run the tests from cli-plugin-tests directory under docker directory.
The test result is available in target/docker-dhcp-relay.gz.log:

[ REASON ] :      target/docker-dhcp-relay.gz does not exist   NON-EXISTENT PREREQUISITES: docker-start target/docker-config-engine-buster.gz-load target/python-wheels/sonic_utilities-1.2-py3-none-any.whl-in
stall target/debs/buster/python3-swsscommon_1.0.0_amd64.deb-install
[ FLAGS  FILE    ] : []
[ FLAGS  DEPENDS ] : []
[ FLAGS  DIFF    ] : []
============================= test session starts ==============================
platform linux -- Python 3.7.3, pytest-3.10.1, py-1.7.0, pluggy-0.8.0 -- /usr/bin/python3
cachedir: .pytest_cache
rootdir: /sonic/dockers/docker-dhcp-relay/cli-plugin-tests, inifile:
plugins: cov-2.6.0
collecting ... collected 10 items

test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_plugin_registration PASSED [ 10%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_add_dhcp_relay_with_nonexist_vlanid PASSED [ 20%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_add_dhcp_relay_with_invalid_vlanid PASSED [ 30%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_add_dhcp_relay_with_invalid_ip PASSED [ 40%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_add_dhcp_relay_with_exist_ip PASSED [ 50%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_add_del_dhcp_relay_dest PASSED [ 60%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_remove_nonexist_dhcp_relay_dest PASSED [ 70%]
test_config_dhcp_relay.py::TestConfigVlanDhcpRelay::test_config_vlan_remove_dhcp_relay_dest_with_nonexist_vlanid PASSED [ 80%]
test_show_dhcp_relay.py::TestVlanDhcpRelay::test_plugin_registration PASSED [ 90%]
test_show_dhcp_relay.py::TestVlanDhcpRelay::test_dhcp_relay_column_output PASSED [100%]

=============================== warnings summary ===============================
/usr/local/lib/python3.7/dist-packages/tabulate.py:7
  /usr/local/lib/python3.7/dist-packages/tabulate.py:7: DeprecationWarning: Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated, and in 3.8 it will stop working
    from collections import namedtuple, Iterable

-- Docs: https://docs.pytest.org/en/latest/warnings.html
==================== 10 passed, 1 warnings in 0.35 seconds =====================

Makefile.work

@@ -270,6 +270,7 @@ SONIC_BUILD_INSTRUCTION :=  make \
                            HTTPS_PROXY=$(https_proxy) \
                            NO_PROXY=$(no_proxy) \
                            SONIC_INCLUDE_SYSTEM_TELEMETRY=$(INCLUDE_SYSTEM_TELEMETRY) \
+                           INCLUDE_DHCP_RELAY=$(INCLUDE_DHCP_RELAY) \
                            SONIC_INCLUDE_RESTAPI=$(INCLUDE_RESTAPI) \
                            TELEMETRY_WRITABLE=$(TELEMETRY_WRITABLE) \
                            EXTRA_DOCKER_TARGETS=$(EXTRA_DOCKER_TARGETS) \

dockers/docker-dhcp-relay/Dockerfile.j2

@@ -32,5 +32,6 @@ COPY ["docker_init.sh", "start.sh", "/usr/bin/"]
 COPY ["docker-dhcp-relay.supervisord.conf.j2", "port-name-alias-map.txt.j2", "wait_for_intf.sh.j2", "/usr/share/sonic/templates/"]
 COPY ["files/supervisor-proc-exit-listener", "/usr/bin"]
 COPY ["critical_processes", "/etc/supervisor"]
+COPY ["cli", "/cli/"]
 
 ENTRYPOINT ["/usr/bin/docker_init.sh"]

dockers/docker-dhcp-relay/cli-plugin-tests/conftest.py

@@ -0,0 +1,27 @@
+import pytest
+import mock_tables # lgtm [py/unused-import]
+from unittest import mock
+
+@pytest.fixture()
+def mock_cfgdb():
+    cfgdb = mock.Mock()
+    CONFIG = {
+        'VLAN': {
+            'Vlan1000': {
+                'dhcp_servers': ['192.0.0.1']
+            }
+        }
+    }
+
+    def get_entry(table, key):
+        return CONFIG[table][key]
+
+    def set_entry(table, key, data):
+        CONFIG[table].setdefault(key, {})
+        CONFIG[table][key] = data
+
+    cfgdb.get_entry = mock.Mock(side_effect=get_entry)
+    cfgdb.set_entry = mock.Mock(side_effect=set_entry)
+
+    yield cfgdb
+

dockers/docker-dhcp-relay/cli-plugin-tests/mock_tables.py

@@ -0,0 +1,154 @@
+# MONKEY PATCH!!!
+import json
+import os
+from unittest import mock
+
+import mockredis
+import redis
+import swsssdk
+from sonic_py_common import multi_asic
+from swsssdk import SonicDBConfig, SonicV2Connector, ConfigDBConnector, ConfigDBPipeConnector
+from swsscommon import swsscommon
+
+
+topo = None
+dedicated_dbs = {}
+
+def clean_up_config():
+    # Set SonicDBConfig variables to initial state
+    # so that it can be loaded with single or multiple
+    # namespaces before the test begins.
+    SonicDBConfig._sonic_db_config = {}
+    SonicDBConfig._sonic_db_global_config_init = False
+    SonicDBConfig._sonic_db_config_init = False
+
+def load_namespace_config():
+    # To support multi asic testing
+    # SonicDBConfig load_sonic_global_db_config
+    # is invoked to load multiple namespaces
+    clean_up_config()
+    SonicDBConfig.load_sonic_global_db_config(
+        global_db_file_path=os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), 'database_global.json'))
+
+def load_database_config():
+    # Load local database_config.json for single namespace test scenario
+    clean_up_config()
+    SonicDBConfig.load_sonic_db_config(
+        sonic_db_file_path=os.path.join(
+            os.path.dirname(os.path.abspath(__file__)), 'database_config.json'))
+
+
+_old_connect_SonicV2Connector = SonicV2Connector.connect
+
+def connect_SonicV2Connector(self, db_name, retry_on=True):
+    # add topo to kwargs for testing different topology
+    self.dbintf.redis_kwargs['topo'] = topo
+    # add the namespace to kwargs for testing multi asic
+    self.dbintf.redis_kwargs['namespace'] = self.namespace
+    # Mock DB filename for unit-test
+    global dedicated_dbs
+    if dedicated_dbs and dedicated_dbs.get(db_name):
+        self.dbintf.redis_kwargs['db_name'] = dedicated_dbs[db_name]
+    else:
+        self.dbintf.redis_kwargs['db_name'] = db_name
+    self.dbintf.redis_kwargs['decode_responses'] = True
+    _old_connect_SonicV2Connector(self, db_name, retry_on)
+
+def _subscribe_keyspace_notification(self, db_name, client):
+    pass
+
+
+def config_set(self, *args):
+    pass
+
+
+class MockPubSub:
+    def get_message(self):
+        return None
+
+    def psubscribe(self, *args, **kwargs):
+        pass
+
+    def __call__(self, *args, **kwargs):
+        return self
+
+    def listen(self):
+        return []
+
+    def punsubscribe(self, *args, **kwargs):
+        pass
+
+    def clear(self):
+        pass
+
+INPUT_DIR = os.path.dirname(os.path.abspath(__file__))
+
+
+class SwssSyncClient(mockredis.MockRedis):
+    def __init__(self, *args, **kwargs):
+        super(SwssSyncClient, self).__init__(strict=True, *args, **kwargs)
+        # Namespace is added in kwargs specifically for unit-test
+        # to identify the file path to load the db json files.
+        topo = kwargs.pop('topo')
+        namespace = kwargs.pop('namespace')
+        db_name = kwargs.pop('db_name')
+        self.decode_responses = kwargs.pop('decode_responses', False) == True
+        fname = db_name.lower() + ".json"
+        self.pubsub = MockPubSub()
+
+        if namespace is not None and namespace is not multi_asic.DEFAULT_NAMESPACE:
+            fname = os.path.join(INPUT_DIR, namespace, fname)
+        elif topo is not None:
+            fname = os.path.join(INPUT_DIR, topo, fname)
+        else:
+            fname = os.path.join(INPUT_DIR, fname)
+
+        if os.path.exists(fname):
+            with open(fname) as f:
+                js = json.load(f)
+                for k, v in js.items():
+                    if 'expireat' in v and 'ttl' in v and 'type' in v and 'value' in v:
+                        # database is in redis-dump format
+                        if v['type'] == 'hash':
+                            # ignore other types for now since sonic has hset keys only in the db
+                            for attr, value in v['value'].items():
+                                self.hset(k, attr, value)
+                    else:
+                        for attr, value in v.items():
+                            self.hset(k, attr, value)
+
+    # Patch mockredis/mockredis/client.py
+    # The offical implementation assume decode_responses=False
+    # Here we detect the option and decode after doing encode
+    def _encode(self, value):
+        "Return a bytestring representation of the value. Taken from redis-py connection.py"
+
+        value = super(SwssSyncClient, self)._encode(value)
+
+        if self.decode_responses:
+           return value.decode('utf-8')
+
+    # Patch mockredis/mockredis/client.py
+    # The official implementation will filter out keys with a slash '/'
+    # ref: https://github.com/locationlabs/mockredis/blob/master/mockredis/client.py
+    def keys(self, pattern='*'):
+        """Emulate keys."""
+        import fnmatch
+        import re
+
+        # Make regex out of glob styled pattern.
+        regex = fnmatch.translate(pattern)
+        regex = re.compile(regex)
+
+        # Find every key that matches the pattern
+        return [key for key in self.redis if regex.match(key)]
+
+
+swsssdk.interface.DBInterface._subscribe_keyspace_notification = _subscribe_keyspace_notification
+mockredis.MockRedis.config_set = config_set
+redis.StrictRedis = SwssSyncClient
+SonicV2Connector.connect = connect_SonicV2Connector
+swsscommon.SonicV2Connector = SonicV2Connector
+swsscommon.ConfigDBConnector = ConfigDBConnector
+swsscommon.ConfigDBPipeConnector = ConfigDBPipeConnector

dockers/docker-dhcp-relay/cli-plugin-tests/pytest.ini

@@ -0,0 +1,3 @@
+[pytest]
+addopts = --cov-config=.coveragerc --cov --cov-report html --cov-report term --cov-report xml --junitxml=test-results.xml -vv
+

dockers/docker-dhcp-relay/cli-plugin-tests/test_config_dhcp_relay.py

@@ -0,0 +1,141 @@
+import os
+import sys
+import traceback
+from unittest import mock
+
+from click.testing import CliRunner
+
+from utilities_common.db import Db
+
+import pytest
+
+sys.path.append('../cli/config/plugins/')
+import dhcp_relay
+
+config_vlan_add_dhcp_relay_output="""\
+Added DHCP relay destination address 192.0.0.100 to Vlan1000
+Restarting DHCP relay service...
+"""
+
+config_vlan_del_dhcp_relay_output="""\
+Removed DHCP relay destination address 192.0.0.100 from Vlan1000
+Restarting DHCP relay service...
+"""
+
+class TestConfigVlanDhcpRelay(object):
+    def test_plugin_registration(self):
+        cli = mock.MagicMock()
+        dhcp_relay.register(cli)
+        cli.commands['vlan'].add_command.assert_called_once_with(dhcp_relay.vlan_dhcp_relay)
+
+    def test_config_vlan_add_dhcp_relay_with_nonexist_vlanid(self):
+        runner = CliRunner()
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["add"],
+                                   ["1001", "192.0.0.100"])
+            print(result.exit_code)
+            print(result.output)
+            # traceback.print_tb(result.exc_info[2])
+            assert result.exit_code != 0
+            assert "Error: Vlan1001 doesn't exist" in result.output
+            assert mock_run_command.call_count == 0
+
+    def test_config_vlan_add_dhcp_relay_with_invalid_vlanid(self):
+        runner = CliRunner()
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["add"],
+                                   ["4096", "192.0.0.100"])
+            print(result.exit_code)
+            print(result.output)
+            # traceback.print_tb(result.exc_info[2])
+            assert result.exit_code != 0
+            assert "Error: Vlan4096 doesn't exist" in result.output
+            assert mock_run_command.call_count == 0
+
+    def test_config_vlan_add_dhcp_relay_with_invalid_ip(self):
+        runner = CliRunner()
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["add"],
+                                   ["1000", "192.0.0.1000"])
+            print(result.exit_code)
+            print(result.output)
+            # traceback.print_tb(result.exc_info[2])
+            assert result.exit_code != 0
+            assert "Error: 192.0.0.1000 is invalid IP address" in result.output
+            assert mock_run_command.call_count == 0
+
+    def test_config_vlan_add_dhcp_relay_with_exist_ip(self, mock_cfgdb):
+        runner = CliRunner()
+        db = Db()
+        db.cfgdb = mock_cfgdb
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["add"],
+                                   ["1000", "192.0.0.1"], obj=db)
+            print(result.exit_code)
+            print(result.output)
+            assert result.exit_code == 0
+            assert "192.0.0.1 is already a DHCP relay destination for Vlan1000" in result.output
+            assert mock_run_command.call_count == 0
+
+    def test_config_vlan_add_del_dhcp_relay_dest(self, mock_cfgdb):
+        runner = CliRunner()
+        db = Db()
+        db.cfgdb = mock_cfgdb
+
+        # add new relay dest
+        with mock.patch("utilities_common.cli.run_command") as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["add"],
+                                   ["1000", "192.0.0.100"], obj=db)
+            print(result.exit_code)
+            print(result.output)
+            assert result.exit_code == 0
+            assert result.output == config_vlan_add_dhcp_relay_output
+            assert mock_run_command.call_count == 3
+            db.cfgdb.set_entry.assert_called_once_with('VLAN', 'Vlan1000', {'dhcp_servers': ['192.0.0.1', '192.0.0.100']})
+
+        db.cfgdb.set_entry.reset_mock()
+
+        # del relay dest
+        with mock.patch("utilities_common.cli.run_command") as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["del"],
+                                   ["1000", "192.0.0.100"], obj=db)
+            print(result.exit_code)
+            print(result.output)
+            assert result.exit_code == 0
+            assert result.output == config_vlan_del_dhcp_relay_output
+            assert mock_run_command.call_count == 3
+            db.cfgdb.set_entry.assert_called_once_with('VLAN', 'Vlan1000', {'dhcp_servers': ['192.0.0.1']})
+
+    def test_config_vlan_remove_nonexist_dhcp_relay_dest(self, mock_cfgdb):
+        runner = CliRunner()
+        db = Db()
+        db.cfgdb = mock_cfgdb
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["del"],
+                                   ["1000", "192.0.0.100"], obj=db)
+            print(result.exit_code)
+            print(result.output)
+            # traceback.print_tb(result.exc_info[2])
+            assert result.exit_code != 0
+            assert "Error: 192.0.0.100 is not a DHCP relay destination for Vlan1000" in result.output
+            assert mock_run_command.call_count == 0
+
+    def test_config_vlan_remove_dhcp_relay_dest_with_nonexist_vlanid(self, mock_cfgdb):
+        runner = CliRunner()
+        db = Db()
+        db.cfgdb = mock_cfgdb
+
+        with mock.patch('utilities_common.cli.run_command') as mock_run_command:
+            result = runner.invoke(dhcp_relay.vlan_dhcp_relay.commands["del"],
+                                   ["1001", "192.0.0.1"], obj=Db)
+            print(result.exit_code)
+            print(result.output)
+            # traceback.print_tb(result.exc_info[2])
+            assert result.exit_code != 0
+            assert "Error: Vlan1001 doesn't exist" in result.output
+            assert mock_run_command.call_count == 0

dockers/docker-dhcp-relay/cli-plugin-tests/test_show_dhcp_relay.py

@@ -0,0 +1,28 @@
+import os
+import sys
+import traceback
+from unittest import mock
+
+from click.testing import CliRunner
+
+import show.vlan as vlan
+from utilities_common.db import Db
+
+sys.path.insert(0, '../cli/show/plugins/')
+import show_dhcp_relay
+
+
+class TestVlanDhcpRelay(object):
+    def test_plugin_registration(self):
+        cli = mock.MagicMock()
+        show_dhcp_relay.register(cli)
+        assert 'DHCP Helper Address' in dict(vlan.VlanBrief.COLUMNS)
+
+    def test_dhcp_relay_column_output(self):
+        ctx = (
+            ({'Vlan100': {'dhcp_servers': ['192.0.0.1', '192.168.0.2']}}, {}, {}),
+            (),
+        )
+        assert show_dhcp_relay.get_dhcp_helper_address(ctx, 'Vlan100') == '192.0.0.1\n192.168.0.2'
+
+