Merge branch 'master' of github.com:Azure/sonic-mgmt into zero_ports_2

.gitignore

@@ -14,3 +14,5 @@ tests/metadata
 
 # Dev tools
 .vscode/
+.idea/
+

ansible/TestbedProcessing.py

@@ -388,7 +388,13 @@ def makeLab(data, devices, testbed, outfile):
                 toWrite.write("[" + key + "]\n")
                 for host in value.get("host"):
                     entry = host
-                    dev = devices.get(host.lower())
+
+                    if host.lower() in devices:
+                        dev = devices.get(host.lower())
+                    elif host.lower() in testbed:
+                        dev = testbed.get(host.lower())
+                    else:
+                        dev = None
 
                     if "ptf" in key:
                         try: #get ansible host
@@ -458,8 +464,8 @@ def makeLab(data, devices, testbed, outfile):
 
                         if card_type != 'supervisor':
                            entry += "\tstart_switchid=" + str( start_switchid )
-                           if num_asic is not None:
-                              start_switchid += int( num_asic )
+                           if num_asics is not None:
+                              start_switchid += int( num_asics )
                            else:
                               start_switchid += 1
 
@@ -526,6 +532,13 @@ def makeLab(data, devices, testbed, outfile):
                         except AttributeError:
                             print("\t\t" + host + " max_cores not found")
 
+                        try: #get os
+                            os = dev.get("os")
+                            if os is not None:
+                               entry += "\tos=" + str( os )
+                        except AttributeError:
+                            print("\t\t" + host + " os not found")
+
                     toWrite.write(entry + "\n")
                 toWrite.write("\n")
 

ansible/fanout_connect.yml

@@ -12,6 +12,9 @@
             server: "{{ inventory_hostname|lower }}"
             server_port: "{{ external_port }}"
 
+        - set_fact: root_fanout_connect=true
+          when: root_fanout_connect is not defined
+
         - debug: msg="Connect {{ server }}:{{ server_port }} to {{ dut }}"
 
         - name: get the username running the deploy
@@ -26,4 +29,6 @@
         - set_fact: connect_leaf=false
 
         - include_tasks: roles/fanout/tasks/rootfanout_connect.yml
+          when: root_fanout_connect|bool == true
+
         when: external_port is defined

ansible/group_vars/sonic/variables

@@ -14,7 +14,7 @@ broadcom_th3_hwskus: ['DellEMC-Z9332f-M-O16C64',  'DellEMC-Z9332f-O32']
 
 mellanox_spc1_hwskus: [ 'ACS-MSN2700', 'ACS-MSN2740', 'ACS-MSN2100', 'ACS-MSN2410', 'ACS-MSN2010', 'Mellanox-SN2700', 'Mellanox-SN2700-D48C8' ]
 mellanox_spc2_hwskus: [ 'ACS-MSN3700', 'ACS-MSN3700C', 'ACS-MSN3800', 'Mellanox-SN3800-D112C8' , 'ACS-MSN3420']
-mellanox_spc3_hwskus: [ 'ACS-MSN4700', 'ACS-MSN4600C', 'ACS-MSN4410', 'Mellanox-SN4600C-D112C8']
+mellanox_spc3_hwskus: [ 'ACS-MSN4700', 'ACS-MSN4600', 'ACS-MSN4600C', 'ACS-MSN4410', 'Mellanox-SN4600C-D112C8']
 mellanox_hwskus: "{{ mellanox_spc1_hwskus + mellanox_spc2_hwskus + mellanox_spc3_hwskus }}"
 
 cavium_hwskus: [ "AS7512", "XP-SIM" ]

ansible/library/announce_routes.py

@@ -5,9 +5,11 @@
 import yaml
 import re
 import requests
+import time
 
 from ansible.module_utils.basic import *
 
+
 DOCUMENTATION = '''
 module:  announce_routes
 short_description: announce routes to exabgp processes running in PTF container
@@ -48,6 +50,17 @@
 IPV4_BASE_PORT = 5000
 IPV6_BASE_PORT = 6000
 
+def wait_for_http(host_ip, http_port, timeout=10):
+    """Waits for HTTP server to open. Tries until timeout is reached and returns whether localhost received HTTP response"""
+    started = False
+    tries = 0
+    while not started and tries < timeout:
+        if os.system("curl {}:{}".format(host_ip, http_port)) == 0:
+            started = True
+        tries += 1
+        time.sleep(1)
+
+    return started
 
 def get_topo_type(topo_name):
     pattern = re.compile(r'^(t0|t1|ptf|fullmesh|dualtor|t2|mgmttor)')
@@ -78,6 +91,7 @@ def announce_routes(ptf_ip, port, routes):
         else:
             messages.append("announce route {} next-hop {}".format(prefix, nexthop))
 
+    wait_for_http(ptf_ip, port, timeout=60)
     url = "http://%s:%d" % (ptf_ip, port)
     data = { "commands": ";".join(messages) }
     r = requests.post(url, data=data, timeout=90)
@@ -115,9 +129,9 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number,
                     spine_asn, leaf_asn_start, tor_asn_start,
                     nexthop, nexthop_v6,
                     tor_subnet_size, max_tor_subnet_number, topo,
-                    router_type = "leaf", tor_index=None, set_num=None):
+                    router_type = "leaf", tor_index=None, set_num=None, no_default_route=False):
     routes = []
-    if router_type != "tor":
+    if not no_default_route and router_type != "tor":
         default_route_as_path = get_uplink_router_as_path(router_type, spine_asn)
 
         if topo != "t2" or (topo == "t2" and router_type == "core"):
@@ -212,7 +226,7 @@ def generate_routes(family, podset_number, tor_number, tor_subnet_number,
     return routes
 
 
-def fib_t0(topo, ptf_ip):
+def fib_t0(topo, ptf_ip, no_default_route=False):
 
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
@@ -234,16 +248,18 @@ def fib_t0(topo, ptf_ip):
 
         routes_v4 = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
                                     spine_asn, leaf_asn_start, tor_asn_start,
-                                    nhipv4, nhipv4, tor_subnet_size, max_tor_subnet_number, "t0")
+                                    nhipv4, nhipv4, tor_subnet_size, max_tor_subnet_number, "t0",
+                                    no_default_route=no_default_route)
         routes_v6 = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
                                     spine_asn, leaf_asn_start, tor_asn_start,
-                                    nhipv6, nhipv6, tor_subnet_size, max_tor_subnet_number, "t0")
+                                    nhipv6, nhipv6, tor_subnet_size, max_tor_subnet_number, "t0",
+                                    no_default_route=no_default_route)
 
         announce_routes(ptf_ip, port, routes_v4)
         announce_routes(ptf_ip, port6, routes_v6)
 
 
-def fib_t1_lag(topo, ptf_ip):
+def fib_t1_lag(topo, ptf_ip, no_default_route=False):
 
     common_config = topo['configuration_properties'].get('common', {})
     podset_number = common_config.get("podset_number", PODSET_NUMBER)
@@ -275,11 +291,11 @@ def fib_t1_lag(topo, ptf_ip):
             routes_v4 = generate_routes("v4", podset_number, tor_number, tor_subnet_number,
                                         None, leaf_asn_start, tor_asn_start,
                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                        router_type=router_type, tor_index=tor_index)
+                                        router_type=router_type, tor_index=tor_index, no_default_route=no_default_route)
             routes_v6 = generate_routes("v6", podset_number, tor_number, tor_subnet_number,
                                         None, leaf_asn_start, tor_asn_start,
                                         nhipv4, nhipv6, tor_subnet_size, max_tor_subnet_number, "t1",
-                                        router_type=router_type, tor_index=tor_index)
+                                        router_type=router_type, tor_index=tor_index, no_default_route=no_default_route)
             announce_routes(ptf_ip, port, routes_v4)
             announce_routes(ptf_ip, port6, routes_v6)
 
@@ -419,14 +435,16 @@ def main():
     if not topo:
         module.fail_json(msg='Unable to load topology "{}"'.format(topo_name))
 
+    is_storage_backend = "backend" in topo_name
+
     topo_type = get_topo_type(topo_name)
 
     try:
         if topo_type == "t0":
-            fib_t0(topo, ptf_ip)
+            fib_t0(topo, ptf_ip, no_default_route=is_storage_backend)
             module.exit_json(changed=True)
         elif topo_type == "t1":
-            fib_t1_lag(topo, ptf_ip)
+            fib_t1_lag(topo, ptf_ip, no_default_route=is_storage_backend)
             module.exit_json(changed=True)
         elif topo_type == "t2":
             fib_t2_lag(topo, ptf_ip)

ansible/library/extract_log.py

@@ -77,11 +77,15 @@
 import gzip
 import re
 import sys
+import logging
+import logging.handlers
 from datetime import datetime
 from functools import cmp_to_key
 from ansible.module_utils.basic import *
 
 
+logger = logging.getLogger('ExtractLog')
+
 def extract_lines(directory, filename, target_string):
     path = os.path.join(directory, filename)
     file = None
@@ -234,9 +238,13 @@ def combine_logs_and_save(directory, filenames, start_string, target_filename):
 
 
 def extract_log(directory, prefixname, target_string, target_filename):
+    logger.debug("extract_log for start string {}".format(target_string.replace("start-", "")))
     filenames = list_files(directory, prefixname)
+    logger.debug("extract_log from files {}".format(filenames))
     file_with_latest_line, file_create_time, latest_line = extract_latest_line_with_string(directory, filenames, target_string)
+    logger.debug("extract_log start file {}".format(file_with_latest_line))
     files_to_copy = calculate_files_to_copy(filenames, file_with_latest_line)
+    logger.debug("extract_log subsequent files {}".format(files_to_copy))
     combine_logs_and_save(directory, files_to_copy, latest_line, target_filename)
 
 
@@ -250,7 +258,12 @@ def main():
         ),
         supports_check_mode=False)
 
+    handler = logging.handlers.SysLogHandler(address='/dev/log')
+    logger.addHandler(handler)
+    logger.setLevel(logging.DEBUG)
+
     p = module.params;
+
     try:
         extract_log(p['directory'], p['file_prefix'], p['start_string'], p['target_filename'])
     except:

ansible/library/lag_facts.py

@@ -3,6 +3,7 @@
 import json
 import sys
 from ansible.module_utils.basic import *
+from ansible.module_utils.multi_asic_utils import load_db_config
 try:
     from sonic_py_common import multi_asic
     NAMESPACE_LIST = multi_asic.get_namespace_list()
@@ -63,6 +64,8 @@ def get_po_names(self):
         '''
             Collect configured lag interface names
         '''
+        # load db config
+        load_db_config()
         for ns in NAMESPACE_LIST:
             rt, out, err = self.module.run_command("sonic-cfggen -m /etc/sonic/minigraph.xml {} -v \"PORTCHANNEL.keys() | join(' ')\"".format('-n ' + ns if ns else ''))
             if rt != 0:

ansible/module_utils/multi_asic_utils.py

@@ -0,0 +1,24 @@
+def load_db_config():
+    '''
+    Load the correct database config file:
+     - database_global.json for multi asic
+     - database_config.json for single asic
+    Loading database config file is not required for
+    201911 images so ignore import error or function
+    name error.
+    '''
+    try:
+        from sonic_py_common import multi_asic
+        from swsscommon import swsscommon
+        if multi_asic.is_multi_asic():
+            if not swsscommon.SonicDBConfig.isGlobalInit():
+                swsscommon.SonicDBConfig.load_sonic_global_db_config()
+        else:
+            if not swsscommon.SonicDBConfig.isInit():
+                swsscommon.SonicDBConfig.load_sonic_db_config()
+    except ImportError:
+        pass
+    except NameError:
+        pass
+    except AttributeError:
+        pass

ansible/module_utils/port_utils.py

@@ -15,6 +15,8 @@ def _port_alias_to_name_map_50G(all_ports, s100G_ports,):
 def get_port_alias_to_name_map(hwsku, asic_id=None):
     try:
         from sonic_py_common import multi_asic
+        from ansible.module_utils.multi_asic_utils  import load_db_config
+        load_db_config()
         namespace_list = multi_asic.get_all_namespaces()
         for key, list in namespace_list.items():
             asic_ids = []
@@ -204,7 +206,7 @@ def get_port_alias_to_name_map(hwsku, asic_id=None):
         else:
             for i in range(1,9):
                 port_alias_to_name_map["Ethernet1/%d" % i] = "Ethernet%d" % ((i - 1) * 4)
-    elif hwsku == "B6510-48VS8CQ" or "RA-B6510-48V8C":
+    elif hwsku == "B6510-48VS8CQ" or hwsku == "RA-B6510-48V8C":
         for i in range(1,49):
             port_alias_to_name_map["twentyfiveGigE0/%d" % i] = "Ethernet%d" % i
         for i in range(49,57):