diff --git a/src/sonic-yang-mgmt/_sonic_yang_ext.py b/src/sonic-yang-mgmt/_sonic_yang_ext.py index 63c61f60cbea..cbdcb196b8dc 100644 --- a/src/sonic-yang-mgmt/_sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/_sonic_yang_ext.py @@ -19,11 +19,15 @@ def loadYangModel(self): try: - yangDir = self.yang_dir - self.yangFiles = glob(yangDir +"/*.yang") + # get all files + self.yangFiles = glob(self.yang_dir +"/*.yang") + # load yang modules for file in self.yangFiles: - if (self.load_schema_module(file) == False): - return False + m = self.load_schema_module(file) + if m is not None: + self.logInFile("module: {} is loaded successfully".format(m.name())) + else: + raise(Exception("Could not load module {}".format(file))) # keep only modules name in self.yangFiles self.yangFiles = [f.split('/')[-1] for f in self.yangFiles] @@ -53,6 +57,7 @@ def loadJsonYangModel(self): if m is not None: xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT) self.yJson.append(parse(xml)) + self.logInFile("Parsed Json for {}".format(m.name())) except Exception as e: print('JSON conversion for yang models failed') raise e @@ -70,7 +75,8 @@ def createDBTableToModuleMap(self): for j in self.yJson: # get module name moduleName = j['module']['@name'] - if "sonic-head" in moduleName or "sonic-common" in moduleName: + # topLevelContainer does not exist in sonic-head and sonic-extension. + if "sonic-head" in moduleName or "sonic-extension" in moduleName: continue; # get all top level container topLevelContainer = j['module']['container'] @@ -248,35 +254,25 @@ def yangConvert(val): """ def xlateList(self, model, yang, config, table): - # TODO: define a keyExt dict as of now, but we should be able to extract - # this from YANG model extentions. - keyExt = { - "VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", - "VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", - "VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)", - "VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$", - "ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$", - "ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$", - "INTERFACE_LIST": "^(Ethernet[0-9]+)$", - "INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$", - "PORT_LIST": "^(Ethernet[0-9]+)$", - "LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$", - "LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$", - } #create a dict to map each key under primary key with a dict yang model. #This is done to improve performance of mapping from values of TABLEs in #config DB to leaf in YANG LIST. leafDict = self.createLeafDict(model) - keyRegEx = keyExt[model['@name']] + # fetch regex from YANG models. + keyRegEx = model['ext:key-regex-configdb-to-yang']['@value'] + # seperator `|` has special meaning in regex, so change it appropriately. + keyRegEx = re.sub('\|', '\\|', keyRegEx) + # get keys from YANG model list itself listKeys = model['key']['@value'] for pkey in config.keys(): try: vKey = None - self.logInFile("xlateList Extract pkey {}".format(pkey)) + self.logInFile("xlateList Extract pkey:{} regex:{} keyList:{}\ + ".format(pkey, keyRegEx, listKeys)) # Find and extracts key from each dict in config keyDict = self.extractKey(pkey, listKeys, keyRegEx) # fill rest of the values in keyDict @@ -290,8 +286,6 @@ def xlateList(self, model, yang, config, table): except Exception as e: self.logInFile("xlateList Exception {}".format(e)) - self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\ - "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey))) # with multilist, we continue matching other keys. continue @@ -333,6 +327,7 @@ def xlateContainer(self, model, yang, config, table): del yang[modelList['@name']] if len(configC): + self.logInFile("Alert: Remaining keys in Config", obj=configC, json=True) raise(Exception("All Keys are not parsed in {}".format(table))) return @@ -421,24 +416,9 @@ def revYangConvert(val): """ def revXlateList(self, model, yang, config, table): - # TODO: define a keyExt dict as of now, but we should be able to - # extract this from YANG model extentions. - keyExt = { - "VLAN_INTERFACE_IPPREFIX_LIST": "|", - "VLAN_INTERFACE_LIST": "", - "VLAN_MEMBER_LIST": "|", - "VLAN_LIST": "", - "ACL_RULE_LIST": "|", - "ACL_TABLE_LIST": "", - "INTERFACE_LIST": "", - "INTERFACE_IPPREFIX_LIST": "|", - "LOOPBACK_INTERFACE_LIST": "", - "LOOPBACK_INTERFACE_IPPREFIX_LIST": "|", - "PORT_LIST": "" - - } - - keyRegEx = keyExt[model['@name']] + # fetch regex from YANG models + keyRegEx = model['ext:key-regex-yang-to-configdb']['@value'] + # create a dict to map each key under primary key with a dict yang model. # This is done to improve performance of mapping from values of TABLEs in # config DB to leaf in YANG LIST. @@ -448,7 +428,9 @@ def revXlateList(self, model, yang, config, table): if "_LIST" in model['@name']: for entry in yang: # create key of config DB table + self.logInFile("revXlateList regex:{}".format(keyRegEx)) pkey, pkeydict = self.createKey(entry, keyRegEx) + self.logInFile("revXlateList pkey:{}".format(pkey)) config[pkey]= dict() # fill rest of the entries for key in entry: @@ -468,10 +450,12 @@ def revXlateContainer(self, model, yang, config, table): if isinstance(model['list'], dict): modelList = model['list'] # Pass matching list from Yang Json + self.logInFile("revXlateContainer {}".format(modelList['@name'])) self.revXlateList(modelList, yang[modelList['@name']], config, table) elif isinstance(model['list'], list): for modelList in model['list']: + self.logInFile("revXlateContainer {}".format(modelList['@name'])) self.revXlateList(modelList, yang[modelList['@name']], config, table) return @@ -494,6 +478,7 @@ def revXlateYangtoConfigDB(self, yangJ, cDbJson): cmap = self.confDbYangMap[table] cDbJson[table] = dict() #print(key + "--" + subkey) + self.logInFile("revXlateYangtoConfigDB {}".format(table)) self.revXlateContainer(cmap['container'], yangJ[module_top][container], \ cDbJson[table], table) diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index 82de2a00ecd0..f45b0aba1684 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -71,7 +71,7 @@ def logInFile(self, header="", obj=None, json=False): """ def load_schema_module(self, yang_file): try: - self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG) + return self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG) except Exception as e: print("Failed to load yang module file: " + yang_file) self.fail(e) diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index 474a3794237b..46c92d521bd0 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -7,12 +7,18 @@ import getopt import subprocess import glob +import logging from ijson import items as ijson_itmes test_path = os.path.dirname(os.path.abspath(__file__)) modules_path = os.path.dirname(test_path) sys.path.insert(0, modules_path) +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger("YANG-TEST") +log.setLevel(logging.INFO) +log.addHandler(logging.NullHandler()) + class Test_SonicYang(object): # class vars yang_test_file = "/sonic/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json" @@ -268,7 +274,7 @@ def test_xlate_rev_xlate(self): syc.get_data() - if syc.jIn == syc.revXlateJson: + if syc.jIn and syc.jIn == syc.revXlateJson: print("Xlate and Rev Xlate Passed") else: print("Xlate and Rev Xlate failed") diff --git a/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py b/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py index dddda2d50c08..d36416adcae7 100644 --- a/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py +++ b/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py @@ -7,6 +7,7 @@ import ijson import json #import sonic_yang as sy +from glob import glob from os import listdir from os.path import isfile, join, splitext @@ -134,27 +135,21 @@ def __init__(self, tests, yangDir, jsonFile): load all YANG models before test run """ def loadYangModel(self, yangDir): + try: - # get all files - yangFiles = [f for f in listdir(yangDir) if isfile(join(yangDir, f))] - # get all yang files - yangFiles = [f for f in yangFiles if splitext(f)[-1].lower()==".yang"] - yangFiles = [f.split('.')[0] for f in yangFiles] - # load yang mdoules + # create context self.ctx = ly.Context(yangDir) - log.debug(yangFiles) - for f in yangFiles: - # load a module - log.debug(f) - m = self.ctx.get_module(f) + # get all files + yangFiles = glob(yangDir +"/*.yang") + # load yang modules + for file in yangFiles: + log.debug(file) + m = self.ctx.parse_module_path(file, ly.LYS_IN_YANG) if m is not None: - log.error("Could not get module: {}".format(m.name())) + log.info("module: {} is loaded successfully".format(m.name())) else: - m = self.ctx.load_module(f) - if m is not None: - log.info("module: {} is loaded successfully".format(m.name())) - else: - return + log.info("Could not load module: {}".format(file)) + except Exception as e: printExceptionDetails() raise e @@ -275,7 +270,8 @@ def main(): yTest = YangModelTesting(tests, yangDir, jsonFile) if (listTests): - log.info(yTest.ExceptionTests.keys()) + for key in yTest.ExceptionTests.keys(): + log.info(key) sys.exit(0) ret = yTest.run() diff --git a/src/sonic-yang-mgmt/yang-models/sonic-acl.yang b/src/sonic-yang-mgmt/yang-models/sonic-acl.yang index 290744a0633e..37b16ef0a003 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-acl.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-acl.yang @@ -18,6 +18,11 @@ module sonic-acl { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -48,6 +53,10 @@ module sonic-acl { key "ACL_TABLE_NAME RULE_NAME"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)|([a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf ACL_TABLE_NAME { type leafref { path "/acl:sonic-acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME"; @@ -232,6 +241,10 @@ module sonic-acl { key "ACL_TABLE_NAME"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf ACL_TABLE_NAME { type string; } diff --git a/src/sonic-yang-mgmt/yang-models/sonic-head.yang b/src/sonic-yang-mgmt/yang-models/sonic-head.yang index f0422c1febcc..95e645c9de54 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-head.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-head.yang @@ -1,6 +1,8 @@ module sonic-head { - namespace "http://sonic-head"; + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-head"; prefix sonic-head; organization "Linkedin Corporation"; @@ -91,4 +93,9 @@ module sonic-head { enum priority_tagged; } } + + extension key-regex-configdb-to-yang { + description "Key regex used to convert config DB keys to YANG Config"; + argument "value"; + } } diff --git a/src/sonic-yang-mgmt/yang-models/sonic-interface.yang b/src/sonic-yang-mgmt/yang-models/sonic-interface.yang index 196e687a5838..f63f56285d9b 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-interface.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-interface.yang @@ -1,5 +1,7 @@ module sonic-interface { + yang-version 1.1; + namespace "http://github.com/Azure/sonic-interface"; prefix intf; @@ -16,6 +18,11 @@ module sonic-interface { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -43,6 +50,10 @@ module sonic-interface { key "port_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf port_name { type leafref { path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name; @@ -64,6 +75,10 @@ module sonic-interface { key "port_name ip-prefix"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)|([a-fA-F0-9:./]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf port_name { /* This node must be present in INTERFACE_LIST */ must "(current() = ../../INTERFACE_LIST[port_name=current()]/port_name)" diff --git a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang index d9bb55e5ab39..ca55e56511c7 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang @@ -12,6 +12,11 @@ module sonic-loopback-interface { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + organization "Linkedin Corporation"; contact "lnos_coders@linkedin.com"; @@ -30,6 +35,10 @@ module sonic-loopback-interface { list LOOPBACK_INTERFACE_LIST { key "loopback_interface_name"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf loopback_interface_name{ type string; } @@ -47,8 +56,11 @@ module sonic-loopback-interface { key "loopback_interface_name ip-prefix"; - leaf loopback_interface_name{ + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)|([a-fA-F0-9:./]+$)"; + ext:key-regex-yang-to-configdb "|"; + + leaf loopback_interface_name{ /* This node must be present in LOOPBACK_INTERFACE_LIST */ must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)" { diff --git a/src/sonic-yang-mgmt/yang-models/sonic-port.yang b/src/sonic-yang-mgmt/yang-models/sonic-port.yang index bbd8c1eb4e3b..25718be95a99 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-port.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-port.yang @@ -1,5 +1,7 @@ module sonic-port{ + yang-version 1.1; + namespace "http://github.com/Azure/sonic-port"; prefix port; @@ -16,6 +18,11 @@ module sonic-port{ revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + organization "Linkedin Corporation"; contact "lnos_coders@linkedin.com"; @@ -27,6 +34,7 @@ module sonic-port{ } container sonic-port{ + container PORT { description "PORT part of config_db.json"; @@ -35,6 +43,10 @@ module sonic-port{ key "port_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf port_name { type string { length 1..128; diff --git a/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang b/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang index 656351f4e635..a9bc5dc3b1bf 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang @@ -1,5 +1,7 @@ module sonic-portchannel { + yang-version 1.1; + namespace "http://github.com/Azure/sonic-portchannel"; prefix lag; @@ -16,6 +18,11 @@ module sonic-portchannel { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -40,6 +47,10 @@ module sonic-portchannel { key "portchannel_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf portchannel_name { type string { length 1..128; diff --git a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang index de52cb1f07b3..d345241c6f74 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang @@ -16,6 +16,11 @@ module sonic-vlan { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -43,6 +48,10 @@ module sonic-vlan { key "vlan_name"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf vlan_name { type leafref { path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:vlan_name; @@ -62,6 +71,10 @@ module sonic-vlan { key "vlan_name ip-prefix"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)|([a-fA-F0-9:./]+$)"; + + ext:key-regex-yang-to-configdb "|"; + leaf vlan_name { /* This node must be present in VLAN_INTERFACE_LIST */ must "(current() = ../../VLAN_INTERFACE_LIST[vlan_name=current()]/vlan_name)" @@ -113,6 +126,10 @@ module sonic-vlan { key "vlan_name"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf vlan_name { type string { pattern 'Vlan([0-9]{1,3}|[0-3][0-9]{4}|[4][0][0-8][0-9]|[4][0][9][0-4])'; @@ -165,6 +182,10 @@ module sonic-vlan { key "vlan_name port"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9-_]+)|(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf vlan_name { type leafref { path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:vlan_name";