From a65c1f319e027fc3bc12424f0ea23e13fefe40a4 Mon Sep 17 00:00:00 2001 From: Praveen Chaudhary Date: Mon, 16 Mar 2020 21:20:35 -0700 Subject: [PATCH] [yang-models]: Add yang extension to control traslation configDB<->yang from YANG models. (#52) Changes: 1.) Added yang extensions in YANG models. 2.) Changed the translation and rev translation code to depend on yang extensions. 3.) Load yang models in test code similar to dev code. 4.) Minor changes in test files. Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com --- src/sonic-yang-mgmt/_sonic_yang_ext.py | 69 ++++++++----------- src/sonic-yang-mgmt/sonic_yang.py | 2 +- .../libyang-python-tests/test_sonic_yang.py | 8 ++- .../yang-model-tests/yangModelTesting.py | 32 ++++----- .../yang-models/sonic-acl.yang | 13 ++++ .../yang-models/sonic-head.yang | 9 ++- .../yang-models/sonic-interface.yang | 15 ++++ .../yang-models/sonic-loopback-interface.yang | 14 +++- .../yang-models/sonic-port.yang | 12 ++++ .../yang-models/sonic-portchannel.yang | 11 +++ .../yang-models/sonic-vlan.yang | 21 ++++++ 11 files changed, 142 insertions(+), 64 deletions(-) diff --git a/src/sonic-yang-mgmt/_sonic_yang_ext.py b/src/sonic-yang-mgmt/_sonic_yang_ext.py index 63c61f60cbea..cbdcb196b8dc 100644 --- a/src/sonic-yang-mgmt/_sonic_yang_ext.py +++ b/src/sonic-yang-mgmt/_sonic_yang_ext.py @@ -19,11 +19,15 @@ def loadYangModel(self): try: - yangDir = self.yang_dir - self.yangFiles = glob(yangDir +"/*.yang") + # get all files + self.yangFiles = glob(self.yang_dir +"/*.yang") + # load yang modules for file in self.yangFiles: - if (self.load_schema_module(file) == False): - return False + m = self.load_schema_module(file) + if m is not None: + self.logInFile("module: {} is loaded successfully".format(m.name())) + else: + raise(Exception("Could not load module {}".format(file))) # keep only modules name in self.yangFiles self.yangFiles = [f.split('/')[-1] for f in self.yangFiles] @@ -53,6 +57,7 @@ def loadJsonYangModel(self): if m is not None: xml = m.print_mem(ly.LYD_JSON, ly.LYP_FORMAT) self.yJson.append(parse(xml)) + self.logInFile("Parsed Json for {}".format(m.name())) except Exception as e: print('JSON conversion for yang models failed') raise e @@ -70,7 +75,8 @@ def createDBTableToModuleMap(self): for j in self.yJson: # get module name moduleName = j['module']['@name'] - if "sonic-head" in moduleName or "sonic-common" in moduleName: + # topLevelContainer does not exist in sonic-head and sonic-extension. + if "sonic-head" in moduleName or "sonic-extension" in moduleName: continue; # get all top level container topLevelContainer = j['module']['container'] @@ -248,35 +254,25 @@ def yangConvert(val): """ def xlateList(self, model, yang, config, table): - # TODO: define a keyExt dict as of now, but we should be able to extract - # this from YANG model extentions. - keyExt = { - "VLAN_INTERFACE_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", - "VLAN_LIST": "^(Vlan[a-zA-Z0-9_-]+)$", - "VLAN_INTERFACE_IPPREFIX_LIST": "^(Vlan[a-zA-Z0-9_-]+)\|([a-fA-F0-9:./]+$)", - "VLAN_MEMBER_LIST": "^(Vlan[a-zA-Z0-9-_]+)\|(Ethernet[0-9]+)$", - "ACL_RULE_LIST": "^([a-zA-Z0-9_-]+)\|([a-zA-Z0-9_-]+)$", - "ACL_TABLE_LIST": "^([a-zA-Z0-9-_]+)$", - "INTERFACE_LIST": "^(Ethernet[0-9]+)$", - "INTERFACE_IPPREFIX_LIST": "^(Ethernet[0-9]+)\|([a-fA-F0-9:./]+)$", - "PORT_LIST": "^(Ethernet[0-9]+)$", - "LOOPBACK_INTERFACE_LIST": "^([a-zA-Z0-9-_]+)$", - "LOOPBACK_INTERFACE_IPPREFIX_LIST": "^([a-zA-Z0-9-_]+)\|([a-fA-F0-9:./]+)$", - } #create a dict to map each key under primary key with a dict yang model. #This is done to improve performance of mapping from values of TABLEs in #config DB to leaf in YANG LIST. leafDict = self.createLeafDict(model) - keyRegEx = keyExt[model['@name']] + # fetch regex from YANG models. + keyRegEx = model['ext:key-regex-configdb-to-yang']['@value'] + # seperator `|` has special meaning in regex, so change it appropriately. + keyRegEx = re.sub('\|', '\\|', keyRegEx) + # get keys from YANG model list itself listKeys = model['key']['@value'] for pkey in config.keys(): try: vKey = None - self.logInFile("xlateList Extract pkey {}".format(pkey)) + self.logInFile("xlateList Extract pkey:{} regex:{} keyList:{}\ + ".format(pkey, keyRegEx, listKeys)) # Find and extracts key from each dict in config keyDict = self.extractKey(pkey, listKeys, keyRegEx) # fill rest of the values in keyDict @@ -290,8 +286,6 @@ def xlateList(self, model, yang, config, table): except Exception as e: self.logInFile("xlateList Exception {}".format(e)) - self.logInFile("Exception while Config DB --> YANG: pkey:{}, "\ - "vKey:{}, value: {}".format(pkey, vKey, config[pkey].get(vKey))) # with multilist, we continue matching other keys. continue @@ -333,6 +327,7 @@ def xlateContainer(self, model, yang, config, table): del yang[modelList['@name']] if len(configC): + self.logInFile("Alert: Remaining keys in Config", obj=configC, json=True) raise(Exception("All Keys are not parsed in {}".format(table))) return @@ -421,24 +416,9 @@ def revYangConvert(val): """ def revXlateList(self, model, yang, config, table): - # TODO: define a keyExt dict as of now, but we should be able to - # extract this from YANG model extentions. - keyExt = { - "VLAN_INTERFACE_IPPREFIX_LIST": "|", - "VLAN_INTERFACE_LIST": "", - "VLAN_MEMBER_LIST": "|", - "VLAN_LIST": "", - "ACL_RULE_LIST": "|", - "ACL_TABLE_LIST": "", - "INTERFACE_LIST": "", - "INTERFACE_IPPREFIX_LIST": "|", - "LOOPBACK_INTERFACE_LIST": "", - "LOOPBACK_INTERFACE_IPPREFIX_LIST": "|", - "PORT_LIST": "" - - } - - keyRegEx = keyExt[model['@name']] + # fetch regex from YANG models + keyRegEx = model['ext:key-regex-yang-to-configdb']['@value'] + # create a dict to map each key under primary key with a dict yang model. # This is done to improve performance of mapping from values of TABLEs in # config DB to leaf in YANG LIST. @@ -448,7 +428,9 @@ def revXlateList(self, model, yang, config, table): if "_LIST" in model['@name']: for entry in yang: # create key of config DB table + self.logInFile("revXlateList regex:{}".format(keyRegEx)) pkey, pkeydict = self.createKey(entry, keyRegEx) + self.logInFile("revXlateList pkey:{}".format(pkey)) config[pkey]= dict() # fill rest of the entries for key in entry: @@ -468,10 +450,12 @@ def revXlateContainer(self, model, yang, config, table): if isinstance(model['list'], dict): modelList = model['list'] # Pass matching list from Yang Json + self.logInFile("revXlateContainer {}".format(modelList['@name'])) self.revXlateList(modelList, yang[modelList['@name']], config, table) elif isinstance(model['list'], list): for modelList in model['list']: + self.logInFile("revXlateContainer {}".format(modelList['@name'])) self.revXlateList(modelList, yang[modelList['@name']], config, table) return @@ -494,6 +478,7 @@ def revXlateYangtoConfigDB(self, yangJ, cDbJson): cmap = self.confDbYangMap[table] cDbJson[table] = dict() #print(key + "--" + subkey) + self.logInFile("revXlateYangtoConfigDB {}".format(table)) self.revXlateContainer(cmap['container'], yangJ[module_top][container], \ cDbJson[table], table) diff --git a/src/sonic-yang-mgmt/sonic_yang.py b/src/sonic-yang-mgmt/sonic_yang.py index 82de2a00ecd0..f45b0aba1684 100644 --- a/src/sonic-yang-mgmt/sonic_yang.py +++ b/src/sonic-yang-mgmt/sonic_yang.py @@ -71,7 +71,7 @@ def logInFile(self, header="", obj=None, json=False): """ def load_schema_module(self, yang_file): try: - self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG) + return self.ctx.parse_module_path(yang_file, ly.LYS_IN_YANG) except Exception as e: print("Failed to load yang module file: " + yang_file) self.fail(e) diff --git a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py index 474a3794237b..46c92d521bd0 100644 --- a/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py +++ b/src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py @@ -7,12 +7,18 @@ import getopt import subprocess import glob +import logging from ijson import items as ijson_itmes test_path = os.path.dirname(os.path.abspath(__file__)) modules_path = os.path.dirname(test_path) sys.path.insert(0, modules_path) +logging.basicConfig(level=logging.DEBUG) +log = logging.getLogger("YANG-TEST") +log.setLevel(logging.INFO) +log.addHandler(logging.NullHandler()) + class Test_SonicYang(object): # class vars yang_test_file = "/sonic/src/sonic-yang-mgmt/tests/yang-model-tests/yangTest.json" @@ -268,7 +274,7 @@ def test_xlate_rev_xlate(self): syc.get_data() - if syc.jIn == syc.revXlateJson: + if syc.jIn and syc.jIn == syc.revXlateJson: print("Xlate and Rev Xlate Passed") else: print("Xlate and Rev Xlate failed") diff --git a/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py b/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py index dddda2d50c08..d36416adcae7 100644 --- a/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py +++ b/src/sonic-yang-mgmt/tests/yang-model-tests/yangModelTesting.py @@ -7,6 +7,7 @@ import ijson import json #import sonic_yang as sy +from glob import glob from os import listdir from os.path import isfile, join, splitext @@ -134,27 +135,21 @@ def __init__(self, tests, yangDir, jsonFile): load all YANG models before test run """ def loadYangModel(self, yangDir): + try: - # get all files - yangFiles = [f for f in listdir(yangDir) if isfile(join(yangDir, f))] - # get all yang files - yangFiles = [f for f in yangFiles if splitext(f)[-1].lower()==".yang"] - yangFiles = [f.split('.')[0] for f in yangFiles] - # load yang mdoules + # create context self.ctx = ly.Context(yangDir) - log.debug(yangFiles) - for f in yangFiles: - # load a module - log.debug(f) - m = self.ctx.get_module(f) + # get all files + yangFiles = glob(yangDir +"/*.yang") + # load yang modules + for file in yangFiles: + log.debug(file) + m = self.ctx.parse_module_path(file, ly.LYS_IN_YANG) if m is not None: - log.error("Could not get module: {}".format(m.name())) + log.info("module: {} is loaded successfully".format(m.name())) else: - m = self.ctx.load_module(f) - if m is not None: - log.info("module: {} is loaded successfully".format(m.name())) - else: - return + log.info("Could not load module: {}".format(file)) + except Exception as e: printExceptionDetails() raise e @@ -275,7 +270,8 @@ def main(): yTest = YangModelTesting(tests, yangDir, jsonFile) if (listTests): - log.info(yTest.ExceptionTests.keys()) + for key in yTest.ExceptionTests.keys(): + log.info(key) sys.exit(0) ret = yTest.run() diff --git a/src/sonic-yang-mgmt/yang-models/sonic-acl.yang b/src/sonic-yang-mgmt/yang-models/sonic-acl.yang index 290744a0633e..37b16ef0a003 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-acl.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-acl.yang @@ -18,6 +18,11 @@ module sonic-acl { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -48,6 +53,10 @@ module sonic-acl { key "ACL_TABLE_NAME RULE_NAME"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9_-]+)|([a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf ACL_TABLE_NAME { type leafref { path "/acl:sonic-acl/acl:ACL_TABLE/acl:ACL_TABLE_LIST/acl:ACL_TABLE_NAME"; @@ -232,6 +241,10 @@ module sonic-acl { key "ACL_TABLE_NAME"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf ACL_TABLE_NAME { type string; } diff --git a/src/sonic-yang-mgmt/yang-models/sonic-head.yang b/src/sonic-yang-mgmt/yang-models/sonic-head.yang index f0422c1febcc..95e645c9de54 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-head.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-head.yang @@ -1,6 +1,8 @@ module sonic-head { - namespace "http://sonic-head"; + yang-version 1.1; + + namespace "http://github.com/Azure/sonic-head"; prefix sonic-head; organization "Linkedin Corporation"; @@ -91,4 +93,9 @@ module sonic-head { enum priority_tagged; } } + + extension key-regex-configdb-to-yang { + description "Key regex used to convert config DB keys to YANG Config"; + argument "value"; + } } diff --git a/src/sonic-yang-mgmt/yang-models/sonic-interface.yang b/src/sonic-yang-mgmt/yang-models/sonic-interface.yang index 196e687a5838..f63f56285d9b 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-interface.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-interface.yang @@ -1,5 +1,7 @@ module sonic-interface { + yang-version 1.1; + namespace "http://github.com/Azure/sonic-interface"; prefix intf; @@ -16,6 +18,11 @@ module sonic-interface { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -43,6 +50,10 @@ module sonic-interface { key "port_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf port_name { type leafref { path /port:sonic-port/port:PORT/port:PORT_LIST/port:port_name; @@ -64,6 +75,10 @@ module sonic-interface { key "port_name ip-prefix"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)|([a-fA-F0-9:./]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf port_name { /* This node must be present in INTERFACE_LIST */ must "(current() = ../../INTERFACE_LIST[port_name=current()]/port_name)" diff --git a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang index d9bb55e5ab39..ca55e56511c7 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-loopback-interface.yang @@ -12,6 +12,11 @@ module sonic-loopback-interface { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + organization "Linkedin Corporation"; contact "lnos_coders@linkedin.com"; @@ -30,6 +35,10 @@ module sonic-loopback-interface { list LOOPBACK_INTERFACE_LIST { key "loopback_interface_name"; + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf loopback_interface_name{ type string; } @@ -47,8 +56,11 @@ module sonic-loopback-interface { key "loopback_interface_name ip-prefix"; - leaf loopback_interface_name{ + ext:key-regex-configdb-to-yang "^([a-zA-Z0-9-_]+)|([a-fA-F0-9:./]+$)"; + ext:key-regex-yang-to-configdb "|"; + + leaf loopback_interface_name{ /* This node must be present in LOOPBACK_INTERFACE_LIST */ must "(current() = ../../LOOPBACK_INTERFACE_LIST[loopback_interface_name=current()]/loopback_interface_name)" { diff --git a/src/sonic-yang-mgmt/yang-models/sonic-port.yang b/src/sonic-yang-mgmt/yang-models/sonic-port.yang index bbd8c1eb4e3b..25718be95a99 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-port.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-port.yang @@ -1,5 +1,7 @@ module sonic-port{ + yang-version 1.1; + namespace "http://github.com/Azure/sonic-port"; prefix port; @@ -16,6 +18,11 @@ module sonic-port{ revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + organization "Linkedin Corporation"; contact "lnos_coders@linkedin.com"; @@ -27,6 +34,7 @@ module sonic-port{ } container sonic-port{ + container PORT { description "PORT part of config_db.json"; @@ -35,6 +43,10 @@ module sonic-port{ key "port_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf port_name { type string { length 1..128; diff --git a/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang b/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang index 656351f4e635..a9bc5dc3b1bf 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-portchannel.yang @@ -1,5 +1,7 @@ module sonic-portchannel { + yang-version 1.1; + namespace "http://github.com/Azure/sonic-portchannel"; prefix lag; @@ -16,6 +18,11 @@ module sonic-portchannel { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -40,6 +47,10 @@ module sonic-portchannel { key "portchannel_name"; + ext:key-regex-configdb-to-yang "^(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf portchannel_name { type string { length 1..128; diff --git a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang index de52cb1f07b3..d345241c6f74 100644 --- a/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang +++ b/src/sonic-yang-mgmt/yang-models/sonic-vlan.yang @@ -16,6 +16,11 @@ module sonic-vlan { revision-date 2019-07-01; } + import sonic-extension { + prefix ext; + revision-date 2019-07-01; + } + import sonic-port { prefix port; revision-date 2019-07-01; @@ -43,6 +48,10 @@ module sonic-vlan { key "vlan_name"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf vlan_name { type leafref { path /vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:vlan_name; @@ -62,6 +71,10 @@ module sonic-vlan { key "vlan_name ip-prefix"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)|([a-fA-F0-9:./]+$)"; + + ext:key-regex-yang-to-configdb "|"; + leaf vlan_name { /* This node must be present in VLAN_INTERFACE_LIST */ must "(current() = ../../VLAN_INTERFACE_LIST[vlan_name=current()]/vlan_name)" @@ -113,6 +126,10 @@ module sonic-vlan { key "vlan_name"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9_-]+)$"; + + ext:key-regex-yang-to-configdb ""; + leaf vlan_name { type string { pattern 'Vlan([0-9]{1,3}|[0-3][0-9]{4}|[4][0][0-8][0-9]|[4][0][9][0-4])'; @@ -165,6 +182,10 @@ module sonic-vlan { key "vlan_name port"; + ext:key-regex-configdb-to-yang "^(Vlan[a-zA-Z0-9-_]+)|(Ethernet[0-9]+)$"; + + ext:key-regex-yang-to-configdb "|"; + leaf vlan_name { type leafref { path "/vlan:sonic-vlan/vlan:VLAN/vlan:VLAN_LIST/vlan:vlan_name";