Store tables without yang (sonic-net#69)

* [sonic_yang_ext.py]: Rename _sonic_yang_ext.py as sonic_yang_ext.py.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic_yang_ext.py]: Store Tables without YANG models seperately for user warning.

Changes:
1.) Store Tables with YANG models seperately, so that a warning to user
can be issued with enough information.
2.) print the tables without YANG models properly in cropConfigDB().

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-mgmt]: Test for Store Tables without YANG models seperately for user warning.

Changes:
1.) Added config for extra tables in yangTest.json.
2.) Added new test case to find tables without yang models.
3.) Move common part in a helper function.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [src/sonic-yang-mgmt/sonic_yang_ext.py]: syslog extra tables.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

* [sonic-yang-mgmt/sonic_yang.py]: fix add_data_node in sonic_yang.

Changes:
-- fix add_data_node in sonic-yang.
-- Fixing tests of sonic-yang.

Signed-off-by: Praveen Chaudhary pchaudhary@linkedin.com

src/sonic-yang-mgmt/setup.py

@@ -90,7 +90,7 @@ def run (self):
     include_package_data=True,
     keywords='sonic_yang_mgmt',
     name='sonic_yang_mgmt',
-    py_modules=['sonic_yang', '_sonic_yang_ext'],
+    py_modules=['sonic_yang', 'sonic_yang_ext'],
     packages=find_packages(),
     setup_requires=setup_requirements,
     version='1.0',

src/sonic-yang-mgmt/sonic_yang.py

@@ -3,7 +3,7 @@
 
 from json import dump
 from glob import glob
-from _sonic_yang_ext import sonic_yang_ext_mixin
+from sonic_yang_ext import sonic_yang_ext_mixin
 
 """
 Yang schema and data tree python APIs based on libyang python
@@ -34,6 +34,8 @@ def __init__(self, yang_dir, debug=False):
         self.xlateJson = dict()
         # reverse translation from yang JSON, == config db json
         self.revXlateJson = dict()
+        # below dict store the input config tables which have no YANG models
+        self.tablesWithOutYang = dict()
 
         try:
             self.ctx = ly.Context(yang_dir)
@@ -368,7 +370,7 @@ def find_data_node_schema_xpath(self, data_xpath):
             return path
 
     """
-    add_node(): add a node to Yang schema or data tree
+    add_data_node(): add a node to Yang schema or data tree
     input:    xpath and value of the node to be added
     returns:  Exception if failed
     """
@@ -378,7 +380,7 @@ def add_data_node(self, data_xpath, value):
             #check if the node added to the data tree
             self.find_data_node(data_xpath)
         except Exception as e:
-            print("add_node(): Failed to add data node for xpath: " + str(data_xpath))
+            print("add_data_node(): Failed to add data node for xpath: " + str(data_xpath))
             self.fail(e)
 
     """

src/sonic-yang-mgmt/_sonic_yang_ext.py → src/sonic-yang-mgmt/sonic_yang_ext.py

@@ -1,6 +1,7 @@
 # This script is used as extension of sonic_yang class. It has methods of
 # class sonic_yang. A separate file is used to avoid a single large file.
 
+from __future__ import print_function
 import yang as ly
 import re
 import syslog
@@ -115,16 +116,23 @@ def get_module_TLC_container(self, table):
     """
     Crop config as per yang models,
     This Function crops from config only those TABLEs, for which yang models is
-    provided.
+    provided. The Tables without YANG models are stored in
+    self.tablesWithOutYangModels.
     """
-    def cropConfigDB(self, croppedFile=None, allowExtraTables=True):
+    def cropConfigDB(self, croppedFile=None):
 
         for table in self.jIn.keys():
             if table not in self.confDbYangMap:
-                if allowExtraTables:
-                    del self.jIn[table]
-                else:
-                    raise(Exception("No Yang Model Exist for {}".format(table)))
+                # store in tablesWithOutYang
+                self.tablesWithOutYang[table] = self.jIn[table]
+                del self.jIn[table]
+
+        if len(self.tablesWithOutYang):
+            print("Note: Below table(s) have no YANG models:")
+            self.sysLog(msg="Extra Tables in Config {}".format(self.tablesWithOutYang.keys))
+            for table in self.tablesWithOutYang.keys():
+                print(unicode(table), end=", ")
+            print()
 
         if croppedFile:
             with open(croppedFile, 'w') as f:
@@ -581,14 +589,15 @@ def findXpathList(self, xpath, list, keys):
     input:    data
     returns:  True - success   False - failed
     """
-    def load_data(self, configdbJson, allowExtraTables=True):
+    def load_data(self, configdbJson):
 
        try:
           self.jIn = configdbJson
-          # reset xlate
+          # reset xlate and tablesWithOutYang
           self.xlateJson = dict()
+          self.tablesWithOutYang = dict()
           # self.jIn will be cropped
-          self.cropConfigDB(allowExtraTables=allowExtraTables)
+          self.cropConfigDB()
           # xlated result will be in self.xlateJson
           self.xlateConfigDB()
           #print(self.xlateJson)

src/sonic-yang-mgmt/tests/libyang-python-tests/test_sonic_yang.py

@@ -261,20 +261,40 @@ def test_get_leafref_type_schema(self, yang_s, data):
             data_type = yang_s.get_leafref_type_schema(xpath)
             assert expected_type == data_type
 
-    def test_xlate_rev_xlate(self):
+    """
+    This is helper function to load YANG models for tests cases, which works
+    on Real SONiC Yang models. Mainly tests  for translation and reverse
+    translation.
+    """
+    @pytest.fixture(autouse=True, scope='class')
+    def sonic_yang_data(self):
+        sonic_yang_dir = "/sonic/src/sonic-yang-models/yang-models/"
+        sonic_yang_test_file = "/sonic/src/sonic-yang-models/tests/yang_model_tests/yangTest.json"
+
+        syc = sy.sonic_yang(sonic_yang_dir)
+        syc.loadYangModel()
+
+        sonic_yang_data = dict()
+        sonic_yang_data['yang_dir'] = sonic_yang_dir
+        sonic_yang_data['test_file'] = sonic_yang_test_file
+        sonic_yang_data['syc'] = syc
+
+        return sonic_yang_data
+
+    def test_xlate_rev_xlate(self, sonic_yang_data):
         # In this test, xlation and revXlation is tested with latest Sonic
         # YANG model.
+        test_file = sonic_yang_data['test_file']
+        syc = sonic_yang_data['syc']
 
-        yang_dir = "/sonic/src/sonic-yang-models/yang-models/"
-        yang_test_file = "/sonic/src/sonic-yang-models/tests/yang_model_tests/yangTest.json"
-        jIn = self.readIjsonInput(yang_test_file, 'SAMPLE_CONFIG_DB_JSON')
-        # load yang models
-        syc = sy.sonic_yang(yang_dir)
+        jIn = self.readIjsonInput(test_file, 'SAMPLE_CONFIG_DB_JSON')
 
-        syc.loadYangModel()
+        syc.load_data(json.loads(jIn))
 
         syc.load_data(json.loads(jIn))
 
+        # TODO: Make sure no extra table is loaded
+
         syc.get_data()
 
         if syc.jIn and syc.jIn == syc.revXlateJson:
@@ -286,5 +306,21 @@ def test_xlate_rev_xlate(self):
 
         return
 
+    def test_table_with_no_yang(self, sonic_yang_data):
+        # in this test, tables with no YANG models must be stored seperately
+        # by this library.
+        test_file = sonic_yang_data['test_file']
+        syc = sonic_yang_data['syc']
+
+        jIn = self.readIjsonInput(test_file, 'SAMPLE_CONFIG_DB_JSON_1')
+
+        syc.load_data(json.loads(jIn))
+
+        ty = syc.tablesWithOutYang
+
+        assert (len(ty) and "UNKNOWN_TABLE" in ty) == True
+
+        return
+
     def teardown_class(self):
         pass

src/sonic-yang-models/tests/yang_model_tests/yangTest.json

@@ -1255,10 +1255,76 @@
 				"family": "IPv4"
 			}
 		},
-		"CRM": {
-			"Config": {
-				"polling_interval": "0"
-			}
-		}
-	}
+        "BREAKOUT_CFG": {
+            "Ethernet0": {
+                "brkout_mode": "1x100G[40G]"
+            },
+            "Ethernet4": {
+                "brkout_mode": "4x25G"
+            },
+            "Ethernet8": {
+                "brkout_mode": "1x100G[40G]"
+            }
+        },
+        "VERSIONS": {
+            "DATABASE": {
+                "VERSION": "version_1_0_3"
+            }
+        },
+        "FLEX_COUNTER_TABLE": {
+            "PFCWD": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "PG_WATERMARK": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "PORT": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "QUEUE": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "QUEUE_WATERMARK": {
+                "FLEX_COUNTER_STATUS": "enable"
+            }
+        },
+        "CRM": {
+            "Config": {
+                "acl_counter_high_threshold": "85",
+                "acl_counter_low_threshold": "70",
+                "acl_counter_threshold_type": "percentage",
+                "polling_interval": "0"
+            }
+        }
+    },
+    "SAMPLE_CONFIG_DB_JSON_1": {
+        "FLEX_COUNTER_TABLE": {
+            "PFCWD": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "PG_WATERMARK": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "PORT": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "QUEUE": {
+                "FLEX_COUNTER_STATUS": "enable"
+            },
+            "QUEUE_WATERMARK": {
+                "FLEX_COUNTER_STATUS": "enable"
+            }
+        },
+        "CRM": {
+            "Config": {
+                "acl_counter_high_threshold": "85",
+                "acl_counter_low_threshold": "70",
+                "acl_counter_threshold_type": "percentage",
+                "polling_interval": "0"
+            }
+        },
+        "UNKNOWN_TABLE": {
+            "Error": "This Table is for testing, This Table does not have YANG models."
+        }
+    }
 }