Updated libyang library to support more than 255 typedef statements

.travis.yml

@@ -8,10 +8,10 @@ matrix:
       dist: trusty
       go: 1.8
       group: edge
-    - os: osx
-      osx_image: xcode7.3
-      language: generic
-      go: 1.8
+    #- os: osx
+    #  osx_image: xcode7.3
+    #  language: generic
+    #  go: 1.8
     - env: DOCKER=true OS_TYPE=centos OS_VERSION=centos7
     - env: DOCKER=true OS_TYPE=ubuntu OS_VERSION=xenial
     #- env: DOCKER=true OS_TYPE=centos OS_VERSION=centos6.9 #TODO needs more work

profiles/test/ydktest-cpp.json

@@ -33,6 +33,7 @@
             "sdk/cpp/core/tests/models/ydktest-filterread@2015-11-17.yang",
             "sdk/cpp/core/tests/models/ydktest-sanity-augm@2015-11-17.yang",
             "sdk/cpp/core/tests/models/ydktest-sanity-submodule@2016-04-25.yang",
+            "sdk/cpp/core/tests/models/ydktest-sanity-typedefs@2018-01-30.yang",
             "sdk/cpp/core/tests/models/ydktest-sanity-types@2016-04-11.yang",
             "sdk/cpp/core/tests/models/ydktest-sanity@2015-11-17.yang",
             "sdk/cpp/core/tests/models/ydktest-types@2016-05-23.yang",

sdk/cpp/core/CMakeLists.txt

@@ -279,13 +279,16 @@ set(netconf_objs
 
 # Set LibYang variables
 set(libyang_location ${CMAKE_CURRENT_BINARY_DIR}/project_libyang/src/project_libyang)
+set(libyang_build_location ${CMAKE_CURRENT_BINARY_DIR}/project_libyang/src/project_libyang-build)
 
 set(libyang_headers
     ${libyang_location}/src/common.h
     ${libyang_location}/src/context.h
     ${libyang_location}/src/dict.h
     ${libyang_location}/src/dict_private.h
-    ${libyang_location}/src/libyang.h
+    ${libyang_location}/src/extensions.h
+    ${libyang_build_location}/src/extensions_config.h
+    ${libyang_build_location}/src/libyang.h
     ${libyang_location}/src/parser.h
     ${libyang_location}/src/parser_yang.h
     ${libyang_location}/src/parser_yang_bis.h
@@ -305,6 +308,7 @@ set(yang_objs
     common.c.o
     context.c.o
     dict.c.o
+    extensions.c.o
     log.c.o
     parser.c.o
     parser_json.c.o
@@ -336,6 +340,7 @@ find_library(xslt_location xslt)
 find_library(pcre_location pcre)
 find_library(pthread_location pthread)
 find_library(dl_location dl)
+find_library(m_location m)
 find_package(LibXml2 REQUIRED)
 find_package(LibSSH REQUIRED)
 
@@ -378,13 +383,21 @@ add_library(libnetconf STATIC IMPORTED)
 set_property(TARGET libnetconf PROPERTY IMPORTED_LOCATION ${libnetconf_archive})
 add_dependencies(libnetconf project_libnetconf)
 
+if(LIBYANG_PLUGINS_DIR)
+    set(LIBYANG_EXT_PLUGINS_DIR ${LIBYANG_PLUGINS_DIR})
+else()
+    set(LIBYANG_EXT_PLUGINS_DIR ${LIB_INSTALL_DIR}/libyang)
+endif()
+set(LIBYANG_PLUGINS_INSTALL_DIR ${CPACK_PACKAGE_INSTALL_DIRECTORY}/${LIBYANG_EXT_PLUGINS_DIR})
+message("Set libyang plugin installation directory to: ${LIBYANG_PLUGINS_INSTALL_DIR}")
+
 # Add LibYang
 set(libyang_include_location include/libyang)
 set(libyang_headers_location ${CMAKE_CURRENT_BINARY_DIR}/project_libyang/headers)
 ExternalProject_Add(project_libyang
     PREFIX "${CMAKE_CURRENT_BINARY_DIR}/project_libyang"
-    URL "https://github.com/manradhaCisco/libyang/archive/master.zip"
-    CMAKE_ARGS -DSTATIC=true -DCMAKE_BUILD_TYPE=Debug
+    URL "https://github.com/ygorelik/libyang/archive/master.zip"
+    CMAKE_ARGS -DSTATIC=true -DCMAKE_BUILD_TYPE=Debug -DPLUGINS_DIR=${LIBYANG_PLUGINS_INSTALL_DIR}
     INSTALL_DIR "${libyang_location}"
     INSTALL_COMMAND mkdir -p ${libyang_headers_location}/libyang && cp ${libyang_headers} ${libyang_headers_location}/libyang
 )
@@ -437,13 +450,18 @@ target_link_libraries(ydk
     ${pcre_location}
     ${xslt_location}
     ${pthread_location}
-    ${dl_location}
     libnetconf
     libyang
+    ${dl_location}
 )
 
 set_property(TARGET ydk PROPERTY CXX_STANDARD 11)
 set_property(TARGET ydk PROPERTY CXX_STANDARD_REQUIRED ON)
+set(libyang_extension_lib 
+	${libyang_build_location}/src/extensions/libyang_ext_test${CMAKE_SHARED_LIBRARY_SUFFIX}
+	${libyang_build_location}/src/extensions/metadata${CMAKE_SHARED_LIBRARY_SUFFIX}
+	${libyang_build_location}/src/extensions/nacm${CMAKE_SHARED_LIBRARY_SUFFIX}
+)
 
 install(TARGETS ydk DESTINATION ${LIB_INSTALL_DIR})
 install(FILES ${libydk_install_headers} DESTINATION ${INCLUDE_INSTALL_DIR})
@@ -452,6 +470,7 @@ install(FILES ${SPDLOG_DETAILS_HEADERS} DESTINATION ${spdlog_details_include_loc
 install(FILES ${SPDLOG_FMT_HEADERS} DESTINATION ${spdlog_fmt_include_location})
 install(FILES ${SPDLOG_FMT_BUNDLED_HEADERS} DESTINATION ${spdlog_fmt_bundled_include_location})
 install(FILES ${SPDLOG_SINKS_HEADERS} DESTINATION ${spdlog_sinks_include_location})
+install(FILES ${libyang_extension_lib} DESTINATION ${LIBYANG_EXT_PLUGINS_DIR})
 
 # generate doxygen documentation for ydk_core API
 find_package(Doxygen)

sdk/cpp/core/src/netconf_ssh_client.cpp

@@ -110,7 +110,7 @@ int NetconfSSHClient::connect()
 std::string NetconfSSHClient::get_hostname_port()
 {
     ostringstream os;
-    os<<hostname<<":"<<port;
+    os<<hostname<<"_"<<port;
     return os.str();
 }
 

sdk/cpp/core/src/netconf_tcp_client.cpp

@@ -148,7 +148,7 @@ std::vector<std::string> NetconfTCPClient::get_capabilities()
 std::string NetconfTCPClient::get_hostname_port()
 {
     std::ostringstream os;
-    os << hostname << ":" << port;
+    os << hostname << "_" << port;
     return os.str();
 }
 

sdk/cpp/core/src/path/data_node.cpp

@@ -308,20 +308,15 @@ ydk::path::DataNodeImpl::find(const std::string& path)
     if(m_node == nullptr) {
         return results;
     }
-    std::string spath{path};
 
-    auto s = get_schema_node().get_statement();
-    if(s.keyword == "rpc"){
-        spath="input/" + spath;
-    }
-    YLOG_DEBUG("Getting child schema with path '{}' in {}", spath, m_node->schema->name);
+    YLOG_DEBUG("Getting child schema with path '{}' in {}", path, m_node->schema->name);
     const lys_node* found_snode =
-        ly_ctx_get_node(m_node->schema->module->ctx, m_node->schema, spath.c_str());
+        ly_ctx_get_node(m_node->schema->module->ctx, m_node->schema, path.c_str(), 0);
 
     if(found_snode)
     {
         YLOG_DEBUG("Getting data nodes with path '{}'", path);
-        ly_set* result_set = lyd_find_xpath(m_node, path.c_str());
+        ly_set* result_set = lyd_find_path(m_node, path.c_str());
         if( result_set )
         {
             if (result_set->number > 0)
@@ -334,7 +329,6 @@ ydk::path::DataNodeImpl::find(const std::string& path)
             }
             ly_set_free(result_set);
         }
-
     }
 
     return results;
@@ -484,13 +478,16 @@ ydk::path::DataNodeImpl::remove_annotation(const ydk::path::Annotation& an)
 
     lyd_attr* attr = m_node->attr;
     while(attr){
-        lys_module *module = attr->module;
-        if(module){
-            Annotation an1{module->ns, attr->name, attr->value};
-            if (an == an1){
-                lyd_free_attr(m_node->schema->module->ctx, m_node, attr, 0);
-                return true;
-            }
+        lyd_node* node = attr->parent;
+        if (node && node->schema) {
+			lys_module* module = node->schema->module;
+			if(module){
+				Annotation an1{module->ns, attr->name, attr->value_str};
+				if (an == an1){
+					lyd_free_attr(m_node->schema->module->ctx, m_node, attr, 0);
+					return true;
+				}
+			}
         }
     }
 
@@ -505,15 +502,16 @@ ydk::path::DataNodeImpl::annotations()
     if(m_node) {
         lyd_attr* attr = m_node->attr;
         while(attr) {
-            lys_module *module = attr->module;
-            if(module) {
-                ann.emplace_back(module->ns, attr->name, attr->value);
-
+            lyd_node* node = attr->parent;
+            if (node && node->schema) {
+    			lys_module* module = node->schema->module;
+				if(module) {
+					ann.emplace_back(module->ns, attr->name, attr->value_str);
+				}
             }
             attr = attr->next;
         }
     }
 
-
     return ann;
 }

sdk/cpp/core/src/path/path.cpp

@@ -165,11 +165,9 @@ ydk::path::ValidationService::validate(const ydk::path::DataNode & dn, ydk::Vali
             break;
         case ydk::ValidationService::Option::GET_CONFIG:
             option_str="GET-CONFIG";
-            ly_option = LYD_OPT_GETCONFIG;
+            ly_option = LYD_OPT_GETCONFIG | LYD_OPT_NOAUTODEL;
             break;
-
     }
-    ly_option = ly_option | LYD_OPT_NOAUTODEL;
 
     YLOG_INFO("Validation called on {} with option {}", dn.get_path(), option_str);
 

sdk/cpp/core/src/path/repository.cpp

@@ -41,8 +41,9 @@ namespace ydk
 {
 static bool file_exists(const std::string & path)
 {
-    struct stat st = {0};
+    struct stat st;
 
+    memset( &st, 0, sizeof(struct stat));
     return stat(path.c_str(), &st) == 0;
 }
 
@@ -279,7 +280,7 @@ ly_ctx* ydk::path::RepositoryPtr::create_ly_context()
         YLOG_INFO("Path where models are to be downloaded: {}", path);
     }
     YLOG_DEBUG("Creating libyang context in path: {}", path);
-    struct ly_ctx* ctx = ly_ctx_new(path.c_str());
+    struct ly_ctx* ctx = ly_ctx_new(path.c_str(), LY_CTX_ALLIMPLEMENTED);
 
     if(!ctx) {
         YLOG_ERROR("Could not create repository in: {}", path);
@@ -289,7 +290,7 @@ ly_ctx* ydk::path::RepositoryPtr::create_ly_context()
     //set module callback (only if there is a model provider)
     if(!model_providers.empty())
     {
-        ly_ctx_set_module_clb(ctx, get_module_callback, this);
+        ly_ctx_set_module_imp_clb(ctx, get_module_callback, this);
     }
 
     return ctx;
@@ -462,11 +463,11 @@ ydk::path::RepositoryPtr::load_module(ly_ctx* ctx, const std::string& module, co
 
     YLOG_DEBUG("Module '{}' Revision '{}'", module.c_str(), revision.c_str());
 
-    auto p = ly_ctx_get_module(ctx, module.c_str(), revision.empty() ? 0 : revision.c_str());
+    auto p = ly_ctx_get_module(ctx, module.c_str(), revision.empty() ? NULL : revision.c_str(), 1);
 
     if(!p)
     {
-        p = ly_ctx_load_module(ctx, module.c_str(), revision.empty() ? 0 : revision.c_str());
+        p = ly_ctx_load_module(ctx, module.c_str(), revision.empty() ? NULL : revision.c_str());
     } else {
         YLOG_DEBUG("Cache hit Module '{}' Revision '{}'", module, revision);
         new_module = false;

sdk/cpp/core/src/path/root_data_node.cpp

@@ -220,7 +220,7 @@ ydk::path::RootDataImpl::find(const std::string& path)
     schema_path+=path;
 
     YLOG_DEBUG("Looking for schema nodes path in root: '{}'", schema_path);
-    const struct lys_node* found_snode = ly_ctx_get_node(m_node->schema->module->ctx, nullptr, schema_path.c_str());
+    const struct lys_node* found_snode = ly_ctx_get_node(m_node->schema->module->ctx, nullptr, schema_path.c_str(), 1);
 
     if(found_snode)
     {

sdk/cpp/core/src/path/root_schema_node.cpp

@@ -39,6 +39,9 @@ static void get_namespaces_from_xml_doc(xmlNodePtr root, std::unordered_set<std:
         if (curr->type == XML_ELEMENT_NODE && curr->ns && curr->ns->href)
         {
             namespaces.insert(std::string{reinterpret_cast<const char*>(curr->ns->href)});
+            if (curr->nsDef && curr->nsDef->href) {
+            	namespaces.insert(std::string{reinterpret_cast<const char*>(curr->nsDef->href)});
+            }
         }
         get_namespaces_from_xml_doc(curr->children, namespaces);
     }
@@ -316,7 +319,7 @@ ydk::path::RootSchemaNodeImpl::find(const std::string& path)
     std::string full_path{"/"};
     full_path+=path;
 
-    const struct lys_node* found_node = ly_ctx_get_node(m_ctx, nullptr, full_path.c_str());
+    const struct lys_node* found_node = ly_ctx_get_node(m_ctx, nullptr, full_path.c_str(), 0);
 
     if (found_node){
         auto p = reinterpret_cast<SchemaNode*>(found_node->priv);

sdk/cpp/core/src/path/rpc.cpp

@@ -104,17 +104,24 @@ static bool is_part_of_output(lys_node* node_result)
 
 bool ydk::path::RpcImpl::has_output_node() const
 {
+    std::string node_path = lys_path( data_node->m_node->schema);
+    std::string search_path = node_path + "//*";	// Patterns includes only descendants of the node
+
     ly_verb(LY_LLSILENT); //turn off libyang logging at the beginning
-    ly_set* result_set = lys_find_xpath(data_node->m_node->schema, "*", LYS_FIND_OUTPUT);
+    ly_set* result_set = lys_find_path(data_node->m_node->schema->module, data_node->m_node->schema, search_path.c_str());
     ly_verb(LY_LLVRB); // enable libyang logging after find has completed
+
+    auto result = false;
     if(result_set && result_set->number > 0)
     {
         for(size_t i=0; i < result_set->number; i++)
         {
             lys_node* node_result = result_set->set.s[i];
-            if(is_part_of_output(node_result))
-                return true;
+            if (is_part_of_output(node_result)) {
+            	result = true;
+            	break;
+            }
         }
     }
-    return false;
+    return result;
 }

sdk/cpp/core/src/path/schema_node.cpp

@@ -139,7 +139,7 @@ ydk::path::SchemaNodeImpl::find(const string& path)
     vector<SchemaNode*> ret;
     struct ly_ctx* ctx = m_node->module->ctx;
 
-    const struct lys_node* found_node = ly_ctx_get_node(ctx, m_node, path.c_str());
+    const struct lys_node* found_node = ly_ctx_get_node(ctx, m_node, path.c_str(), 0);
 
     if (found_node)
     {
@@ -261,6 +261,9 @@ ydk::path::SchemaNodeImpl::get_statement() const
     case LYS_ACTION:
         s.keyword = "action";
         break;
+    case LYS_EXT:
+        s.keyword = "extension";
+        break;
     case LYS_ANYDATA:
     case LYS_UNKNOWN:
         break;

sdk/cpp/core/tests/CMakeLists.txt

@@ -25,7 +25,8 @@ foreach(test_name IN LISTS core_tests)
         pcre
         ssh_threads
         xml2
-        ssh)
+        ssh
+        dl)
 
     add_test(NAME ${test_name} COMMAND $<TARGET_FILE:${test_name}>)
 endforeach(test_name)

sdk/cpp/core/tests/models/ydktest-aug-ietf-5.yang

@@ -35,6 +35,11 @@ module ydktest-aug-ietf-5 {
     description "aug-identity";
   }
 
+  identity derived-aug-identity {
+    description "derived aug-identity";
+    base aug-identity;
+  }
+
   augment /base-one:cpython/base-one:doc {
     leaf aug-5-identityref {
       type identityref {