CM v2.1.2 (#1212)

CONTRIBUTING.md

@@ -22,10 +22,10 @@ and Modular Inference Library) or participate in collaborative developments.
 
 Thank you for your support and looking forward to collaborating with you!
 
-## Authors and project coordinators
+## Project coordinators
 
-* [Grigori Fursin](https://cKnowledge.org/gfursin) (MLCommons.org, cTuning.org, cKnowledge.org)
-* [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) (MLCommons.org, cTuning.org, cKnowledge.org)
+* [Grigori Fursin](https://cKnowledge.org/gfursin)
+* [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh)
 
 ## Contributors to MLCommons' Collective Mind (aka Collective Knowledge v3) in alphabetical order
 

cm-mlops/script/print-hello-world/run.bat

@@ -5,3 +5,5 @@ echo CM_ENV_TEST3 = %CM_ENV_TEST3%
 
 echo.
 echo HELLO WORLD!
+
+echo.

cm-mlops/script/print-hello-world/run.sh

@@ -7,3 +7,5 @@ echo "CM_ENV_TEST3 = ${CM_ENV_TEST3}"
 
 echo ""
 echo "HELLO WORLD!"
+
+echo ""

cm/CHANGES.md

@@ -1,3 +1,8 @@
+## V2.1.2
+   - added support for deps on other CM repos 
+     (if conflict = True - then fail if this repo is already installed
+      otherwise print that repo is missing)
+
 ## V2.1.1
    - added --skip-zip-parent-dir to "cm pull repo --url=..." to support downloading 
      of stable CM-MLOps repositories from https://github.com/mlcommons/cm4mlops/releases .

cm/cmind/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "2.1.1"
+__version__ = "2.1.2"
 
 from cmind.core import access
 from cmind.core import error

cm/cmind/repo/automation/repo/module.py

@@ -110,6 +110,8 @@ def pull(self, i):
         repo_meta = {}
         repo_metas = {}
 
+        warnings = []
+
         for repo in pull_repos:
              alias = repo['alias']
              url = repo.get('url', '')
@@ -151,13 +153,18 @@ def pull(self, i):
 
              repo_metas[alias] = repo_meta
 
+             if len(r.get('warnings', []))>0:
+                 warnings += r['warnings']
+
         if len(pull_repos)>0 and self.cmind.use_index:
             if console:
                 print (self.cmind.cfg['line'])
 
             ii = {'out':'con'} if console else {}
             rx = self.reindex(ii)
 
+        print_warnings(warnings)    
+
         return {'return':0, 'meta':repo_meta, 'metas': repo_metas}
 
 
@@ -581,6 +588,9 @@ def init(self, i):
             ii = {'out':'con'} if console else {}
             rx = self.reindex(ii)
 
+        warnings = r.get('warnings', [])
+        print_warnings(warnings)
+
         return r
 
     ############################################################
@@ -1196,3 +1206,14 @@ def convert_ck_dir_to_cm(rpath):
                                if r['return']>0: return r
 
     return {'return':0}
+
+def print_warnings(warnings):
+
+    if len(warnings)>0:
+        print ('')
+        print ('WARNINGS:')
+        print ('')
+        for w in warnings:
+            print ('  {}'.format(w))
+
+    return

cm/cmind/repos.py

@@ -183,6 +183,8 @@ def process(self, repo_path, mode='add'):
             * return (int): return code == 0 if no error and >0 if error
             * (error) (str): error string if return>0
 
+            * (warnings) (list of str): warnings to install more CM repositories
+
         """
 
         # Load clean file with repo paths
@@ -193,43 +195,71 @@ def process(self, repo_path, mode='add'):
 
         modified = False
 
+        warnings = []
+
         if mode == 'add':
             if repo_path not in paths:
-                if len(paths)>0:
-                    # Load meta of the current repo
-                    path_to_repo_desc = os.path.join(repo_path, self.cfg['file_meta_repo'])
-                    r=utils.load_yaml_and_json(file_name_without_ext=path_to_repo_desc)
+                # Load meta of the current repo
+                path_to_repo_desc = os.path.join(repo_path, self.cfg['file_meta_repo'])
+
+                r=utils.load_yaml_and_json(file_name_without_ext=path_to_repo_desc)
+                if r['return']>0: return r
+
+                meta = r['meta']
+
+                alias = meta.get('alias', '')
+                uid = meta.get('uid', '')
+
+                deps_on_other_repos = meta.get('deps', {})
+
+                # Check that no repos exist with the same alias and/or uid 
+                # (to avoid adding forks and original repos)
+
+                for path in paths:
+                    path_to_existing_repo_desc = os.path.join(path, self.cfg['file_meta_repo'])
+                    r=utils.load_yaml_and_json(file_name_without_ext=path_to_existing_repo_desc)
                     if r['return']>0: return r
 
-                    meta = r['meta']
+                    existing_meta = r['meta']
 
-                    alias = meta.get('alias', '')
-                    uid = meta.get('uid', '')
+                    existing_alias = existing_meta.get('alias', '')
+                    existing_uid = existing_meta.get('uid', '')
 
-                    # Check that no repos exist with the same alias and/or uid 
-                    # (to avoid adding forks and original repos)
+                    # Check if repository already exists under different name
+                    exist = False
+                    if alias != '' and existing_alias !='' and alias == existing_alias:
+                        exist = True
 
-                    for path in paths:
-                        path_to_existing_repo_desc = os.path.join(path, self.cfg['file_meta_repo'])
-                        r=utils.load_yaml_and_json(file_name_without_ext=path_to_existing_repo_desc)
-                        if r['return']>0: return r
+                    if not exist and uid !='' and existing_uid !='' and uid == existing_uid:
+                        exist = True
 
-                        existing_meta = r['meta']
+                    if exist:
+                        return {'return':1, 'error':'CM repository with the same alias "{}" and/or uid "{}" already exists in {}'.format(alias, uid, path)}
 
-                        existing_alias = existing_meta.get('alias', '')
-                        existing_uid = existing_meta.get('uid', '')
+                    # Check if there is a conflict
+                    if len(deps_on_other_repos)>0:
+                        for d in deps_on_other_repos:
+                            d_alias = d.get('alias', '')
+                            d_uid = d.get('uid', '')
 
-                        exist = False
-                        if alias != '' and existing_alias !='' and alias == existing_alias:
-                            exist = True
+                            r = utils.match_objects(existing_uid, existing_alias, d_uid, d_alias)
+                            if r['return']>0: return r
+                            match = r['match']
 
-                        if not exist and uid !='' and existing_uid !='' and uid == existing_uid:
-                            exist = True
+                            if match:
+                                if d.get('conflict', False):
+                                    return {'return':1, 'error':'Can\'t install this repository because it conflicts with the already installed one ({}) - you may need to remove it to proceed (cm rm repo {} --all)'.format(d_alias,d_alias)}
 
-                        if exist:
-                            return {'return':1, 'error':'CM repository with the same alias "{}" and/or uid "{}" already exists in {}'.format(alias, uid, path)}
+                                d['matched'] = True
 
+                                break
 
+
+                # Check if has missing deps on other CM repos
+                for d in deps_on_other_repos:
+                    if not d.get('conflict', False) and not d.get('matched', False):
+                        warnings.append('You must install extra CM repository: cm pull repo {}'.format(d['alias']))
+
                 paths.append(repo_path)
                 modified = True
 
@@ -249,7 +279,12 @@ def process(self, repo_path, mode='add'):
             # Reload repos
             self.load(init=True)
 
-        return {'return':0}
+        rr = {'return':0}
+
+        if len(warnings)>0:
+            rr['warnings'] = warnings
+
+        return rr
 
     ############################################################
     def pull(self, alias, url = '', branch = '', checkout = '', console = False, desc = '', prefix = '', depth = None, 
@@ -280,6 +315,8 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des
 
             * (meta) (dict): meta of the CM repository
 
+            * (warnings) (list of str): warnings to install more CM repositories
+
         """
 
         # Prepare path
@@ -500,14 +537,21 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des
         r = self.process(path_to_repo, 'add')
         if r['return']>0: return r
 
+        warnings = r.get('warnings', [])
+
         # Go back to original directory
         os.chdir(cur_dir)
 
         if console:
             print ('')
             print ('CM alias for this repository: {}'.format(alias))
 
-        return {'return':0, 'meta':meta}
+        rr = {'return':0, 'meta':meta}
+
+        if len(warnings)>0: rr['warnings'] = warnings
+
+        return rr
+
 
     ############################################################
     def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', only_register = False):
@@ -534,6 +578,8 @@ def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', o
             * path_to_repo_desc (str): path to repository description
             * path_to_repo_with_prefix (str): path to repository with prefix (== path_to_repo if prefix == "")
 
+            * (warnings) (list of str): warnings to install more CM repositories
+                
         """
 
         # Prepare path
@@ -612,10 +658,16 @@ def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', o
         r = self.process(path_to_repo, 'add')
         if r['return']>0: return r
 
-        return {'return':0, 'meta':meta, 
-                            'path_to_repo': path_to_repo, 
-                            'path_to_repo_desc': path_to_repo_desc,
-                            'path_to_repo_with_prefix': path_to_repo_with_prefix}
+        warnings = r.get('warnings', [])
+
+        rr =  {'return':0, 'meta':meta, 
+                           'path_to_repo': path_to_repo, 
+                           'path_to_repo_desc': path_to_repo_desc,
+                           'path_to_repo_with_prefix': path_to_repo_with_prefix}
+
+        if len(warnings)>0: rr['warnings'] = warnings
+
+        return rr
 
     ############################################################
     def delete(self, lst, remove_all = False, console = False, force = False):

cmr.yaml

@@ -6,3 +6,8 @@ git: true
 prefix: cm-mlops
 
 version: 2.0.4
+
+deps:
+  - alias: mlcommons@cm4mlops
+    uid: 9e97bb72b0474657
+    conflict: True