From 49957ffc51340229b5f6737eb7d546d7b918b753 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 17 Apr 2024 16:26:49 +0200 Subject: [PATCH 1/3] Added conflict on new CM repo mlcommons@cm4mlops --- cmr.yaml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/cmr.yaml b/cmr.yaml index 7795aee19..ed28641ad 100644 --- a/cmr.yaml +++ b/cmr.yaml @@ -6,3 +6,8 @@ git: true prefix: cm-mlops version: 2.0.4 + +deps: + - alias: mlcommons@cm4mlops + uid: 9e97bb72b0474657 + conflict: True From 1bdb3389d58b3a649f5383ecf26375e9ab8e8671 Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 17 Apr 2024 16:57:55 +0200 Subject: [PATCH 2/3] - added support for deps on other CM repos (if conflict = True - then fail if this repo is already installed otherwise print that repo is missing) --- CONTRIBUTING.md | 6 +- cm/CHANGES.md | 5 ++ cm/cmind/__init__.py | 2 +- cm/cmind/repo/automation/repo/module.py | 21 +++++ cm/cmind/repos.py | 110 +++++++++++++++++------- 5 files changed, 111 insertions(+), 33 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 8a5fd8517..734ff0f7c 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -22,10 +22,10 @@ and Modular Inference Library) or participate in collaborative developments. Thank you for your support and looking forward to collaborating with you! -## Authors and project coordinators +## Project coordinators -* [Grigori Fursin](https://cKnowledge.org/gfursin) (MLCommons.org, cTuning.org, cKnowledge.org) -* [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) (MLCommons.org, cTuning.org, cKnowledge.org) +* [Grigori Fursin](https://cKnowledge.org/gfursin) +* [Arjun Suresh](https://www.linkedin.com/in/arjunsuresh) ## Contributors to MLCommons' Collective Mind (aka Collective Knowledge v3) in alphabetical order diff --git a/cm/CHANGES.md b/cm/CHANGES.md index a9957d40f..3a2e47a1a 100644 --- a/cm/CHANGES.md +++ b/cm/CHANGES.md @@ -1,3 +1,8 @@ +## V2.1.2 + - added support for deps on other CM repos + (if conflict = True - then fail if this repo is already installed + otherwise print that repo is missing) + ## V2.1.1 - added --skip-zip-parent-dir to "cm pull repo --url=..." to support downloading of stable CM-MLOps repositories from https://github.com/mlcommons/cm4mlops/releases . diff --git a/cm/cmind/__init__.py b/cm/cmind/__init__.py index 97d8ef25a..8e3afdf83 100644 --- a/cm/cmind/__init__.py +++ b/cm/cmind/__init__.py @@ -1,4 +1,4 @@ -__version__ = "2.1.1" +__version__ = "2.1.2" from cmind.core import access from cmind.core import error diff --git a/cm/cmind/repo/automation/repo/module.py b/cm/cmind/repo/automation/repo/module.py index e73e1f5d6..11768b356 100644 --- a/cm/cmind/repo/automation/repo/module.py +++ b/cm/cmind/repo/automation/repo/module.py @@ -110,6 +110,8 @@ def pull(self, i): repo_meta = {} repo_metas = {} + warnings = [] + for repo in pull_repos: alias = repo['alias'] url = repo.get('url', '') @@ -151,6 +153,9 @@ def pull(self, i): repo_metas[alias] = repo_meta + if len(r.get('warnings', []))>0: + warnings += r['warnings'] + if len(pull_repos)>0 and self.cmind.use_index: if console: print (self.cmind.cfg['line']) @@ -158,6 +163,8 @@ def pull(self, i): ii = {'out':'con'} if console else {} rx = self.reindex(ii) + print_warnings(warnings) + return {'return':0, 'meta':repo_meta, 'metas': repo_metas} @@ -581,6 +588,9 @@ def init(self, i): ii = {'out':'con'} if console else {} rx = self.reindex(ii) + warnings = r.get('warnings', []) + print_warnings(warnings) + return r ############################################################ @@ -1196,3 +1206,14 @@ def convert_ck_dir_to_cm(rpath): if r['return']>0: return r return {'return':0} + +def print_warnings(warnings): + + if len(warnings)>0: + print ('') + print ('WARNINGS:') + print ('') + for w in warnings: + print (' {}'.format(w)) + + return diff --git a/cm/cmind/repos.py b/cm/cmind/repos.py index 8a93a4945..bff9f8749 100644 --- a/cm/cmind/repos.py +++ b/cm/cmind/repos.py @@ -183,6 +183,8 @@ def process(self, repo_path, mode='add'): * return (int): return code == 0 if no error and >0 if error * (error) (str): error string if return>0 + * (warnings) (list of str): warnings to install more CM repositories + """ # Load clean file with repo paths @@ -193,43 +195,71 @@ def process(self, repo_path, mode='add'): modified = False + warnings = [] + if mode == 'add': if repo_path not in paths: - if len(paths)>0: - # Load meta of the current repo - path_to_repo_desc = os.path.join(repo_path, self.cfg['file_meta_repo']) - r=utils.load_yaml_and_json(file_name_without_ext=path_to_repo_desc) + # Load meta of the current repo + path_to_repo_desc = os.path.join(repo_path, self.cfg['file_meta_repo']) + + r=utils.load_yaml_and_json(file_name_without_ext=path_to_repo_desc) + if r['return']>0: return r + + meta = r['meta'] + + alias = meta.get('alias', '') + uid = meta.get('uid', '') + + deps_on_other_repos = meta.get('deps', {}) + + # Check that no repos exist with the same alias and/or uid + # (to avoid adding forks and original repos) + + for path in paths: + path_to_existing_repo_desc = os.path.join(path, self.cfg['file_meta_repo']) + r=utils.load_yaml_and_json(file_name_without_ext=path_to_existing_repo_desc) if r['return']>0: return r - meta = r['meta'] + existing_meta = r['meta'] - alias = meta.get('alias', '') - uid = meta.get('uid', '') + existing_alias = existing_meta.get('alias', '') + existing_uid = existing_meta.get('uid', '') - # Check that no repos exist with the same alias and/or uid - # (to avoid adding forks and original repos) + # Check if repository already exists under different name + exist = False + if alias != '' and existing_alias !='' and alias == existing_alias: + exist = True - for path in paths: - path_to_existing_repo_desc = os.path.join(path, self.cfg['file_meta_repo']) - r=utils.load_yaml_and_json(file_name_without_ext=path_to_existing_repo_desc) - if r['return']>0: return r + if not exist and uid !='' and existing_uid !='' and uid == existing_uid: + exist = True - existing_meta = r['meta'] + if exist: + return {'return':1, 'error':'CM repository with the same alias "{}" and/or uid "{}" already exists in {}'.format(alias, uid, path)} - existing_alias = existing_meta.get('alias', '') - existing_uid = existing_meta.get('uid', '') + # Check if there is a conflict + if len(deps_on_other_repos)>0: + for d in deps_on_other_repos: + d_alias = d.get('alias', '') + d_uid = d.get('uid', '') - exist = False - if alias != '' and existing_alias !='' and alias == existing_alias: - exist = True + r = utils.match_objects(existing_uid, existing_alias, d_uid, d_alias) + if r['return']>0: return r + match = r['match'] - if not exist and uid !='' and existing_uid !='' and uid == existing_uid: - exist = True + if match: + if d.get('conflict', False): + return {'return':1, 'error':'Can\'t install this repository because it conflicts with the already installed one ({}) - you may need to remove it to proceed (cm rm repo {} --all)'.format(d_alias,d_alias)} - if exist: - return {'return':1, 'error':'CM repository with the same alias "{}" and/or uid "{}" already exists in {}'.format(alias, uid, path)} + d['matched'] = True + break + + # Check if has missing deps on other CM repos + for d in deps_on_other_repos: + if not d.get('conflict', False) and not d.get('matched', False): + warnings.append('You must install extra CM repository: cm pull repo {}'.format(d['alias'])) + paths.append(repo_path) modified = True @@ -249,7 +279,12 @@ def process(self, repo_path, mode='add'): # Reload repos self.load(init=True) - return {'return':0} + rr = {'return':0} + + if len(warnings)>0: + rr['warnings'] = warnings + + return rr ############################################################ def pull(self, alias, url = '', branch = '', checkout = '', console = False, desc = '', prefix = '', depth = None, @@ -280,6 +315,8 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des * (meta) (dict): meta of the CM repository + * (warnings) (list of str): warnings to install more CM repositories + """ # Prepare path @@ -500,6 +537,8 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des r = self.process(path_to_repo, 'add') if r['return']>0: return r + warnings = r.get('warnings', []) + # Go back to original directory os.chdir(cur_dir) @@ -507,7 +546,12 @@ def pull(self, alias, url = '', branch = '', checkout = '', console = False, des print ('') print ('CM alias for this repository: {}'.format(alias)) - return {'return':0, 'meta':meta} + rr = {'return':0, 'meta':meta} + + if len(warnings)>0: rr['warnings'] = warnings + + return rr + ############################################################ def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', only_register = False): @@ -534,6 +578,8 @@ def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', o * path_to_repo_desc (str): path to repository description * path_to_repo_with_prefix (str): path to repository with prefix (== path_to_repo if prefix == "") + * (warnings) (list of str): warnings to install more CM repositories + """ # Prepare path @@ -612,10 +658,16 @@ def init(self, alias, uid, path = '', console = False, desc = '', prefix = '', o r = self.process(path_to_repo, 'add') if r['return']>0: return r - return {'return':0, 'meta':meta, - 'path_to_repo': path_to_repo, - 'path_to_repo_desc': path_to_repo_desc, - 'path_to_repo_with_prefix': path_to_repo_with_prefix} + warnings = r.get('warnings', []) + + rr = {'return':0, 'meta':meta, + 'path_to_repo': path_to_repo, + 'path_to_repo_desc': path_to_repo_desc, + 'path_to_repo_with_prefix': path_to_repo_with_prefix} + + if len(warnings)>0: rr['warnings'] = warnings + + return rr ############################################################ def delete(self, lst, remove_all = False, console = False, force = False): From 2f6ba562be39d6ca62a2f20a8d1a79dd90f0a94a Mon Sep 17 00:00:00 2001 From: Grigori Fursin Date: Wed, 17 Apr 2024 17:09:51 +0200 Subject: [PATCH 3/3] run all tests for new CM --- cm-mlops/script/print-hello-world/run.bat | 2 ++ cm-mlops/script/print-hello-world/run.sh | 2 ++ 2 files changed, 4 insertions(+) diff --git a/cm-mlops/script/print-hello-world/run.bat b/cm-mlops/script/print-hello-world/run.bat index 03810bf9c..f105ce79d 100644 --- a/cm-mlops/script/print-hello-world/run.bat +++ b/cm-mlops/script/print-hello-world/run.bat @@ -5,3 +5,5 @@ echo CM_ENV_TEST3 = %CM_ENV_TEST3% echo. echo HELLO WORLD! + +echo. diff --git a/cm-mlops/script/print-hello-world/run.sh b/cm-mlops/script/print-hello-world/run.sh index 32824d982..6f8c93090 100644 --- a/cm-mlops/script/print-hello-world/run.sh +++ b/cm-mlops/script/print-hello-world/run.sh @@ -7,3 +7,5 @@ echo "CM_ENV_TEST3 = ${CM_ENV_TEST3}" echo "" echo "HELLO WORLD!" + +echo ""