project.py: Add new 'joblib' dependency to parallelize update

setup.py

@@ -46,6 +46,7 @@
         'pykwalify',
         'setuptools',
         'packaging',
+        'joblib'
     ],
     python_requires='>=3.8',
     entry_points={'console_scripts': ('west = west.app.main:main',)},

src/west/app/project.py

@@ -30,6 +30,12 @@
 from west.manifest import QUAL_MANIFEST_REV_BRANCH as QUAL_MANIFEST_REV
 from west.manifest import QUAL_REFS_WEST as QUAL_REFS
 
+JOBLIB_PRESENT = True
+try:
+    from joblib import Parallel, delayed
+except ModuleNotFoundError:
+    JOBLIB_PRESENT = False
+
 #
 # Project-related or multi-repo commands, like "init", "update",
 # "diff", etc.
@@ -883,6 +889,12 @@ def do_add_parser(self, parser_adder):
         parser.add_argument('--stats', action='store_true',
                             help='''print performance statistics for
                             update operations''')
+        if JOBLIB_PRESENT:
+            parser.add_argument('-j', '--jobs', nargs='?', const=-1,
+                                default=1, type=int, action='store',
+                                help='''Use multiple jobs to paralelize update process.
+                                Pass -1 to use all avaliable jobs.
+                                ''')
 
         group = parser.add_argument_group(
             title='local project clone caches',
@@ -1018,18 +1030,26 @@ def update_all(self):
             import_flags=ImportFlag.FORCE_PROJECTS)
 
         failed = []
-        for project in self.manifest.projects:
+
+        def project_update(project):
             if (isinstance(project, ManifestProject) or
                     project.name in self.updated):
-                continue
+                return
             try:
                 if not self.project_is_active(project):
                     self.dbg(f'{project.name}: skipping inactive project')
-                    continue
-                self.update(project)
+                    return
                 self.updated.add(project.name)
+                self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in self.manifest.projects:
+                project_update(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update)(project) for project in self.manifest.projects)
         self._handle_failed(self.args, failed)
 
     def update_importer(self, project, path):
@@ -1090,13 +1110,22 @@ def update_some(self):
             projects = self._projects(self.args.projects)
 
         failed = []
-        for project in projects:
+
+        def project_update_some(project):
             if isinstance(project, ManifestProject):
-                continue
+                return
             try:
                 self.update(project)
             except subprocess.CalledProcessError:
                 failed.append(project)
+
+        if not JOBLIB_PRESENT or self.args.jobs == 1:
+            for project in projects:
+                project_update_some(project)
+        else:
+            Parallel(n_jobs=self.args.jobs, require='sharedmem')(
+                delayed(project_update_some)(project) for project in projects)
+
         self._handle_failed(self.args, failed)
 
     def toplevel_projects(self):

tests/test_project.py

@@ -416,14 +416,14 @@ def test_grep(west_init_tmpdir):
 
     assert re.search('west-commands', cmd('grep -- -- -commands'))
 
-
-def test_update_projects(west_init_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j"])
+def test_update_projects(options, west_init_tmpdir):
     # Test the 'west update' command. It calls through to the same backend
     # functions that are used for automatic updates and 'west init'
     # reinitialization.
 
     # create local repositories
-    cmd('update')
+    cmd('update ' + options)
 
     # Add commits to the local repos.
     ur = update_helper(west_init_tmpdir)
@@ -613,7 +613,8 @@ def test_update_head_0(west_init_tmpdir):
     assert modified_files.strip() == "M CODEOWNERS", \
            'local zephyr change not preserved'
 
-def test_update_some_with_imports(repos_tmpdir):
+@pytest.mark.parametrize("options", ["", "-j 1", "-j 2", "-j -1"])
+def test_update_some_with_imports(options, repos_tmpdir):
     # 'west update project1 project2' should work fine even when
     # imports are used, as long as the relevant projects are all
     # defined in the manifest repository.
@@ -654,19 +655,19 @@ def test_update_some_with_imports(repos_tmpdir):
     # Updating unknown projects should fail as always.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update unknown-project', cwd=ws)
+        cmd(f'update {options} unknown-project', cwd=ws)
 
     # Updating a list of projects when some are resolved via project
     # imports must fail.
 
     with pytest.raises(subprocess.CalledProcessError):
-        cmd('update Kconfiglib net-tools', cwd=ws)
+        cmd(f'update {options} Kconfiglib net-tools', cwd=ws)
 
     # Updates of projects defined in the manifest repository or all
     # projects must succeed, and behave the same as if no imports
     # existed.
 
-    cmd('update net-tools', cwd=ws)
+    cmd(f'update {options} net-tools', cwd=ws)
     with pytest.raises(ManifestImportFailed):
         Manifest.from_topdir(topdir=ws)
     manifest = Manifest.from_topdir(topdir=ws,
@@ -677,10 +678,10 @@ def test_update_some_with_imports(repos_tmpdir):
     assert net_tools_project.is_cloned()
     assert not zephyr_project.is_cloned()
 
-    cmd('update zephyr', cwd=ws)
+    cmd(f'update {options} zephyr', cwd=ws)
     assert zephyr_project.is_cloned()
 
-    cmd('update', cwd=ws)
+    cmd(f'update {options}', cwd=ws)
     manifest = Manifest.from_topdir(topdir=ws)
     assert manifest.get_projects(['Kconfiglib'])[0].is_cloned()