dist: Add --include-subprojects option

docs/markdown/snippets/dist_subprojects.md

@@ -0,0 +1,6 @@
+## meson dist --include-subprojects
+
+`meson dist` command line now gained `--include-subprojects` command line option.
+When enabled, the source tree of all subprojects used by the current build will
+also be included in the final tarball. This is useful to distribute self contained
+tarball that can be built offline (i.e. `--wrap-mode=nodownload`).

mesonbuild/mdist.py

@@ -36,6 +36,8 @@ def add_arguments(parser):
                         help='directory to cd into before running')
     parser.add_argument('--formats', default='xztar',
                         help='Comma separated list of archive types to create.')
+    parser.add_argument('--include-subprojects', action='store_true',
+                        help='Include source code of subprojects that have been used for the build.')
 
 
 def create_hash(fname):
@@ -87,22 +89,37 @@ def run_dist_scripts(dist_root, dist_scripts):
             print('Failed to run dist script {!r}'.format(name))
             sys.exit(1)
 
+def is_git(src_root):
+    _git = os.path.join(src_root, '.git')
+    return os.path.isdir(_git) or os.path.isfile(_git)
 
 def git_have_dirty_index(src_root):
     '''Check whether there are uncommitted changes in git'''
     ret = subprocess.call(['git', '-C', src_root, 'diff-index', '--quiet', 'HEAD'])
     return ret == 1
 
-def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts):
+def git_clone(src_root, distdir):
     if git_have_dirty_index(src_root):
         mlog.warning('Repository has uncommitted changes that will not be included in the dist tarball')
-    distdir = os.path.join(dist_sub, dist_name)
     if os.path.exists(distdir):
         shutil.rmtree(distdir)
     os.makedirs(distdir)
     subprocess.check_call(['git', 'clone', '--shared', src_root, distdir])
     process_submodules(distdir)
     del_gitfiles(distdir)
+
+def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scripts, subprojects):
+    distdir = os.path.join(dist_sub, dist_name)
+    git_clone(src_root, distdir)
+    for path in subprojects:
+        sub_src_root = os.path.join(src_root, path)
+        sub_distdir = os.path.join(distdir, path)
+        if os.path.exists(sub_distdir):
+            continue
+        if is_git(sub_src_root):
+            git_clone(sub_src_root, sub_distdir)
+        else:
+            shutil.copytree(sub_src_root, sub_distdir)
     run_dist_scripts(distdir, dist_scripts)
     output_names = []
     for a in archives:
@@ -112,6 +129,8 @@ def create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, dist_scri
     shutil.rmtree(distdir)
     return output_names
 
+def is_hg(src_root):
+    return os.path.isdir(os.path.join(src_root, '.hg'))
 
 def hg_have_dirty_index(src_root):
     '''Check whether there are uncommitted changes in hg'''
@@ -147,7 +166,7 @@ def create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, dist_scrip
     return output_names
 
 
-def check_dist(packagename, meson_command, bld_root, privdir):
+def check_dist(packagename, meson_command, extra_meson_args, bld_root, privdir):
     print('Testing distribution package %s' % packagename)
     unpackdir = os.path.join(privdir, 'dist-unpack')
     builddir = os.path.join(privdir, 'dist-build')
@@ -165,6 +184,7 @@ def check_dist(packagename, meson_command, bld_root, privdir):
         with open(os.path.join(bld_root, 'meson-info', 'intro-buildoptions.json')) as boptions:
             meson_command += ['-D{name}={value}'.format(**o) for o in json.load(boptions)
                               if o['name'] not in ['backend', 'install_umask']]
+        meson_command += extra_meson_args
         if subprocess.call(meson_command + ['--backend=ninja', unpacked_src_dir, builddir]) != 0:
             print('Running Meson on distribution package failed')
             return 1
@@ -214,18 +234,26 @@ def run(options):
 
     archives = determine_archives_to_generate(options)
 
-    _git = os.path.join(src_root, '.git')
-    if os.path.isdir(_git) or os.path.isfile(_git):
-        names = create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts)
-    elif os.path.isdir(os.path.join(src_root, '.hg')):
+    subprojects = []
+    extra_meson_args = []
+    if options.include_subprojects:
+        subprojects = [os.path.join(b.subproject_dir, sub) for sub in b.subprojects]
+        extra_meson_args.append('-Dwrap_mode=nodownload')
+
+    if is_git(src_root):
+        names = create_dist_git(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts, subprojects)
+    elif is_hg(src_root):
+        if subprojects:
+            print('--include-subprojects option currently not supported with Mercurial')
+            return 1
         names = create_dist_hg(dist_name, archives, src_root, bld_root, dist_sub, b.dist_scripts)
     else:
         print('Dist currently only works with Git or Mercurial repos')
         return 1
     if names is None:
         return 1
     # Check only one.
-    rc = check_dist(names[0], meson_command, bld_root, priv_dir)
+    rc = check_dist(names[0], meson_command, extra_meson_args, bld_root, priv_dir)
     if rc == 0:
         for name in names:
             create_hash(name)

run_unittests.py

@@ -31,6 +31,7 @@
 import threading
 import urllib.error
 import urllib.request
+import zipfile
 from itertools import chain
 from unittest import mock
 from configparser import ConfigParser
@@ -2546,7 +2547,7 @@ def hg_init(project_dir):
             subprocess.check_call(['hg', 'commit', '-m', 'I am a project'], cwd=project_dir)
 
         try:
-            self.dist_impl(hg_init)
+            self.dist_impl(hg_init, include_subprojects=False)
         except PermissionError:
             # When run under Windows CI, something (virus scanner?)
             # holds on to the hg files so cleaning up the dir
@@ -2573,14 +2574,23 @@ def test_dist_git_script(self):
             # fails sometimes.
             pass
 
-    def dist_impl(self, vcs_init):
+    def create_dummy_subproject(self, project_dir, name):
+        path = os.path.join(project_dir, 'subprojects', name)
+        os.makedirs(path)
+        with open(os.path.join(path, 'meson.build'), 'w') as ofile:
+            ofile.write("project('{}')".format(name))
+        return path
+
+    def dist_impl(self, vcs_init, include_subprojects=True):
         # Create this on the fly because having rogue .git directories inside
         # the source tree leads to all kinds of trouble.
         with tempfile.TemporaryDirectory() as project_dir:
             with open(os.path.join(project_dir, 'meson.build'), 'w') as ofile:
                 ofile.write('''project('disttest', 'c', version : '1.4.3')
 e = executable('distexe', 'distexe.c')
 test('dist test', e)
+subproject('vcssub', required : false)
+subproject('tarballsub', required : false)
 ''')
             with open(os.path.join(project_dir, 'distexe.c'), 'w') as ofile:
                 ofile.write('''#include<stdio.h>
@@ -2595,6 +2605,10 @@ def dist_impl(self, vcs_init):
             zip_distfile = os.path.join(self.distdir, 'disttest-1.4.3.zip')
             zip_checksumfile = zip_distfile + '.sha256sum'
             vcs_init(project_dir)
+            if include_subprojects:
+                vcs_init(self.create_dummy_subproject(project_dir, 'vcssub'))
+                self.create_dummy_subproject(project_dir, 'tarballsub')
+                self.create_dummy_subproject(project_dir, 'unusedsub')
             self.init(project_dir)
             self.build('dist')
             self.assertPathExists(xz_distfile)
@@ -2606,6 +2620,27 @@ def dist_impl(self, vcs_init):
             self.assertPathExists(zip_distfile)
             self.assertPathExists(zip_checksumfile)
 
+            if include_subprojects:
+                z = zipfile.ZipFile(zip_distfile)
+                self.assertEqual(sorted(['disttest-1.4.3/',
+                                         'disttest-1.4.3/meson.build',
+                                         'disttest-1.4.3/distexe.c']),
+                                 sorted(z.namelist()))
+
+                self._run(self.meson_command + ['dist', '--formats', 'zip', '--include-subprojects'],
+                          workdir=self.builddir)
+                z = zipfile.ZipFile(zip_distfile)
+                self.assertEqual(sorted(['disttest-1.4.3/',
+                                         'disttest-1.4.3/subprojects/',
+                                         'disttest-1.4.3/meson.build',
+                                         'disttest-1.4.3/distexe.c',
+                                         'disttest-1.4.3/subprojects/tarballsub/',
+                                         'disttest-1.4.3/subprojects/vcssub/',
+                                         'disttest-1.4.3/subprojects/tarballsub/meson.build',
+                                         'disttest-1.4.3/subprojects/vcssub/meson.build']),
+                                 sorted(z.namelist()))
+
+
     def test_rpath_uses_ORIGIN(self):
         '''
         Test that built targets use $ORIGIN in rpath, which ensures that they