[Build] Make cmark build a build-script product (#37102)

utils/build-script

@@ -886,11 +886,12 @@ class BuildScriptInvocation(object):
         if self.args.build_early_swift_driver:
             before_impl_product_classes.append(products.EarlySwiftDriver)
 
+        if self.args.build_cmark:
+            before_impl_product_classes.append(products.CMark)
+
         # FIXME: This is a weird division (returning a list of class objects),
         # but it matches the existing structure of the `build-script-impl`.
         impl_product_classes = []
-        if self.args.build_cmark:
-            impl_product_classes.append(products.CMark)
 
         # If --skip-build-llvm is passed in, LLVM cannot be completely disabled, as
         # Swift still needs a few LLVM targets like tblgen to be built for it to be

utils/build_swift/build_swift/driver_arguments.py

@@ -288,6 +288,9 @@ def create_argument_parser():
            help='instead of building, write JSON to stdout containing '
                 'various values used to build in this configuration')
 
+    option(['--reconfigure'], store_true,
+           help="Reconfigure all projects as we build")
+
     option('--legacy-impl', store_true('legacy_impl'),
            help='use legacy implementation')
 

utils/build_swift/tests/expected_options.py

@@ -197,6 +197,7 @@
     'native_llvm_tools_path': None,
     'native_swift_tools_path': None,
     'dump_config': False,
+    'reconfigure': False,
     'relocate_xdg_cache_home_under_build_subdir': False,
     'show_sdks': False,
     'skip_build': False,
@@ -506,6 +507,7 @@ class BuildScriptImplOption(_BaseOption):
 
     SetTrueOption('--legacy-impl', dest='legacy_impl'),
     SetTrueOption('--infer', dest='infer_dependencies'),
+    SetTrueOption('--reconfigure'),
 
     EnableOption('--android'),
     EnableOption('--build-external-benchmarks'),

utils/swift_build_support/swift_build_support/products/cmake_product.py

@@ -0,0 +1,116 @@
+# swift_build_support/products/product.py -----------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2021 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+#
+# ----------------------------------------------------------------------------
+
+import os
+
+from . import product
+from .. import cmake
+from .. import shell
+
+
+class CMakeProduct(product.Product):
+    def build_with_cmake(self, build_targets, build_type, build_args):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+        _cmake = cmake.CMake(self.args, self.toolchain)
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_build.extend([self.toolchain.cmake, "--build"])
+
+        generator_output_path = ""
+        if self.args.cmake_generator == "Ninja":
+            generator_output_path = os.path.join(self.build_dir, "build.ninja")
+
+        cmake_cache_path = os.path.join(self.build_dir, "CMakeCache.txt")
+        if self.args.reconfigure or not os.path.isfile(cmake_cache_path) or \
+                (generator_output_path and not os.path.isfile(generator_output_path)):
+            if not os.path.exists(self.build_dir):
+                os.makedirs(self.build_dir)
+
+            # Use `cmake-file-api` in case it is available.
+            query_dir = os.path.join(self.build_dir, ".cmake", "api", "v1", "query")
+            if not os.path.exists(query_dir):
+                os.makedirs(query_dir)
+            open(os.path.join(query_dir, "codemodel-v2"), 'a').close()
+            open(os.path.join(query_dir, "cache-v2"), 'a').close()
+
+            env = None
+            if self.toolchain.distcc:
+                env = {
+                    "DISTCC_HOSTS": "localhost,lzo,cpp"
+                }
+
+            with shell.pushd(self.build_dir):
+                shell.call([self.toolchain.cmake] + list(self.cmake_options) +
+                           list(_cmake.common_options()) +
+                           self.args.extra_cmake_options + [self.source_dir],
+                           env=env)
+
+        if not self.args.skip_build or self.product_name() == "llvm":
+            if self.args.cmake_generator == "Xcode":
+                # Xcode generator uses "ALL_BUILD" instead of "all".
+                # Also, xcodebuild uses -target instead of bare names.
+                build_targets = build_targets.copy()
+                build_targets = [val for target in build_targets
+                                 for val in ["-target",
+                                             target if target != "all"
+                                             else "ALL_BUILD"]]
+
+                # Xcode can't restart itself if it turns out we need to reconfigure.
+                # Do an advance build to handle that.
+                shell.call(cmake_build + [self.build_dir, build_type])
+
+            shell.call(cmake_build + [self.build_dir, "--config", build_type, "--"]
+                       + build_args + build_targets)
+
+    def test_with_cmake(self, executable_target, results_targets,
+                        build_type, build_args):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_args = [self.toolchain.cmake, "--build", self.build_dir,
+                      "--config", build_type, "--"]
+        cmake_build.extend(cmake_args + build_args)
+
+        def target_flag(target):
+            if self.args.cmake_generator == "Xcode":
+                return ["-target", target]
+            return [target]
+
+        if executable_target:
+            shell.call(cmake_build + target_flag(executable_target))
+
+        for target in results_targets:
+            if target:
+                test_target = target
+                print("--- %s ---" % target)
+                if test_target.startswith("check-swift") and self.args.test_paths:
+                    test_target = test_target + "-custom"
+
+                shell.call(cmake_build + target_flag(test_target))
+
+                print("--- %s finished ---" % target)
+
+    def install_with_cmake(self, install_targets, install_destdir):
+        assert self.toolchain.cmake is not None
+        cmake_build = []
+
+        if self.toolchain.distcc_pump:
+            cmake_build.append(self.toolchain.distcc_pump)
+        cmake_args = [self.toolchain.cmake, "--build", self.build_dir, "--"]
+        cmake_build.extend(cmake_args + install_targets)
+
+        environment = {'DESTDIR': install_destdir}
+        shell.call(cmake_build, env=environment)

utils/swift_build_support/swift_build_support/products/cmark.py

@@ -10,27 +10,87 @@
 #
 # ----------------------------------------------------------------------------
 
-from . import product
+from . import cmake_product
 
 
-class CMark(product.Product):
+class CMark(cmake_product.CMakeProduct):
     @classmethod
     def is_build_script_impl_product(cls):
         """is_build_script_impl_product -> bool
 
         Whether this product is produced by build-script-impl.
         """
-        return True
+        return False
 
     @classmethod
     def is_before_build_script_impl_product(cls):
         """is_before_build_script_impl_product -> bool
 
         Whether this product is build before any build-script-impl products.
         """
-        return False
+        return True
 
     # This is the root of the build-graph, so it doesn't have any dependencies.
     @classmethod
     def get_dependencies(cls):
         return []
+
+    def should_build(self, host_target):
+        """should_build() -> Bool
+
+        Whether or not this product should be built with the given arguments.
+        """
+        return self.args.build_cmark
+
+    def build(self, host_target):
+        """build() -> void
+
+        Perform the build, for a non-build-script-impl product.
+        """
+        self.cmake_options.define('CMAKE_BUILD_TYPE:STRING',
+                                  self.args.cmark_build_variant)
+
+        self.build_with_cmake(["all"], self.args.cmark_build_variant, [])
+
+    def should_test(self, host_target):
+        """should_test() -> Bool
+
+        Whether or not this product should be tested with the given arguments.
+        """
+        if self.args.cross_compile_hosts and \
+                host_target in self.args.cross_compile_hosts:
+            return False
+
+        return self.args.test
+
+    def test(self, host_target):
+        """
+        Perform the test phase for the product.
+
+        This phase might build and execute the product tests.
+        """
+        executable_target = 'api_test'
+        results_targets = ['test']
+        if self.args.cmake_generator == 'Xcode':
+            # Xcode generator uses "RUN_TESTS" instead of "test".
+            results_targets = ['RUN_TESTS']
+
+        self.test_with_cmake(executable_target, results_targets,
+                             self.args.cmark_build_variant, [])
+
+    def should_install(self, host_target):
+        """should_install() -> Bool
+
+        Whether or not this product should be installed with the given
+        arguments.
+        """
+        return self.args.install_all
+
+    def install(self, host_target):
+        """
+        Perform the install phase for the product.
+
+        This phase might copy the artifacts from the previous phases into a
+        destination directory.
+        """
+        self.install_with_cmake(["install"], self.host_install_destdir(host_target))

utils/swift_build_support/swift_build_support/products/product.py

@@ -190,6 +190,31 @@ def install_toolchain_path(self, host_target):
         return targets.toolchain_path(install_destdir,
                                       self.args.install_prefix)
 
+    def should_include_host_in_lipo(self, host_target):
+        if self.args.cross_compile_hosts:
+            if host_target.startswith("macosx") or \
+               host_target.startswith("iphone") or \
+               host_target.startswith("appletv") or \
+               host_target.startswith("watch"):
+                return True
+        return False
+
+    def host_install_destdir(self, host_target):
+        if self.args.cross_compile_hosts:
+            # If cross compiling tools, install into a host-specific subdirectory.
+            if self.should_include_host_in_lipo(host_target):
+                # If this is one of the hosts we should lipo,
+                # install in to a temporary subdirectory.
+                return '%s/intermediate-install/%s' % \
+                    (self.args.install_destdir, host_target)
+            elif host_target == "merged-hosts":
+                # This assumes that all hosts are merged to the lipo.
+                return self.args.install_destdir
+            else:
+                return '%s/%s' % (self.args.install_destdir, host_target)
+        else:
+            return self.args.install_destdir
+
 
 class ProductBuilder(object):
     """

utils/swift_build_support/tests/products/test_cmark.py

@@ -0,0 +1,113 @@
+# tests/products/test_ninja.py ----------------------------------*- python -*-
+#
+# This source file is part of the Swift.org open source project
+#
+# Copyright (c) 2014 - 2017 Apple Inc. and the Swift project authors
+# Licensed under Apache License v2.0 with Runtime Library Exception
+#
+# See https://swift.org/LICENSE.txt for license information
+# See https://swift.org/CONTRIBUTORS.txt for the list of Swift project authors
+# ----------------------------------------------------------------------------
+
+import argparse
+import os
+import shutil
+import sys
+import tempfile
+import unittest
+try:
+    # py2
+    from StringIO import StringIO
+except ImportError:
+    # py3
+    from io import StringIO
+
+from swift_build_support import cmake
+from swift_build_support import shell
+from swift_build_support.products import CMark
+from swift_build_support.targets import StdlibDeploymentTarget
+from swift_build_support.toolchain import host_toolchain
+from swift_build_support.workspace import Workspace
+
+
+class CMarkTestCase(unittest.TestCase):
+
+    def setUp(self):
+        # Setup workspace
+        tmpdir1 = os.path.realpath(tempfile.mkdtemp())
+        tmpdir2 = os.path.realpath(tempfile.mkdtemp())
+        os.makedirs(os.path.join(tmpdir1, 'cmark'))
+
+        self.workspace = Workspace(source_root=tmpdir1,
+                                   build_root=tmpdir2)
+
+        self.host = StdlibDeploymentTarget.host_target()
+
+        # Setup toolchain
+        self.toolchain = host_toolchain()
+        self.toolchain.cc = '/path/to/cc'
+        self.toolchain.cxx = '/path/to/cxx'
+
+        # Setup args
+        self.args = argparse.Namespace(
+            build_cmark=True,
+            cmake_generator="Ninja",
+            cmark_build_type="Release",
+            rebuild=False,
+            extra_cmake_options=[],
+            skip_build=False,
+            darwin_deployment_version_osx="10.9",
+            cmark_build_variant="Debug",
+            export_compile_commands=False,
+            reconfigure=False,
+            distcc=None,
+            sccache=None,
+            cmake_c_launcher=None,
+            cmake_cxx_launcher=None,
+            clang_user_visible_version=None,
+            build_ninja=False,
+            enable_asan=False,
+            enable_lsan=False,
+            enable_sanitize_coverage=False,
+            enable_tsan=False,
+            enable_ubsan=False)
+
+        # Setup shell
+        shell.dry_run = True
+        self._orig_stdout = sys.stdout
+        self._orig_stderr = sys.stderr
+        self.stdout = StringIO()
+        self.stderr = StringIO()
+        sys.stdout = self.stdout
+        sys.stderr = self.stderr
+
+    def tearDown(self):
+        shutil.rmtree(self.workspace.build_root)
+        shutil.rmtree(self.workspace.source_root)
+        sys.stdout = self._orig_stdout
+        sys.stderr = self._orig_stderr
+        shell.dry_run = False
+        self.workspace = None
+        self.toolchain = None
+        self.args = None
+
+    def test_build(self):
+        cmark = CMark(
+            args=self.args,
+            toolchain=self.toolchain,
+            source_dir=self.workspace.source_root,
+            build_dir=self.workspace.build_root)
+
+        cmark.build(host_target=self.host)
+        _cmake = cmake.CMake(self.args, self.toolchain)
+
+        self.assertEqual(self.stdout.getvalue(), """\
++ pushd {build_dir}
++ {cmake} -DCMAKE_BUILD_TYPE:STRING={build_variant} {cmake_args} {source_dir}
++ popd
++ {cmake} --build {build_dir} --config {build_variant} -- all
+""".format(build_dir=self.workspace.build_root,
+           source_dir=self.workspace.source_root,
+           cmake=self.toolchain.cmake,
+           cmake_args=' '.join(_cmake.common_options()),
+           build_variant=self.args.cmark_build_variant))