build_tasks.js

/**
 * @license
 * Copyright 2018 Google LLC
 * SPDX-License-Identifier: Apache-2.0
 */

/**
 * @fileoverview Gulp script to build Blockly for Node & NPM.
 */

const gulp = require('gulp');
gulp.replace = require('gulp-replace');
gulp.rename = require('gulp-rename');
gulp.sourcemaps = require('gulp-sourcemaps');

const path = require('path');
const fs = require('fs');
const {exec, execSync} = require('child_process');
const through2 = require('through2');

const clangFormat = require('clang-format');
const clangFormatter = require('gulp-clang-format');
const closureCompiler = require('google-closure-compiler').gulp();
const closureDeps = require('google-closure-deps');
const argv = require('yargs').argv;
const {rimraf} = require('rimraf');

const {BUILD_DIR, DEPS_FILE, RELEASE_DIR, TEST_DEPS_FILE, TSC_OUTPUT_DIR, TYPINGS_BUILD_DIR} = require('./config');
const {getPackageJson} = require('./helper_tasks');

const {posixPath} = require('../helpers');

////////////////////////////////////////////////////////////
//                        Build                           //
////////////////////////////////////////////////////////////

/**
 * Suffix to add to compiled output files.
 */
const COMPILED_SUFFIX = '_compressed';

/**
 * Name of an object to be used as a shared "global" namespace by
 * chunks generated by the Closure Compiler with the
 * --rename_prefix_namespace option (see
 * https://github.com/google/closure-compiler/wiki/Chunk-output-for-dynamic-loading#using-global_namespace-as-the-chunk-output-type
 * for more information.)  The wrapper for the first chunk will create
 * an object with this name and save it; wrappers for other chunks
 * will ensure that the same object is available with this same name.
 * The --rename_prefix_namespace option will then cause the compiled
 * chunks to create properties on this object instead of creating
 * "global" (really chunk-local) variables.  This allows later chunks
 * to depend upon modules from earlier chunks.
 *
 * It can be any value that doesn't clash with a global variable or
 * wrapper argument, but as it will appear many times in the compiled
 * output it is preferable that it be short.
 */
const NAMESPACE_VARIABLE = '$';

/**
 * Property that will be used to store the value of the namespace
 * object on each chunk's exported object.  This is so that dependent
 * chunks can retrieve the namespace object and thereby access modules
 * defined in the parent chunk (or it's parent, etc.).  This should be
 * chosen so as to not collide with any exported name.
 */
const NAMESPACE_PROPERTY = '__namespace__';

/**
 * A list of chunks.  Order matters: later chunks can depend on
 * earlier ones, but not vice-versa.  All chunks are assumed to depend
 * on the first chunk.  Properties are as follows:
 *
 * - .name: the name of the chunk.  Used to label it when describing
 *   it to Closure Compiler and forms the prefix of filename the chunk
 *   will be written to.
 * - .entry: the source .js file which is the entrypoint for the
 *   chunk.
 * - .exports: an expression evaluating to the exports/Module object
 *   of module that is the chunk's entrypoint / top level module.
 * - .reexport: if running in a browser, save the chunk's exports
 *   object (or a single export of it; see reexportOnly, below) at
 *   this location in the global namespace.
 * - .reexportOnly: if reexporting and this property is set,
 *   save only the correspondingly-named export.  Otherwise
 *   save the whole export object.
 *
 * The function getChunkOptions will, after running
 * closure-calculate-chunks, update each chunk to add the following
 * properties:
 *
 * - .parent: the parent chunk of the given chunk.  Typically
 *    chunks[0], except for chunk[0].parent which will be null.
 * - .wrapper: the generated chunk wrapper.
 *
 * Output files will be named <chunk.name><COMPILED_SUFFIX>.js.
 */
const chunks = [
  {
    name: 'blockly',
    entry: path.join(TSC_OUTPUT_DIR, 'core', 'main.js'),
    exports: 'module$build$src$core$blockly',
    reexport: 'Blockly',
  },
  {
    name: 'blocks',
    entry: path.join(TSC_OUTPUT_DIR, 'blocks', 'blocks.js'),
    exports: 'module$exports$Blockly$libraryBlocks',
    reexport: 'Blockly.libraryBlocks',
  },
  {
    name: 'javascript',
    entry: path.join(TSC_OUTPUT_DIR, 'generators', 'javascript', 'all.js'),
    exports: 'module$exports$Blockly$JavaScript',
    reexport: 'Blockly.JavaScript',
    reexportOnly: 'javascriptGenerator',
  },
  {
    name: 'python',
    entry: path.join(TSC_OUTPUT_DIR, 'generators', 'python', 'all.js'),
    exports: 'module$exports$Blockly$Python',
    reexport: 'Blockly.Python',
    reexportOnly: 'pythonGenerator',
  },
  {
    name: 'php',
    entry: path.join(TSC_OUTPUT_DIR, 'generators', 'php', 'all.js'),
    exports: 'module$exports$Blockly$PHP',
    reexport: 'Blockly.PHP',
    reexportOnly: 'phpGenerator',
  },
  {
    name: 'lua',
    entry: path.join(TSC_OUTPUT_DIR, 'generators', 'lua', 'all.js'),
    exports: 'module$exports$Blockly$Lua',
    reexport: 'Blockly.Lua',
    reexportOnly: 'luaGenerator',
  },
  {
    name: 'dart',
    entry: path.join(TSC_OUTPUT_DIR, 'generators', 'dart', 'all.js'),
    exports: 'module$exports$Blockly$Dart',
    reexport: 'Blockly.Dart',
    reexportOnly: 'dartGenerator',
  }
];

const licenseRegex = `\\/\\*\\*
 \\* @license
 \\* (Copyright \\d+ (Google LLC|Massachusetts Institute of Technology))
( \\* All rights reserved.
)? \\* SPDX-License-Identifier: Apache-2.0
 \\*\\/`;

/**
 * Helper method for stripping the Google's and MIT's Apache Licenses.
 */
function stripApacheLicense() {
  // Strip out Google's and MIT's Apache licences.
  // Closure Compiler preserves dozens of Apache licences in the Blockly code.
  // Remove these if they belong to Google or MIT.
  // MIT's permission to do this is logged in Blockly issue #2412.
  return gulp.replace(new RegExp(licenseRegex, 'g'), '\n\n\n\n');
  // Replace with the same number of lines so that source-maps are not affected.
}

/**
 * Closure Compiler diagnostic groups we want to be treated as errors.
 * These are effected when the --debug or --strict flags are passed.
 * For a full list of Closure Compiler groups, consult the output of
 * google-closure-compiler --help or look in the source  here:
 * https://github.com/google/closure-compiler/blob/master/src/com/google/javascript/jscomp/DiagnosticGroups.java#L117
 *
 * The list in JSCOMP_ERROR contains all the diagnostic groups we know
 * about, but some are commented out if we don't want them, and may
 * appear in JSCOMP_WARNING or JSCOMP_OFF instead.  Items not
 * appearing on any list will default to setting provided by the
 * compiler, which may vary depending on compilation level.
 */
const JSCOMP_ERROR = [
  // 'accessControls',  // Deprecated; means same as visibility.
  // 'checkPrototypalTypes',  // override annotations are stripped by tsc.
  'checkRegExp',
  // 'checkTypes',  // Disabled; see note in JSCOMP_OFF.
  'checkVars',
  'conformanceViolations',
  'const',
  'constantProperty',
  'deprecated',
  'deprecatedAnnotations',
  'duplicateMessage',
  'es5Strict',
  'externsValidation',
  'extraRequire',  // Undocumented but valid.
  'functionParams',
  // 'globalThis',  // This types are stripped by tsc.
  'invalidCasts',
  'misplacedTypeAnnotation',
  // 'missingOverride',  // There are many of these, which should be fixed.
  'missingPolyfill',
  // 'missingProperties',  // Unset static properties are stripped by tsc.
  'missingProvide',
  'missingRequire',
  'missingReturn',
  // 'missingSourcesWarnings',  // Group of several other options.
  'moduleLoad',
  'msgDescriptions',
  // 'nonStandardJsDocs',  // Disabled; see note in JSCOMP_OFF.
  // 'partialAlias',  // Don't want this to be an error yet; only warning.
  // 'polymer',  // Not applicable.
  // 'reportUnknownTypes',  // VERY verbose.
  // 'strictCheckTypes',  // Use --strict to enable.
  // 'strictMissingProperties',  // Part of strictCheckTypes.
  'strictModuleChecks',  // Undocumented but valid.
  'strictModuleDepCheck',
  // 'strictPrimitiveOperators',  // Part of strictCheckTypes.
  'suspiciousCode',
  'typeInvalidation',
  'undefinedVars',
  'underscore',
  'unknownDefines',
  // 'unusedLocalVariables',  // Disabled; see note in JSCOMP_OFF.
  'unusedPrivateMembers',
  'uselessCode',
  'untranspilableFeatures',
  // 'visibility',  // Disabled; see note in JSCOMP_OFF.
];

/**
 * Closure Compiler diagnostic groups we want to be treated as warnings.
 * These are effected when the --debug or --strict flags are passed.
 *
 * For most (all?) diagnostic groups this is the default level, so
 * it's generally sufficient to remove them from JSCOMP_ERROR.
 */
const JSCOMP_WARNING = [
];

/**
 * Closure Compiler diagnostic groups we want to be ignored.  These
 * suppressions are always effected by default.
 *
 * Make sure that anything added here is commented out of JSCOMP_ERROR
 * above, as that takes precedence.)
 */
const JSCOMP_OFF = [
  /* The removal of Closure type system types from our JSDoc
   * annotations means that the Closure Compiler now generates certain
   * diagnostics because it no longer has enough information to be
   * sure that the input code is correct.  The following diagnostic
   * groups are turned off to suppress such errors.
   *
   * When adding additional items to this list it may be helpful to
   * search the compiler source code
   * (https://github.com/google/closure-compiler/) for the JSC_*
   * diagnostic name (omitting the JSC_ prefix) to find the corresponding
   * DiagnosticGroup.
   */
  'checkTypes',
  'nonStandardJsDocs',  // Due to @internal
  'unusedLocalVariables',  // Due to code generated for merged namespaces.

  /* In order to transition to ES modules, modules will need to import
   * one another by relative paths. This means that the previous
   * practice of moving all source files into the same directory for
   * compilation would break imports.
   *
   * Not flattening files in this way breaks our usage
   * of @package however: files were flattened so that all Blockly
   * source files are in the same directory and can use @package to
   * mark methods that are only allowed for use by Blockly, while
   * still allowing access between e.g. core/events/* and
   * core/utils/*. We were downgrading access control violations
   * (including @private) to warnings, but this ends up being so
   * spammy that it makes the compiler output nearly useless.
   *
   * Once ES module migration is complete, they will be re-enabled and
   * an alternative to @package will be established.
   */
  'visibility',
];

/**
 * Builds Blockly as a JS program, by running tsc on all the files in
 * the core directory.
 */
function buildJavaScript(done) {
  execSync(
      `tsc -outDir "${TSC_OUTPUT_DIR}" -declarationDir "${TYPINGS_BUILD_DIR}"`,
      {stdio: 'inherit'});
  execSync(`node scripts/tsick.js "${TSC_OUTPUT_DIR}"`, {stdio: 'inherit'});
  done();
}

/**
 * This task updates DEPS_FILE (deps.js), used by
 * bootstrap.js when loading Blockly in uncompiled mode.
 *
 * Also updates TEST_DEPS_FILE (deps.mocha.js), used by the mocha test
 * suite.
 *
 * Prerequisite: buildJavaScript.
 */
function buildDeps() {
  const roots = [
    path.join(TSC_OUTPUT_DIR, 'closure', 'goog', 'base.js'),
    TSC_OUTPUT_DIR,
    'tests/mocha',
  ];

  /** Maximum buffer size, in bytes for child process stdout/stderr. */
  const MAX_BUFFER_SIZE = 10 * 1024 * 1024;

  /**
   * Filter a string to extract lines containing (or not containing) the
   * specified target string.
   *
   * @param {string} text Text to filter.
   * @param {string} target String to search for.
   * @param {boolean?} exclude If true, extract only non-matching lines.
   * @returns {string} Filtered text.
   */
  function filter(text, target, exclude) {
    return text.split('\n')
        .filter((line) => Boolean(line.match(target)) !== Boolean(exclude))
        .join('\n');
  }

  /**
   * Log unexpected diagnostics, after removing expected warnings.
   *
   * @param {string} text Standard error output from closure-make-deps
   */
  function log(text) {
    for (const line of text.split('\n')) {
      if (line &&
          !/^WARNING .*: Bounded generic semantics are currently/.test(line) &&
          !/^WARNING .*: Missing type declaration/.test(line) &&
          !/^WARNING .*: illegal use of unknown JSDoc tag/.test(line)) {
        console.error(line);
      }
    }
  }

  return new Promise((resolve, reject) => {
    const args = roots.map(root => `--root '${root}' `).join('');
    exec(
        `closure-make-deps ${args}`, {maxBuffer: MAX_BUFFER_SIZE},
        (error, stdout, stderr) => {
          if (error) {
            // Remove warnings from stack trace to show only errors.
            error.stack = filter(error.stack, /^WARNING/, true);
            // Due to some race condition, the stderr parameter is
            // often badly truncated if an error is non-null, so the
            // error message might not actually be shown to the user.
            // Print a helpful message to the user to help them find
            // out what the problem is.
            error.stack += `

If you do not see an helpful diagnostic from closure-make-deps in the
error message above, try running:

    npx closure-make-deps ${args} 2>&1 |grep -v WARNING`;
            reject(error);
          } else {
            log(stderr);
            // Anything not about mocha goes in DEPS_FILE.
            fs.writeFileSync(DEPS_FILE, filter(stdout, 'tests/mocha', true));
            // Anything about mocha does in TEST_DEPS_FILE.
            fs.writeFileSync(TEST_DEPS_FILE, filter(stdout, 'tests/mocha'));
            resolve();
          }
        });
  });
}

/**
 * This task regenerates msg/json/en.js and msg/json/qqq.js from
 * msg/messages.js.
 */
function generateMessages(done) {
  // Run js_to_json.py
  const jsToJsonCmd = `python3 scripts/i18n/js_to_json.py \
      --input_file ${path.join('msg', 'messages.js')} \
      --output_dir ${path.join('msg', 'json')} \
      --quiet`;
  execSync(jsToJsonCmd, {stdio: 'inherit'});

  console.log(`
Regenerated several flies in msg/json/.  Now run

    git diff msg/json/*.json

and check that operation has not overwritten any modifications made to
hints, etc. by the TranslateWiki volunteers.  If it has, backport
their changes to msg/messages.js and re-run 'npm run messages'.

Once you are satisfied that any new hints have been backported you may
go ahead and commit the changes, but note that the messages script
will have removed the translator credits - be careful not to commit
this removal!
`);

  done();
}

/**
 * This task builds Blockly's lang files.
 *     msg/*.js
 */
function buildLangfiles(done) {
  // Create output directory.
  const outputDir = path.join(BUILD_DIR, 'msg');
  fs.mkdirSync(outputDir, {recursive: true});

  // Run create_messages.py.
  let json_files = fs.readdirSync(path.join('msg', 'json'));
  json_files = json_files.filter(file => file.endsWith('json') &&
      !(new RegExp(/(keys|synonyms|qqq|constants)\.json$/).test(file)));
  json_files = json_files.map(file => path.join('msg', 'json', file));
  const createMessagesCmd = `python3 ./scripts/i18n/create_messages.py \
  --source_lang_file ${path.join('msg', 'json', 'en.json')} \
  --source_synonym_file ${path.join('msg', 'json', 'synonyms.json')} \
  --source_constants_file ${path.join('msg', 'json', 'constants.json')} \
  --key_file ${path.join('msg', 'json', 'keys.json')} \
  --output_dir ${outputDir} \
  --quiet ${json_files.join(' ')}`;
  execSync(createMessagesCmd, {stdio: 'inherit'});

  done();
}

/**
 * A helper method to return an Closure Compiler chunk wrapper that
 * wraps the compiler output for the given chunk in a Universal Module
 * Definition.
 */
function chunkWrapper(chunk) {
  // Each chunk can have only a single dependency, which is its parent
  // chunk.  It is used only to retrieve the namespace object, which
  // is saved on to the exports object for the chunk so that any child
  // chunk(s) can obtain it.

  // JavaScript expressions for the amd, cjs and browser dependencies.
  let amdDepsExpr = '';
  let cjsDepsExpr = '';
  let browserDepsExpr = '';
  // Arguments for the factory function.
  let factoryArgs = '';
  // Expression to get or create the namespace object.
  let namespaceExpr = `{}`;

  if (chunk.parent) {
    const parentFilename =
        JSON.stringify(`./${chunk.parent.name}${COMPILED_SUFFIX}.js`);
    amdDepsExpr = parentFilename;
    cjsDepsExpr = `require(${parentFilename})`;
    browserDepsExpr = `root.${chunk.parent.reexport}`;
    factoryArgs = '__parent__';
    namespaceExpr = `${factoryArgs}.${NAMESPACE_PROPERTY}`;
  }

  // Code to assign the result of the factory function to the desired
  // export location when running in a browser.  When
  // chunk.reexportOnly is set, this additionally does two other
  // things:
  // - It ensures that only the desired property of the exports object
  //   is assigned to the specified reexport location.
  // - It ensures that the namesspace object is accessible via the
  //   selected sub-object, so that any dependent modules can obtain
  //   it.
  const browserExportStatements = chunk.reexportOnly ?
      `root.${chunk.reexport} = factoryExports.${chunk.reexportOnly};\n` +
      `    root.${chunk.reexport}.${NAMESPACE_PROPERTY} = ` +
      `factoryExports.${NAMESPACE_PROPERTY};` :
      `root.${chunk.reexport} = factoryExports;`;

  // Note that when loading in a browser the base of the exported path
  // (e.g. Blockly.blocks.all - see issue #5932) might not exist
  // before factory has been executed, so calling factory() and
  // assigning the result are done in separate statements to ensure
  // they are sequenced correctly.
  return `// Do not edit this file; automatically generated.

/* eslint-disable */
;(function(root, factory) {
  if (typeof define === 'function' && define.amd) { // AMD
    define([${amdDepsExpr}], factory);
  } else if (typeof exports === 'object') { // Node.js
    module.exports = factory(${cjsDepsExpr});
  } else { // Browser
    var factoryExports = factory(${browserDepsExpr});
    ${browserExportStatements}
  }
}(this, function(${factoryArgs}) {
var ${NAMESPACE_VARIABLE}=${namespaceExpr};
%output%
${chunk.exports}.${NAMESPACE_PROPERTY}=${NAMESPACE_VARIABLE};
return ${chunk.exports};
}));
`;
}

/**
 * Get chunking options to pass to Closure Compiler by using
 * closure-calculate-chunks (hereafter "ccc") to generate them based
 * on the deps.js file (which must be up to date!).
 *
 * The generated options are modified to use the original chunk names
 * given in chunks instead of the entry-point based names used by ccc.
 *
 * @return {{chunk: !Array<string>, js: !Array<string>}} The chunking
 *     information, in the same form as emitted by
 *     closure-calculate-chunks.
 */
function getChunkOptions() {
  const basePath =
      path.join(TSC_OUTPUT_DIR, 'closure', 'goog', 'base_minimal.js');
  const cccArgs = [
    `--closure-library-base-js-path ./${basePath}`,
    `--deps-file './${DEPS_FILE}'`,
    ...(chunks.map(chunk => `--entrypoint '${chunk.entry}'`)),
  ];
  const cccCommand = `closure-calculate-chunks ${cccArgs.join(' ')}`;

  const rawOptions = JSON.parse(execSync(cccCommand));

  // rawOptions should now be of the form:
  //
  // {
  //   chunk: [
  //     'blockly:258',
  //     'all:10:blockly',
  //     'all1:11:blockly',
  //     'all2:11:blockly',
  //     /* ... remaining handful of chunks */
  //   ],
  //   js: [
  //     './build/ts/core/serialization/workspaces.js',
  //     './build/ts/core/serialization/variables.js',
  //     /* ... remaining several hundred files */
  //   ],
  // }
  //
  // This is designed to be passed directly as-is as the options
  // object to the Closure Compiler node API, but we want to replace
  // the unhelpful entry-point based chunk names (let's call these
  // "nicknames") with the ones from chunks.  Unforutnately there's no
  // guarnatee they will be in the same order that the entry points
  // were supplied in (though it happens to work out that way if no
  // chunk depends on any chunk but the first), so we look for
  // one of the entrypoints amongst the files in each chunk.
  const chunkByNickname = Object.create(null);
  // Copy and convert to posix js file paths.
  // Result will be modified via `.splice`!
  const jsFiles = rawOptions.js.map(p => posixPath(p));
  const chunkList = rawOptions.chunk.map((element) => {
    const [nickname, numJsFiles, parentNick] = element.split(':');

    // Get array of files for just this chunk.
    const chunkFiles = jsFiles.splice(0, numJsFiles);

    // Figure out which chunk this is by looking for one of the
    // known chunk entrypoints in chunkFiles.  N.B.: O(n*m).  :-(
    const chunk = chunks.find(
        chunk => chunkFiles.find(f => f.endsWith('/' + chunk.entry)));
    if (!chunk) throw new Error('Unable to identify chunk');

    // Replace nicknames with the names we chose.
    chunkByNickname[nickname] = chunk;
    if (!parentNick) {  // Chunk has no parent.
      chunk.parent = null;
      return `${chunk.name}:${numJsFiles}`;
    }
    chunk.parent = chunkByNickname[parentNick];
    return `${chunk.name}:${numJsFiles}:${chunk.parent.name}`;
  });

  // Generate a chunk wrapper for each chunk.
  for (const chunk of chunks) {
    chunk.wrapper = chunkWrapper(chunk);
  }
  const chunkWrappers = chunks.map(chunk => `${chunk.name}:${chunk.wrapper}`);

  return {chunk: chunkList, js: rawOptions.js, chunk_wrapper: chunkWrappers};
}

/**
 * RegExp that globally matches path.sep (i.e., "/" or "\").
 */
const pathSepRegExp = new RegExp(path.sep.replace(/\\/, '\\\\'), 'g');

/**
 * Helper method for calling the Closure Compiler, establishing
 * default options (that can be overridden by the caller).
 * @param {*} options Caller-supplied options that will override the
 *     defaultOptions.
 */
function compile(options) {
  const defaultOptions = {
    compilation_level: 'SIMPLE_OPTIMIZATIONS',
    warning_level: argv.verbose ? 'VERBOSE' : 'DEFAULT',
    language_in: 'ECMASCRIPT_2020',
    language_out: 'ECMASCRIPT_2015',
    jscomp_off: [...JSCOMP_OFF],
    rewrite_polyfills: true,
    // N.B.: goog.js refers to lots of properties on goog that are not
    // declared by base_minimal.js, while if you compile against
    // base.js instead you will discover that it uses @deprecated
    // inherits, forwardDeclare etc.
    hide_warnings_for: [
      'node_modules',
      path.join(TSC_OUTPUT_DIR, 'closure', 'goog', 'goog.js'),
    ],
    define: ['COMPILED=true'],
  };
  if (argv.debug || argv.strict) {
    defaultOptions.jscomp_error = [...JSCOMP_ERROR];
    defaultOptions.jscomp_warning = [...JSCOMP_WARNING];
    if (argv.strict) {
      defaultOptions.jscomp_error.push('strictCheckTypes');
    }
  }
  // Extra options for Closure Compiler gulp plugin.
  const platform = ['native', 'java', 'javascript'];

  return closureCompiler({...defaultOptions, ...options}, {platform});
}

/**
 * This task compiles the core library, blocks and generators, creating
 * blockly_compressed.js, blocks_compressed.js, etc.
 *
 * The deps.js file must be up-to-date.
 *
 * Prerequisite: buildDeps.
 */
function buildCompiled() {
  // Get chunking.
  const chunkOptions = getChunkOptions();
  // Closure Compiler options.
  const packageJson = getPackageJson();  // For version number.
  const options = {
    // The documentation for @define claims you can't use it on a
    // non-global, but the Closure Compiler turns everything in to a
    // global - you just have to know what the new name is!  With
    // declareLegacyNamespace this was very straightforward.  Without
    // it, we have to rely on implmentation details.  See
    // https://github.com/google/closure-compiler/issues/1601#issuecomment-483452226
    define: `VERSION$$${chunks[0].exports}='${packageJson.version}'`,
    chunk: chunkOptions.chunk,
    chunk_wrapper: chunkOptions.chunk_wrapper,
    rename_prefix_namespace: NAMESPACE_VARIABLE,
    // Don't supply the list of source files in chunkOptions.js as an
    // option to Closure Compiler; instead feed them as input via gulp.src.
  };

  // Fire up compilation pipline.
  return gulp.src(chunkOptions.js, {base: './'})
      .pipe(stripApacheLicense())
      .pipe(gulp.sourcemaps.init())
      .pipe(compile(options))
      .pipe(gulp.rename({suffix: COMPILED_SUFFIX}))
      .pipe(gulp.sourcemaps.write('.'))
      .pipe(gulp.dest(RELEASE_DIR));
}

/**
 * This task builds Blockly core, blocks and generators together and uses
 * Closure Compiler's ADVANCED_COMPILATION mode.
 *
 * Prerequisite: buildDeps.
 */
function buildAdvancedCompilationTest() {
  // If main_compressed.js exists (from a previous run) delete it so that
  // a later browser-based test won't check it should the compile fail.
  try {
    fs.unlinkSync('./tests/compile/main_compressed.js');
  } catch (_e) {
    // Probably it didn't exist.
  }

  const srcs = [
    TSC_OUTPUT_DIR + '/**/*.js',
    'tests/compile/main.js',
    'tests/compile/test_blocks.js',
  ];
  const ignore = [
    TSC_OUTPUT_DIR + '/closure/goog/base.js',  // Use base_minimal.js only.
  ];

  // Closure Compiler options.
  const options = {
    dependency_mode: 'PRUNE',
    compilation_level: 'ADVANCED_OPTIMIZATIONS',
    entry_point: './tests/compile/main.js',
    js_output_file: 'main_compressed.js',
  };
  return gulp.src(srcs, {base: './', ignore})
      .pipe(stripApacheLicense())
      .pipe(gulp.sourcemaps.init())
      .pipe(compile(options))
      .pipe(gulp.sourcemaps.write(
          '.', {includeContent: false, sourceRoot: '../../'}))
      .pipe(gulp.dest('./tests/compile/'));
}

/**
 * This task cleans the build directory (by deleting it).
 */
function cleanBuildDir() {
  // Sanity check.
  if (BUILD_DIR === '.' || BUILD_DIR === '/') {
    return Promise.reject(`Refusing to rm -rf ${BUILD_DIR}`);
  }
  return rimraf(BUILD_DIR);
}

/**
 * Runs clang format on all files in the core directory.
 */
function format() {
  return gulp.src([
    'core/**/*.js', 'core/**/*.ts',
    'blocks/**/*.js', 'blocks/**/*.ts',
    '.eslintrc.js'
  ], {base: '.'})
      .pipe(clangFormatter.format('file', clangFormat))
      .pipe(gulp.dest('.'));
}

// Main sequence targets.  Each should invoke any immediate prerequisite(s).
exports.cleanBuildDir = cleanBuildDir;
exports.langfiles = buildLangfiles;  // Build build/msg/*.js from msg/json/*.
exports.tsc = buildJavaScript;
exports.deps = gulp.series(exports.tsc, buildDeps);
exports.minify = gulp.series(exports.deps, buildCompiled);
exports.build = gulp.parallel(exports.minify, exports.langfiles);

// Manually-invokable targets, with prerequisites where required.
exports.format = format;
exports.messages = generateMessages;  // Generate msg/json/en.json et al.
exports.buildAdvancedCompilationTest =
    gulp.series(exports.deps, buildAdvancedCompilationTest);

// Targets intended only for invocation by scripts; may omit prerequisites.
exports.onlyBuildAdvancedCompilationTest = buildAdvancedCompilationTest;