build: publish tree of files rather than FESMs (angular#18541)

* Remove now unnecessary portions of build.
* Add a compilePackageES5 method to build ES5 from sources
* Rework all package.json and rollup config files to new format
* Remove "extends" from tsconfig-build.json files and fixup compilation roots

PR Close angular#18541

Author: jasonaden

build.sh

@@ -26,7 +26,8 @@ PACKAGES=(core
   benchpress)
 
 NODE_PACKAGES=(compiler-cli
-  benchpress)
+  benchpress
+  tsc-wrapped)
 
 BUILD_ALL=true
 BUNDLE=true
@@ -107,76 +108,36 @@ containsElement () {
   return 1
 }
 
-#######################################
-# Downlevel ES2015 to ESM/ES5
-# Arguments:
-#   param1 - Source folder
-#   param2 - Naming suffix to apply. Must end in ".ts" (defaults to .es5.ts)
-# Returns:
-#   None
-#######################################
-downlevelES2015() {
-  # Iterate over the files in this directory, converting to .es5.ts
-  regex="(.+).js"
-  for file in ${1}/*.js ; do
-    if [[ ${file} =~ $regex ]]; then
-      ts_file="${BASH_REMATCH[1]}${2:-".es5.ts"}"
-      cp ${file} ${ts_file}
-
-      echo "======           $TSC ${ts_file} --target es5 --module es2015 --noLib --sourceMap --importHelpers"
-      ($TSC ${ts_file} --target es5 --module es2015 --noLib --sourceMap --importHelpers) > /dev/null 2>&1 || true
-      mapSources "${BASH_REMATCH[1]}${2:-".es5.js"}"
-      rm -f ${ts_file}
-    fi
-  done
-
-  # Recurse for sub directories
-  for DIR in ${1}/* ; do
-    isIgnoredDirectory ${DIR} && continue
-    downlevelES2015 ${DIR}
-  done
-}
-
 #######################################
 # Rollup index files recursively, ignoring blacklisted directories
 # Arguments:
 #   param1 - Base source folder
 #   param2 - Destination directory
-#   param3 - Config file
 # Returns:
 #   None
 #######################################
 rollupIndex() {
   # Iterate over the files in this directory, rolling up each into ${2} directory
   local regex=".+/(.+)/index.js"
-  if [[ "${1}/index.js" =~ $regex ]]; then
-    in_file="${1}/index.js"
-    out_file="${2}/${BASH_REMATCH[1]}.js"
+  in_file="${1}/index.js"
+  out_file="${2}/index.js"
 
-    echo "======           $ROLLUP -i ${in_file} -o ${out_file}"
+  BANNER_TEXT=`cat ${LICENSE_BANNER}`
 
-    if [[ -f "${3}" ]]; then
-      $ROLLUP -i ${in_file} -o ${out_file} -c ${3} --sourcemap
-    else
-      $ROLLUP -i ${in_file} -o ${out_file} --sourcemap
-    fi
-    cat ${LICENSE_BANNER} > ${out_file}.tmp
-    cat ${out_file} >> ${out_file}.tmp
-    mv ${out_file}.tmp ${out_file}
-
-    mapSources "${out_file}"
-
-    # Recurse for sub directories
-    for DIR in ${1}/* ; do
-      isIgnoredDirectory ${DIR} && continue
-      # NOTE: We need to re-run this regex and use the new match as Bash doesn't have closures
-      if [[ "${1}/index.js" =~ $regex ]]; then
-        rollupIndex ${DIR} ${2}/${BASH_REMATCH[1]} "$(dirname $3)/${BASH_REMATCH[1]}/rollup.config.js"
-      fi
-    done
+  if [[ -f ${in_file} ]]; then
+    echo "===========           $ROLLUP -i ${in_file} -o ${out_file} -c ${ROOT_DIR}/rollup.config.js --sourcemap"
+    $ROLLUP -i ${in_file} -o ${out_file} --sourcemap -f es --banner "$BANNER_TEXT" >/dev/null 2>&1
   fi
-}
 
+  # Recurse for sub directories
+  for DIR in ${1}/* ; do
+    isIgnoredDirectory ${DIR} && continue
+    # NOTE: We need to re-run this regex and use the new match as Bash doesn't have closures
+    if [[ "${DIR}/index.js" =~ $regex ]]; then
+      rollupIndex ${DIR} ${2}/${BASH_REMATCH[1]}
+    fi
+  done
+}
 
 #######################################
 # Recursively runs rollup on any entry point that has a "rollup.config.js" file
@@ -186,17 +147,11 @@ rollupIndex() {
 #   None
 #######################################
 runRollup() {
-  local regex="dest: ['\"](.+)['\"],*"
   if [[ -f "${1}/rollup.config.js" ]]; then
     cd ${1}
 
     echo "======           $ROLLUP -c ${1}/rollup.config.js"
-    $ROLLUP -c rollup.config.js --sourcemap
-
-    local dest_line=$(cat "${1}/rollup.config.js" | grep 'dest:')
-    if [[ ${dest_line} =~ $regex ]]; then
-      mapSources "${BASH_REMATCH[1]}"
-    fi
+    $ROLLUP -c rollup.config.js --sourcemap >/dev/null 2>&1
 
     # Recurse for sub directories
     for DIR in ${1}/* ; do
@@ -241,7 +196,6 @@ minify() {
     if [[ "${base_file}" =~ $regex && "${base_file##*.}" != "map" ]]; then
       local out_file=$(dirname "${file}")/${BASH_REMATCH[1]}.min.js
       $UGLIFYJS -c --screw-ie8 --comments -o ${out_file} --source-map ${out_file}.map --source-map-include-sources ${file}
-      mapSources "${out_file}"
     fi
   done
 }
@@ -252,19 +206,18 @@ minify() {
 #   param1 - Source directory
 #   param2 - Out dir
 #   param3 - Package Name
-#   param4 - Is child (are we recursing?)
 # Returns:
 #   None
 #######################################
 compilePackage() {
   echo "======      [${3}]: COMPILING: ${NGC} -p ${1}/tsconfig-build.json"
   # For NODE_PACKAGES items (not getting rolled up)
-  if containsElement "${PACKAGE}" "${NODE_PACKAGES[@]}"; then
+  if containsElement "${3}" "${NODE_PACKAGES[@]}"; then
     $TSC -p ${1}/tsconfig-build.json
   else
     local package_name=$(basename "${2}")
     $NGC -p ${1}/tsconfig-build.json
-    echo "======           Create ${1}/../${package_name}.d.ts re-export file for Closure"
+    echo "======           Create ${1}/../${package_name}.d.ts re-export file for tsickle"
     echo "$(cat ${LICENSE_BANNER}) ${N} export * from './${package_name}/index'" > ${2}/../${package_name}.d.ts
     echo "{\"__symbolic\":\"module\",\"version\":3,\"metadata\":{},\"exports\":[{\"from\":\"./${package_name}/index\"}],\"flatModuleIndexRedirect\":true}" > ${2}/../${package_name}.metadata.json
   fi
@@ -274,24 +227,31 @@ compilePackage() {
     BASE_DIR=$(basename "${DIR}")
     # Skip over directories that are not nested entry points
     [[ -e ${DIR}/tsconfig-build.json && "${BASE_DIR}" != "integrationtest" ]] || continue
-    compilePackage ${DIR} ${2}/${BASE_DIR} ${3} true
+    compilePackage ${DIR} ${2}/${BASE_DIR} ${3}
   done
 }
 
 #######################################
-# Moves typings and metadata files appropriately
+# Recursively compile package
 # Arguments:
-#   param1 - Source of typings & metadata files
-#   param2 - Root of destination directory
-#   param3 - Package name (needed to correspond to name of d.ts and metadata.json files)
+#   param1 - Source directory
+#   param2 - Out dir
+#   param3 - Package Name
 # Returns:
 #   None
 #######################################
-moveTypings() {
-  if [[ -f ${1}/index.d.ts && -f ${1}/index.metadata.json ]]; then
-    mv ${1}/index.d.ts ${1}/${2}.d.ts
-    mv ${1}/index.metadata.json ${1}/${2}.metadata.json
-  fi
+compilePackageES5() {
+  echo "======      [${3}]: COMPILING: ${NGC} -p ${1}/tsconfig-build.json --target es5 -d false --outDir ${2} --importHelpers true --sourceMap"
+  local package_name=$(basename "${2}")
+  $NGC -p ${1}/tsconfig-build.json --target es5 -d false --outDir ${2} --importHelpers true --sourceMap
+
+  for DIR in ${1}/* ; do
+    [ -d "${DIR}" ] || continue
+    BASE_DIR=$(basename "${DIR}")
+    # Skip over directories that are not nested entry points
+    [[ -e ${DIR}/tsconfig-build.json && "${BASE_DIR}" != "integrationtest" ]] || continue
+    compilePackageES5 ${DIR} ${2} ${3}
+  done
 }
 
 #######################################
@@ -314,19 +274,6 @@ addNgcPackageJson() {
   done
 }
 
-#######################################
-# This is read by NGC to be able to find the flat module index.
-# Arguments:
-#   param1 - JavaScript file on which to re-map sources
-# Returns:
-#   None
-#######################################
-mapSources() {
-  if [[ -f "${1}.map" ]]; then
-    $MAP_SOURCES -f "${1}"
-  fi
-}
-
 updateVersionReferences() {
   NPM_DIR="$1"
   (
@@ -343,8 +290,7 @@ echo "====== BUILDING: Version ${VERSION}"
 N="
 "
 TSC=`pwd`/node_modules/.bin/tsc
-NGC="node --max-old-space-size=3000 dist/packages-dist/tsc-wrapped/src/main"
-MAP_SOURCES="node `pwd`/scripts/build/map_sources.js "
+NGC="node --max-old-space-size=3000 `pwd`/dist/packages-dist/tsc-wrapped/src/main"
 UGLIFYJS=`pwd`/node_modules/.bin/uglifyjs
 TSCONFIG=./tools/tsconfig.json
 ROLLUP=`pwd`/node_modules/.bin/rollup
@@ -446,8 +392,10 @@ do
   SRC_DIR=${ROOT_DIR}/${PACKAGE}
   ROOT_OUT_DIR=${PWD}/dist/packages
   OUT_DIR=${ROOT_OUT_DIR}/${PACKAGE}
+  OUT_DIR_ESM5=${ROOT_OUT_DIR}/${PACKAGE}/esm5
   NPM_DIR=${PWD}/dist/packages-dist/${PACKAGE}
-  MODULES_DIR=${NPM_DIR}/@angular
+  ESM15_DIR=${NPM_DIR}/esm15
+  ESM5_DIR=${NPM_DIR}/esm5
   BUNDLES_DIR=${NPM_DIR}/bundles
 
   LICENSE_BANNER=${ROOT_DIR}/license-banner.txt
@@ -466,15 +414,15 @@ do
 
       echo "======        Copy ${PACKAGE} typings"
       rsync -a --exclude=*.js --exclude=*.js.map ${OUT_DIR}/ ${NPM_DIR}
-      moveTypings ${NPM_DIR} ${PACKAGE}
 
       (
         cd  ${SRC_DIR}
         echo "======         Rollup ${PACKAGE}"
-        rollupIndex ${OUT_DIR} ${MODULES_DIR} ${ROOT_DIR}
+        rollupIndex ${OUT_DIR} ${ESM15_DIR} ${ROOT_DIR}
 
-        echo "======         Downleveling ES2015 to ESM/ES5"
-        downlevelES2015 ${MODULES_DIR}
+        echo "======         Produce ESM5 version"
+        compilePackageES5 ${SRC_DIR} ${OUT_DIR_ESM5} ${PACKAGE}
+        rollupIndex ${OUT_DIR_ESM5} ${ESM5_DIR} ${ROOT_DIR}
 
         echo "======         Run rollup conversions on ${PACKAGE}"
         runRollup ${SRC_DIR}

integration/hello_world__closure/closure.conf

@@ -13,18 +13,20 @@
 node_modules/zone.js/dist/zone_externs.js
 
 --js node_modules/rxjs/**.js
---process_common_js_modules
---module_resolution=node
 
-node_modules/@angular/core/@angular/core.js
---js_module_root=node_modules/@angular/core
-node_modules/@angular/core/src/testability/testability.externs.js
+--js node_modules/@angular/core/package.json
+--js node_modules/@angular/core/esm15/index.js
+--js node_modules/@angular/core/src/testability/testability.externs.js
+
+--js node_modules/@angular/common/package.json
+--js node_modules/@angular/common/esm15/index.js
 
-node_modules/@angular/common/@angular/common.js
---js_module_root=node_modules/@angular/common
+--js node_modules/@angular/platform-browser/package.json
+--js node_modules/@angular/platform-browser/esm15/index.js
 
-node_modules/@angular/platform-browser/@angular/platform-browser.js
---js_module_root=node_modules/@angular/platform-browser
+--module_resolution=node
+--package_json_entry_names es2015
+--process_common_js_modules
 
 --js built/**.js
 --entry_point=built/src/main

integration/hello_world__closure/package.json

@@ -11,7 +11,7 @@
     "@angular/platform-browser": "file:../../dist/packages-dist/platform-browser",
     "@angular/platform-server": "file:../../dist/packages-dist/platform-server",
     "@angular/tsc-wrapped": "file:../../dist/packages-dist/tsc-wrapped",
-    "google-closure-compiler": "20170409.0.0",
+    "google-closure-compiler": "git+https://github.com/alexeagle/closure-compiler.git#packagejson.dist",
     "rxjs": "5.3.1",
     "typescript": "~2.3.1",
     "zone.js": "0.8.6"
@@ -29,4 +29,4 @@
     "preprotractor": "tsc -p e2e",
     "protractor": "protractor e2e/protractor.config.js"
   }
-}
+}

integration/i18n/closure.conf

@@ -15,16 +15,17 @@ node_modules/zone.js/dist/zone_externs.js
 --js node_modules/rxjs/**.js
 --process_common_js_modules
 --module_resolution=node
+--package_json_entry_names es2015
 
-node_modules/@angular/core/@angular/core.js
---js_module_root=node_modules/@angular/core
-node_modules/@angular/core/src/testability/testability.externs.js
+--js node_modules/@angular/core/package.json
+--js node_modules/@angular/core/esm15/index.js
+--js node_modules/@angular/core/src/testability/testability.externs.js
 
-node_modules/@angular/common/@angular/common.js
---js_module_root=node_modules/@angular/common
+--js node_modules/@angular/common/package.json
+--js node_modules/@angular/common/esm15/index.js
 
-node_modules/@angular/platform-browser/@angular/platform-browser.js
---js_module_root=node_modules/@angular/platform-browser
+--js node_modules/@angular/platform-browser/package.json
+--js node_modules/@angular/platform-browser/esm15/index.js
 
 --js built/**.js
 --entry_point=built/src/main

integration/i18n/package.json

@@ -11,7 +11,7 @@
     "@angular/platform-browser": "file:../../dist/packages-dist/platform-browser",
     "@angular/platform-server": "file:../../dist/packages-dist/platform-server",
     "@angular/tsc-wrapped": "file:../../dist/packages-dist/tsc-wrapped",
-    "google-closure-compiler": "20170409.0.0",
+    "google-closure-compiler": "git+https://github.com/alexeagle/closure-compiler.git#packagejson.dist",
     "rxjs": "5.3.1",
     "typescript": "~2.3.1",
     "zone.js": "0.8.6"