Allow mjs routes (#30378)

* make it so that src transformation only happens strictly in scripts/script-transform.ts

each module should do 1 thing and 1 thing only. currently the module
transformer also did a src/url transformation. this caused weird
mismatches when they are ran together.

- added looseScriptSrcCheck to allow for non cdn domains. This will
  allow us to transition from the bad fixture files to having all html
  files be valid AMP HTML by default.

* temp

* temp

* fix imports

* add transform back to integration

* fix html

* fix integration

* temp

* temp

* make transformers have a loose mode to be more forgiving on script src

* add more tests and fix bug for extention retention

* allow for mjs files to be served by the test server

* add glob to load mjs files in karma server

* lint fixes

* lint html

* remove typescript changes

* revert test/fixtures changes

* remove more new server changes

* apply recs

* apply fixes

* fix missing \

* revert expansion

build-system/server/app-utils.js

@@ -127,51 +127,51 @@ const replaceUrls = (mode, file, hostName, inabox, storyV1) => {
     }
   } else if (mode == 'compiled') {
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/v0\.js/g,
-      hostName + '/dist/v0.js'
+      /https:\/\/cdn\.ampproject\.org\/v0\.(m?js)/g,
+      hostName + '/dist/v0.$1'
     );
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/shadow-v0\.js/g,
-      hostName + '/dist/shadow-v0.js'
+      /https:\/\/cdn\.ampproject\.org\/shadow-v0\.(m?js)/g,
+      hostName + '/dist/shadow-v0.$1'
     );
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/amp4ads-v0\.js/g,
-      hostName + '/dist/amp4ads-v0.js'
+      /https:\/\/cdn\.ampproject\.org\/amp4ads-v0\.(m?js)/g,
+      hostName + '/dist/amp4ads-v0.$1'
     );
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/video-iframe-integration-v0\.js/g,
-      hostName + '/dist/video-iframe-integration-v0.js'
+      /https:\/\/cdn\.ampproject\.org\/video-iframe-integration-v0\.(m?js)/g,
+      hostName + '/dist/video-iframe-integration-v0.$1'
     );
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/v0\/(.+?).js/g,
-      hostName + '/dist/v0/$1.js'
+      /https:\/\/cdn\.ampproject\.org\/v0\/(.+?)\.(m?js)/g,
+      hostName + '/dist/v0/$1.$2'
     );
     file = file.replace(
-      /\/dist\/v0\/examples\/(.*)\.max.js/g,
-      '/dist/v0/examples/$1.js'
+      /\/dist\/v0\/examples\/(.*)\.max\.(m?js)/g,
+      '/dist/v0/examples/$1.$2'
     );
     file = file.replace(
       /\/dist.3p\/current\/(.*)\.max.html/g,
       hostName + '/dist.3p/current-min/$1.html'
     );
 
     if (inabox) {
-      file = file.replace(/\/dist\/v0\.js/g, '/dist/amp4ads-v0.js');
+      file = file.replace(/\/dist\/v0\.(m?js)/g, '/dist/amp4ads-v0.$1');
     }
   } else if (isRtvMode(mode)) {
     hostName = `https://cdn.ampproject.org/rtv/${mode}/`;
     file = file.replace(/https:\/\/cdn\.ampproject\.org\//g, hostName);
 
     if (inabox) {
       file = file.replace(
-        /https:\/\/cdn\.ampproject\.org\/rtv\/\d{15}\/v0\.js/g,
-        hostName + 'amp4ads-v0.js'
+        /https:\/\/cdn\.ampproject\.org\/rtv\/\d{15}\/v0\.(m?js)/g,
+        hostName + 'amp4ads-v0.$1'
       );
     }
   } else if (inabox) {
     file = file.replace(
-      /https:\/\/cdn\.ampproject\.org\/v0\.js/g,
-      'https://cdn.ampproject.org/amp4ads-v0.js'
+      /https:\/\/cdn\.ampproject\.org\/v0\.(m?js)/g,
+      'https://cdn.ampproject.org/amp4ads-v0.$1'
     );
   }
   return file;

build-system/server/app.js

@@ -911,15 +911,15 @@ app.get('/iframe-echo-message', (req, res) => {
  * <script async custom-element="amp-form"
  *    src="https://cdn.ampproject.org/v0/amp-form-0.1.js?sleep=5"></script>
  */
-app.use(['/dist/v0/amp-*.js', '/dist/amp*.js'], (req, res, next) => {
+app.use(['/dist/v0/amp-*.(m?js)', '/dist/amp*.(m?js)'], (req, res, next) => {
   const sleep = parseInt(req.query.sleep || 0, 10) * 1000;
   setTimeout(next, sleep);
 });
 
 /**
  * Disable caching for extensions if the --no_caching_extensions flag is used.
  */
-app.get(['/dist/v0/amp-*.js'], (req, res, next) => {
+app.get(['/dist/v0/amp-*.(m?js)'], (req, res, next) => {
   if (argv.no_caching_extensions) {
     res.header('Cache-Control', 'no-store');
   }
@@ -1202,7 +1202,7 @@ app.get('/adzerk/*', (req, res) => {
  * Serve extension scripts and their source maps.
  */
 app.get(
-  ['/dist/rtv/*/v0/*.js', '/dist/rtv/*/v0/*.js.map'],
+  ['/dist/rtv/*/v0/*.(m?js)', '/dist/rtv/*/v0/*.(m?js).map'],
   (req, res, next) => {
     const mode = SERVE_MODE;
     const fileName = path.basename(req.path).replace('.max.', '.');
@@ -1222,7 +1222,7 @@ app.get(
     }
     const isJsMap = filePath.endsWith('.map');
     if (isJsMap) {
-      filePath = filePath.replace(/\.js\.map$/, '.js');
+      filePath = filePath.replace(/\.(m?js)\.map$/, '.$1');
     }
     filePath = replaceUrls(mode, filePath);
     req.url = filePath + (isJsMap ? '.map' : '');
@@ -1234,7 +1234,7 @@ app.get(
  * Serve entry point script url
  */
 app.get(
-  ['/dist/sw.js', '/dist/sw-kill.js', '/dist/ww.js'],
+  ['/dist/sw.(m?js)', '/dist/sw-kill.(m?js)', '/dist/ww.(m?js)'],
   (req, res, next) => {
     // Special case for entry point script url. Use compiled for testing
     const mode = SERVE_MODE;
@@ -1254,18 +1254,18 @@ app.get(
       return;
     }
     if (mode == 'default') {
-      req.url = req.url.replace(/\.js$/, '.max.js');
+      req.url = req.url.replace(/\.(m?js)$/, '.max.$1');
     }
     next();
   }
 );
 
-app.get('/dist/iframe-transport-client-lib.js', (req, res, next) => {
+app.get('/dist/iframe-transport-client-lib.(m?js)', (req, res, next) => {
   req.url = req.url.replace(/dist/, 'dist.3p/current');
   next();
 });
 
-app.get('/dist/amp-inabox-host.js', (req, res, next) => {
+app.get('/dist/amp-inabox-host.(m?js)', (req, res, next) => {
   const mode = SERVE_MODE;
   if (mode != 'default') {
     req.url = req.url.replace('amp-inabox-host', 'amp4ads-host-v0');
@@ -1276,7 +1276,7 @@ app.get('/dist/amp-inabox-host.js', (req, res, next) => {
 /*
  * Start Cache SW LOCALDEV section
  */
-app.get('/dist/sw(.max)?.js', (req, res, next) => {
+app.get('/dist/sw(.max)?.(m?js)', (req, res, next) => {
   const filePath = req.path;
   fs.promises
     .readFile(pc.cwd() + filePath, 'utf8')
@@ -1301,7 +1301,7 @@ app.get('/dist/sw(.max)?.js', (req, res, next) => {
     .catch(next);
 });
 
-app.get('/dist/rtv/9[89]*/*.js', (req, res, next) => {
+app.get('/dist/rtv/9[89]*/*.(m?js)', (req, res, next) => {
   res.setHeader('Content-Type', 'application/javascript');
   res.setHeader('Date', new Date().toUTCString());
   res.setHeader('Cache-Control', 'no-cache;max-age=31536000');
@@ -1373,7 +1373,7 @@ app.get('/dist/diversions', (req, res) => {
 /**
  * Web worker binary.
  */
-app.get('/dist/ww(.max)?.js', (req, res) => {
+app.get('/dist/ww(.max)?.(m?js)', (req, res) => {
   fs.promises.readFile(pc.cwd() + req.path).then((file) => {
     res.setHeader('Content-Type', 'text/javascript');
     res.setHeader('Access-Control-Allow-Origin', '*');

build-system/test-configs/config.js

@@ -40,6 +40,12 @@ const builtRuntimePaths = [
     nocache: false,
     watched: true,
   },
+  {
+    pattern: 'dist/**/*.mjs',
+    included: false,
+    nocache: false,
+    watched: true,
+  },
   {
     pattern: 'dist.3p/**/*',
     included: false,
@@ -52,6 +58,12 @@ const builtRuntimePaths = [
     nocache: false,
     watched: true,
   },
+  {
+    pattern: 'dist.tools/**/*.mjs',
+    included: false,
+    nocache: false,
+    watched: true,
+  },
 ];
 
 const commonUnitTestPaths = initTestsPath.concat(fixturesExamplesPaths);