🎉 Thumbor

.dockerignore

@@ -13,6 +13,7 @@ MAINTENANCE.md
 README.md
 CONTRIBUTING.md
 packager/
+thumbor/
 
 # The complete frontend and pages folders aren't needed on App Engine
 /frontend/

.gcloudignore

@@ -24,6 +24,7 @@ MAINTENANCE.md
 README.md
 CONTRIBUTING.md
 packager/
+thumbor/
 
 # The complete frontend and pages folders aren't needed on App Engine
 /frontend/

.gitignore

@@ -40,6 +40,7 @@ pages/shared/data/roadmap.yaml
 platform/pages*
 platform/static
 platform/config/build-info.yaml
+thumbor/static
 
 # Ignore pipeline cache
 .cache/**/*

.travis.yml

@@ -103,4 +103,6 @@ jobs:
     - scripts/unbuffer.sh gulp buildFinalize
     - scripts/unbuffer.sh npm run smoke-test
     script:
+    - scripts/unbuffer.sh gulp thumborCollectImages
+    - gcloud app deploy thumbor/app.yaml --project=amp-dev-staging --quiet --version=1
     - gcloud app deploy --project=amp-dev-staging --quiet --version=1

gulpfile.js/deploy.js

@@ -20,10 +20,11 @@ const {series} = require('gulp');
 const {join} = require('path');
 const {sh} = require('@lib/utils/sh.js');
 const mri = require('mri');
-const {ROOT} = require('@lib/utils/project').paths;
+const {ROOT, THUMBOR_ROOT} = require('@lib/utils/project').paths;
 
 const PREFIX = 'amp-dev';
 const PACKAGER_PREFIX = PREFIX + '-packager';
+const THUMBOR_PREFIX = PREFIX + '-thumbor';
 
 // Parse commandline arguments
 const argv = mri(process.argv.slice(2));
@@ -94,6 +95,28 @@ const config = {
       current: `gcr.io/${PROJECT_ID}/${PACKAGER_PREFIX}:${TAG}`,
     },
   },
+  thumbor: {
+    opts: {
+      workingDir: THUMBOR_ROOT,
+    },
+    prefix: THUMBOR_PREFIX,
+    tag: TAG,
+    instance: {
+      groups: [
+        {
+          name: `ig-${THUMBOR_PREFIX}`,
+          zone: 'us-east1-b',
+        },
+      ],
+      template: `it-${THUMBOR_PREFIX}-${TAG}`,
+      count: 1,
+      machine: 'n1-standard-1',
+    },
+    image: {
+      name: `gcr.io/${PROJECT_ID}/${THUMBOR_PREFIX}`,
+      current: `gcr.io/${PROJECT_ID}/${THUMBOR_PREFIX}:${TAG}`,
+    },
+  },
 };
 
 /**
@@ -293,6 +316,52 @@ async function packagerUpdateStart() {
   console.log('Rolling update started, this can take a few minutes...');
 }
 
+/* Thumbor */
+/**
+ * Create a new VM instance template based on the latest docker image.
+ */
+function thumborInstanceTemplateCreate() {
+  return sh(
+    `gcloud compute instance-templates create-with-container \
+                                   ${config.thumbor.instance.template} \
+                 --container-image ${config.thumbor.image.current} \
+                 --machine-type ${config.thumbor.instance.machine}`,
+    config.thumbor.opts
+  );
+}
+
+/**
+ * Builds and uploads the thumbor docker image to Google Cloud Container Registry.
+ */
+function thumborImageUpload() {
+  return sh(
+    `gcloud builds submit --tag ${config.thumbor.image.current} .`,
+    config.thumbor.opts
+  );
+}
+
+/**
+ * Start a rolling update to a new thumbor VM instance template. This will ensure
+ * that there's always at least 1 active instance running during the update.
+ */
+async function thumborUpdateStart() {
+  const updates = config.thumbor.instance.groups.map((group) => {
+    return sh(
+      `gcloud beta compute instance-groups managed rolling-action \
+                 start-update ${group.name} \
+                 --version template=${config.thumbor.instance.template} \
+                 --zone=${group.zone} \
+                 --min-ready 1m \
+                 --max-surge 1 \
+                 --max-unavailable 1`,
+      config.thumbor.opts
+    );
+  });
+  await Promise.all(updates);
+
+  console.log('Rolling update started, this can take a few minutes...');
+}
+
 exports.verifyTag = verifyTag;
 exports.gcloudSetup = gcloudSetup;
 exports.deploy = series(
@@ -316,6 +385,11 @@ exports.packagerDeploy = series(
 exports.packagerImageUpload = packagerImageUpload;
 exports.packagerInstanceTemplateCreate = packagerInstanceTemplateCreate;
 exports.packagerUpdateStart = packagerUpdateStart;
+
+exports.thumborImageUpload = thumborImageUpload;
+exports.thumborInstanceTemplateCreate = thumborInstanceTemplateCreate;
+exports.thumborUpdateStart = thumborUpdateStart;
+
 exports.updateStop = updateStop;
 exports.updateStatus = updateStatus;
 exports.updateStart = updateStart;

gulpfile.js/thumbor.js

@@ -0,0 +1,51 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const gulp = require('gulp');
+const {join} = require('path');
+
+const config = require('@lib/config');
+const {sh} = require('@lib/utils/sh.js');
+const {project} = require('@lib/utils');
+
+const IMAGE_TAG = 'amp-dev-thumbor';
+const opts = {
+  workingDir: project.paths.THUMBOR_ROOT,
+};
+
+async function thumborRunLocal() {
+  await sh('pwd', opts);
+  await sh(`docker build -t ${IMAGE_TAG} .`, opts);
+  return await sh(
+    `docker run -p ${config.hosts.thumbor.port}:8080 ${IMAGE_TAG}`,
+    opts
+  );
+}
+
+async function thumborCollectImages() {
+  const imagePaths = config.shared.thumbor.fileExtensions.map((extension) => {
+    return join(project.paths.STATICS_DEST, '/**/', `*.${extension}`);
+  });
+
+  return gulp
+    .src(imagePaths)
+    .pipe(gulp.dest(`${project.paths.THUMBOR_STATICS_DEST}`));
+}
+
+exports.thumborRunLocal = thumborRunLocal;
+exports.thumborCollectImages = thumborCollectImages;

package.json

@@ -57,7 +57,7 @@
   ],
   "dependencies": {
     "@ampproject/toolbox-cors": "2.4.0-alpha.1",
-    "@ampproject/toolbox-optimizer": "2.4.0",
+    "@ampproject/toolbox-optimizer": "2.5.0-alpha.0",
     "@google-cloud/datastore": "5.1.0",
     "acorn": "7.2.0",
     "casual": "1.6.2",

platform/config/environments/development.json

@@ -44,6 +44,11 @@
       "scheme": "https",
       "host": "amp-dev-sxg.appspot.com",
       "port": ""
+    },
+    "thumbor": {
+      "scheme": "http",
+      "host": "localhost",
+      "port": "8088"
     }
   }
 }

platform/config/environments/local.json

@@ -44,6 +44,11 @@
       "scheme": "https",
       "host": "amp-dev-sxg.appspot.com",
       "port": ""
+    },
+    "thumbor": {
+      "scheme": "http",
+      "host": "localhost",
+      "port": "8088"
     }
   }
 }

platform/config/environments/staging.json

@@ -44,6 +44,11 @@
       "scheme": "https",
       "host": "amp-dev-sxg.appspot.com",
       "port": ""
+    },
+		"thumbor": {
+      "scheme": "https",
+      "host": "thumbor-dot-amp-dev-staging.appspot.com",
+      "port": ""
     }
 	},
 	"redis": {

platform/config/shared.json

@@ -3,5 +3,13 @@
 	 	"playground": "/#url=",
 		"repository": "https://github.com/ampproject/docs/blob/future/"
 	},
-  "gaTrackingId": "UA-67833617-1"
+  "gaTrackingId": "UA-67833617-1",
+	"thumbor": {
+		"fileExtensions": [
+			"jpg",
+			"jpeg",
+			"png",
+			"webp"
+		]
+	}
 }

platform/lib/routers/growPages.js

@@ -24,6 +24,7 @@ const {Templates, createRequestContext} = require('@lib/templates/index.js');
 const AmpOptimizer = require('@ampproject/toolbox-optimizer');
 const CssTransformer = require('@lib/utils/cssTransformer');
 const pageCache = require('@lib/utils/pageCache');
+const imageOptimizer = require('@lib/utils/imageOptimizer');
 const HeadDedupTransformer = require('@lib/utils/HeadDedupTransformer');
 const signale = require('signale');
 const {promisify} = require('util');
@@ -178,6 +179,7 @@ const growPages = express.Router();
 const optimizer = AmpOptimizer.create({
   experimentPreloadHeroImage: true,
   preloadHeroImage: true,
+  imageOptimizer,
   transformations: [
     HeadDedupTransformer,
     ...AmpOptimizer.TRANSFORMATIONS_AMP_FIRST,

platform/lib/routers/static.js

@@ -22,10 +22,13 @@ const {join} = require('path');
 const config = require('@lib/config');
 const project = require('@lib/utils/project');
 const robots = require('./robots');
+const thumbor = require('./thumbor');
 
 // eslint-disable-next-line new-cap
 const staticRouter = express.Router();
 
+staticRouter.use(thumbor);
+
 staticRouter.use('/static', express.static(project.paths.STATICS_DEST));
 
 if (config.isProdMode()) {

platform/lib/routers/thumbor.js

@@ -0,0 +1,67 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+'use strict';
+
+const express = require('express');
+const {join} = require('path');
+const config = require('@lib/config');
+const HttpProxy = require('http-proxy');
+const log = require('@lib/utils/log')('Thumbor');
+
+const SECURITY_KEY = 'unsafe';
+
+const proxyOptions = {
+  target: config.hosts.thumbor.base,
+  changeOrigin: true,
+};
+const proxy = HttpProxy.createProxyServer(proxyOptions);
+
+// eslint-disable-next-line new-cap
+const thumborRouter = express.Router();
+
+const imagePaths = config.shared.thumbor.fileExtensions.map((extension) => {
+  return join('/static/', '/**/', `*.${extension}`);
+});
+
+thumborRouter.get(imagePaths, (request, response, next) => {
+  const imageUrl = new URL(request.url, config.hosts.platform.base);
+  const imageWidth = imageUrl.searchParams.get('width');
+  // Thumbor expects SECURITY_KEY as URL partial and the desired
+  // size (x * y) as another one
+  request.url = join(
+    SECURITY_KEY,
+    imageWidth ? `/${imageWidth}x0/` : '/',
+    imageUrl.pathname
+  );
+
+  proxy.web(request, response, proxyOptions, (error) => {
+    // Silently fail over if no thumbor instance can be reached. Therefore
+    // rewrite the URL back to the original one
+    if (e.code == 'ECONNREFUSED') {
+      request.url = imageUrl.href;
+      next();
+      return;
+    }
+
+    // Everything else is considered a error: malformed URLs,
+    // unavailable assets, ...
+    log.error(error);
+    response.status(502).end();
+  });
+});
+
+module.exports = thumborRouter;

platform/lib/utils/imageOptimizer.js

@@ -0,0 +1,33 @@
+/**
+ * Copyright 2020 The AMP HTML Authors. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS-IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+const config = require('@lib/config');
+
+/**
+ * Adds the desired image width to a URL as query paramter.
+ * This URL gets resolved to a thumbor compatible URL
+ * in platform/lib/routers/thumbor.js
+ *
+ * @param {String} src - the original img's src URL
+ * @param {Integer} width  - the target width
+ */
+function imageOptimizer(src, width) {
+  const imageUrl = new URL(src, config.hosts.platform.base);
+  imageUrl.searchParams.set('width', width);
+  return imageUrl.href;
+}
+
+module.exports = imageOptimizer;