diff --git a/.dockerignore b/.dockerignore index abb40311d80..318be1bbcf4 100644 --- a/.dockerignore +++ b/.dockerignore @@ -13,6 +13,7 @@ MAINTENANCE.md README.md CONTRIBUTING.md packager/ +thumbor/ # The complete frontend and pages folders aren't needed on App Engine /frontend/ diff --git a/.gcloudignore b/.gcloudignore index b6d7adee292..64091189bde 100644 --- a/.gcloudignore +++ b/.gcloudignore @@ -24,6 +24,7 @@ MAINTENANCE.md README.md CONTRIBUTING.md packager/ +thumbor/ # The complete frontend and pages folders aren't needed on App Engine /frontend/ diff --git a/.gitignore b/.gitignore index 7aa5124400f..448c4c2e3b0 100644 --- a/.gitignore +++ b/.gitignore @@ -40,6 +40,7 @@ pages/shared/data/roadmap.yaml platform/pages* platform/static platform/config/build-info.yaml +thumbor/static # Ignore pipeline cache .cache/**/* diff --git a/.travis.yml b/.travis.yml index c99a7828dcd..2708d487544 100644 --- a/.travis.yml +++ b/.travis.yml @@ -103,4 +103,6 @@ jobs: - scripts/unbuffer.sh gulp buildFinalize - scripts/unbuffer.sh npm run smoke-test script: + - scripts/unbuffer.sh gulp thumborCollectImages + - gcloud app deploy thumbor/app.yaml --project=amp-dev-staging --quiet --version=1 - gcloud app deploy --project=amp-dev-staging --quiet --version=1 diff --git a/gulpfile.js/deploy.js b/gulpfile.js/deploy.js index fdae1870fd2..1c5ee6cc5d0 100644 --- a/gulpfile.js/deploy.js +++ b/gulpfile.js/deploy.js @@ -20,10 +20,11 @@ const {series} = require('gulp'); const {join} = require('path'); const {sh} = require('@lib/utils/sh.js'); const mri = require('mri'); -const {ROOT} = require('@lib/utils/project').paths; +const {ROOT, THUMBOR_ROOT} = require('@lib/utils/project').paths; const PREFIX = 'amp-dev'; const PACKAGER_PREFIX = PREFIX + '-packager'; +const THUMBOR_PREFIX = PREFIX + '-thumbor'; // Parse commandline arguments const argv = mri(process.argv.slice(2)); @@ -94,6 +95,28 @@ const config = { current: `gcr.io/${PROJECT_ID}/${PACKAGER_PREFIX}:${TAG}`, }, }, + thumbor: { + opts: { + workingDir: THUMBOR_ROOT, + }, + prefix: THUMBOR_PREFIX, + tag: TAG, + instance: { + groups: [ + { + name: `ig-${THUMBOR_PREFIX}`, + zone: 'us-east1-b', + }, + ], + template: `it-${THUMBOR_PREFIX}-${TAG}`, + count: 1, + machine: 'n1-standard-1', + }, + image: { + name: `gcr.io/${PROJECT_ID}/${THUMBOR_PREFIX}`, + current: `gcr.io/${PROJECT_ID}/${THUMBOR_PREFIX}:${TAG}`, + }, + }, }; /** @@ -293,6 +316,52 @@ async function packagerUpdateStart() { console.log('Rolling update started, this can take a few minutes...'); } +/* Thumbor */ +/** + * Create a new VM instance template based on the latest docker image. + */ +function thumborInstanceTemplateCreate() { + return sh( + `gcloud compute instance-templates create-with-container \ + ${config.thumbor.instance.template} \ + --container-image ${config.thumbor.image.current} \ + --machine-type ${config.thumbor.instance.machine}`, + config.thumbor.opts + ); +} + +/** + * Builds and uploads the thumbor docker image to Google Cloud Container Registry. + */ +function thumborImageUpload() { + return sh( + `gcloud builds submit --tag ${config.thumbor.image.current} .`, + config.thumbor.opts + ); +} + +/** + * Start a rolling update to a new thumbor VM instance template. This will ensure + * that there's always at least 1 active instance running during the update. + */ +async function thumborUpdateStart() { + const updates = config.thumbor.instance.groups.map((group) => { + return sh( + `gcloud beta compute instance-groups managed rolling-action \ + start-update ${group.name} \ + --version template=${config.thumbor.instance.template} \ + --zone=${group.zone} \ + --min-ready 1m \ + --max-surge 1 \ + --max-unavailable 1`, + config.thumbor.opts + ); + }); + await Promise.all(updates); + + console.log('Rolling update started, this can take a few minutes...'); +} + exports.verifyTag = verifyTag; exports.gcloudSetup = gcloudSetup; exports.deploy = series( @@ -316,6 +385,11 @@ exports.packagerDeploy = series( exports.packagerImageUpload = packagerImageUpload; exports.packagerInstanceTemplateCreate = packagerInstanceTemplateCreate; exports.packagerUpdateStart = packagerUpdateStart; + +exports.thumborImageUpload = thumborImageUpload; +exports.thumborInstanceTemplateCreate = thumborInstanceTemplateCreate; +exports.thumborUpdateStart = thumborUpdateStart; + exports.updateStop = updateStop; exports.updateStatus = updateStatus; exports.updateStart = updateStart; diff --git a/gulpfile.js/thumbor.js b/gulpfile.js/thumbor.js new file mode 100644 index 00000000000..c30a510a1de --- /dev/null +++ b/gulpfile.js/thumbor.js @@ -0,0 +1,51 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const gulp = require('gulp'); +const {join} = require('path'); + +const config = require('@lib/config'); +const {sh} = require('@lib/utils/sh.js'); +const {project} = require('@lib/utils'); + +const IMAGE_TAG = 'amp-dev-thumbor'; +const opts = { + workingDir: project.paths.THUMBOR_ROOT, +}; + +async function thumborRunLocal() { + await sh('pwd', opts); + await sh(`docker build -t ${IMAGE_TAG} .`, opts); + return await sh( + `docker run -p ${config.hosts.thumbor.port}:8080 ${IMAGE_TAG}`, + opts + ); +} + +async function thumborCollectImages() { + const imagePaths = config.shared.thumbor.fileExtensions.map((extension) => { + return join(project.paths.STATICS_DEST, '/**/', `*.${extension}`); + }); + + return gulp + .src(imagePaths) + .pipe(gulp.dest(`${project.paths.THUMBOR_STATICS_DEST}`)); +} + +exports.thumborRunLocal = thumborRunLocal; +exports.thumborCollectImages = thumborCollectImages; diff --git a/package-lock.json b/package-lock.json index f8cf4a0b4ac..6006b2b9fca 100644 --- a/package-lock.json +++ b/package-lock.json @@ -40,9 +40,9 @@ } }, "@ampproject/toolbox-optimizer": { - "version": "2.4.0", - "resolved": "https://registry.npmjs.org/@ampproject/toolbox-optimizer/-/toolbox-optimizer-2.4.0.tgz", - "integrity": "sha512-Bmb+eMF9/VB3H0qPdZy0V5yPSkWe5RwuGbXiMxzqYdJgmMat+NL75EtozQnlpa0uBlESnOGe7bMojm/SA1ImrA==", + "version": "2.5.0-alpha.0", + "resolved": "https://registry.npmjs.org/@ampproject/toolbox-optimizer/-/toolbox-optimizer-2.5.0-alpha.0.tgz", + "integrity": "sha512-pl6Np/uqzOazfxHUN3YmHQxMAMvrnefOqZyZf4dJ4Nvxejvzha9ZJfp2HtQ8wXXLEFymN0BZ/Gy6VzVD2LH8YQ==", "requires": { "@ampproject/toolbox-core": "^2.4.0-alpha.1", "@ampproject/toolbox-runtime-version": "^2.4.0-alpha.1", diff --git a/package.json b/package.json index bd7e796e653..2cc896d51a3 100644 --- a/package.json +++ b/package.json @@ -57,7 +57,7 @@ ], "dependencies": { "@ampproject/toolbox-cors": "2.4.0-alpha.1", - "@ampproject/toolbox-optimizer": "2.4.0", + "@ampproject/toolbox-optimizer": "2.5.0-alpha.0", "@google-cloud/datastore": "5.1.0", "acorn": "7.2.0", "casual": "1.6.2", diff --git a/platform/config/environments/development.json b/platform/config/environments/development.json index f9d1b1d4e25..88f575a0dc8 100644 --- a/platform/config/environments/development.json +++ b/platform/config/environments/development.json @@ -44,6 +44,11 @@ "scheme": "https", "host": "amp-dev-sxg.appspot.com", "port": "" + }, + "thumbor": { + "scheme": "http", + "host": "localhost", + "port": "8088" } } } diff --git a/platform/config/environments/local.json b/platform/config/environments/local.json index aa9774fc926..d299126b056 100644 --- a/platform/config/environments/local.json +++ b/platform/config/environments/local.json @@ -44,6 +44,11 @@ "scheme": "https", "host": "amp-dev-sxg.appspot.com", "port": "" + }, + "thumbor": { + "scheme": "http", + "host": "localhost", + "port": "8088" } } } diff --git a/platform/config/environments/staging.json b/platform/config/environments/staging.json index b0fa20381f3..29b683a11d0 100644 --- a/platform/config/environments/staging.json +++ b/platform/config/environments/staging.json @@ -44,6 +44,11 @@ "scheme": "https", "host": "amp-dev-sxg.appspot.com", "port": "" + }, + "thumbor": { + "scheme": "https", + "host": "thumbor-dot-amp-dev-staging.appspot.com", + "port": "" } }, "redis": { diff --git a/platform/config/shared.json b/platform/config/shared.json index 6c46e9881d9..e7bc6e22664 100644 --- a/platform/config/shared.json +++ b/platform/config/shared.json @@ -3,5 +3,13 @@ "playground": "/#url=", "repository": "https://github.com/ampproject/docs/blob/future/" }, - "gaTrackingId": "UA-67833617-1" + "gaTrackingId": "UA-67833617-1", + "thumbor": { + "fileExtensions": [ + "jpg", + "jpeg", + "png", + "webp" + ] + } } diff --git a/platform/lib/routers/growPages.js b/platform/lib/routers/growPages.js index a6bfd5fa8a6..2698ad3a7dd 100644 --- a/platform/lib/routers/growPages.js +++ b/platform/lib/routers/growPages.js @@ -24,6 +24,7 @@ const {Templates, createRequestContext} = require('@lib/templates/index.js'); const AmpOptimizer = require('@ampproject/toolbox-optimizer'); const CssTransformer = require('@lib/utils/cssTransformer'); const pageCache = require('@lib/utils/pageCache'); +const imageOptimizer = require('@lib/utils/imageOptimizer'); const HeadDedupTransformer = require('@lib/utils/HeadDedupTransformer'); const signale = require('signale'); const {promisify} = require('util'); @@ -178,6 +179,7 @@ const growPages = express.Router(); const optimizer = AmpOptimizer.create({ experimentPreloadHeroImage: true, preloadHeroImage: true, + imageOptimizer, transformations: [ HeadDedupTransformer, ...AmpOptimizer.TRANSFORMATIONS_AMP_FIRST, diff --git a/platform/lib/routers/static.js b/platform/lib/routers/static.js index 70953f4fc36..23bbcbabcec 100644 --- a/platform/lib/routers/static.js +++ b/platform/lib/routers/static.js @@ -22,10 +22,13 @@ const {join} = require('path'); const config = require('@lib/config'); const project = require('@lib/utils/project'); const robots = require('./robots'); +const thumbor = require('./thumbor'); // eslint-disable-next-line new-cap const staticRouter = express.Router(); +staticRouter.use(thumbor); + staticRouter.use('/static', express.static(project.paths.STATICS_DEST)); if (config.isProdMode()) { diff --git a/platform/lib/routers/thumbor.js b/platform/lib/routers/thumbor.js new file mode 100644 index 00000000000..8b72499f96c --- /dev/null +++ b/platform/lib/routers/thumbor.js @@ -0,0 +1,67 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +'use strict'; + +const express = require('express'); +const {join} = require('path'); +const config = require('@lib/config'); +const HttpProxy = require('http-proxy'); +const log = require('@lib/utils/log')('Thumbor'); + +const SECURITY_KEY = 'unsafe'; + +const proxyOptions = { + target: config.hosts.thumbor.base, + changeOrigin: true, +}; +const proxy = HttpProxy.createProxyServer(proxyOptions); + +// eslint-disable-next-line new-cap +const thumborRouter = express.Router(); + +const imagePaths = config.shared.thumbor.fileExtensions.map((extension) => { + return join('/static/', '/**/', `*.${extension}`); +}); + +thumborRouter.get(imagePaths, (request, response, next) => { + const imageUrl = new URL(request.url, config.hosts.platform.base); + const imageWidth = imageUrl.searchParams.get('width'); + // Thumbor expects SECURITY_KEY as URL partial and the desired + // size (x * y) as another one + request.url = join( + SECURITY_KEY, + imageWidth ? `/${imageWidth}x0/` : '/', + imageUrl.pathname + ); + + proxy.web(request, response, proxyOptions, (error) => { + // Silently fail over if no thumbor instance can be reached. Therefore + // rewrite the URL back to the original one + if (e.code == 'ECONNREFUSED') { + request.url = imageUrl.href; + next(); + return; + } + + // Everything else is considered a error: malformed URLs, + // unavailable assets, ... + log.error(error); + response.status(502).end(); + }); +}); + +module.exports = thumborRouter; diff --git a/platform/lib/utils/imageOptimizer.js b/platform/lib/utils/imageOptimizer.js new file mode 100644 index 00000000000..ff08299975a --- /dev/null +++ b/platform/lib/utils/imageOptimizer.js @@ -0,0 +1,33 @@ +/** + * Copyright 2020 The AMP HTML Authors. All Rights Reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS-IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +const config = require('@lib/config'); + +/** + * Adds the desired image width to a URL as query paramter. + * This URL gets resolved to a thumbor compatible URL + * in platform/lib/routers/thumbor.js + * + * @param {String} src - the original img's src URL + * @param {Integer} width - the target width + */ +function imageOptimizer(src, width) { + const imageUrl = new URL(src, config.hosts.platform.base); + imageUrl.searchParams.set('width', width); + return imageUrl.href; +} + +module.exports = imageOptimizer; diff --git a/platform/lib/utils/project.js b/platform/lib/utils/project.js index 9a7b57403d8..0eb612fccea 100644 --- a/platform/lib/utils/project.js +++ b/platform/lib/utils/project.js @@ -64,6 +64,8 @@ const paths = { GUIDES_PATH_RELATIVE: '/content/amp-dev/documentation/guides-and-tutorials/', RECENT_GUIDES_DEST: 'pages/shared/data/recent-guides.yaml', BUILD_INFO_PATH: absolute('platform/config/build-info.yaml'), + THUMBOR_ROOT: absolute('thumbor'), + THUMBOR_STATICS_DEST: absolute('thumbor/static'), }; module.exports = { diff --git a/thumbor/Dockerfile b/thumbor/Dockerfile new file mode 100644 index 00000000000..ebf26823aae --- /dev/null +++ b/thumbor/Dockerfile @@ -0,0 +1,19 @@ +FROM minimalcompact/thumbor + +ENV LOG_LEVEL "DEBUG" +ENV THUMBOR_PORT "8080" + +ENV AUTO_WEBP "True" + +ENV LOADER "thumbor.loaders.file_loader" +ENV FILE_LOADER_ROOT_PATH "/app/" + +COPY static /app/static + +# ENV LOADER = 'thumbor_cloud_storage.loaders.cloud_storage_loader' +# ENV CLOUD_STORAGE_PROJECT_ID = 'amp-dev-staging' +# ENV CLOUD_STORAGE_BUCKET_ID = 'amp-dev-staging-thumbor-storage' +# +# ENV RESULT_STORAGE = 'thumbor_cloud_storage.result_storages.cloud_storage' +# ENV RESULT_STORAGE_CLOUD_STORAGE_PROJECT_ID = 'amp-dev-staging' +# ENV RESULT_STORAGE_CLOUD_STORAGE_BUCKET_ID = 'amp-dev-staging-thumbor-storage' diff --git a/thumbor/app.yaml b/thumbor/app.yaml new file mode 100644 index 00000000000..e75b4b23589 --- /dev/null +++ b/thumbor/app.yaml @@ -0,0 +1,16 @@ +runtime: custom +env: flex +service: thumbor +manual_scaling: + instances: 1 + +liveness_check: + path: /healthcheck +readiness_check: + app_start_timeout_sec: 500 + path: /healthcheck + +resources: + cpu: 1 + memory_gb: 2 + disk_size_gb: 10