Use brotli compression for KP assets

package.json

@@ -310,6 +310,7 @@
     "@percy/agent": "^0.26.0",
     "@testing-library/react": "^9.3.2",
     "@testing-library/react-hooks": "^3.2.1",
+    "@types/accept": "3.1.1",
     "@types/angular": "^1.6.56",
     "@types/angular-mocks": "^1.7.0",
     "@types/babel__core": "^7.1.2",

packages/kbn-optimizer/package.json

@@ -14,6 +14,7 @@
     "@kbn/babel-preset": "1.0.0",
     "@kbn/dev-utils": "1.0.0",
     "@kbn/ui-shared-deps": "1.0.0",
+    "@types/compression-webpack-plugin": "^2.0.1",
     "@types/estree": "^0.0.44",
     "@types/loader-utils": "^1.1.3",
     "@types/watchpack": "^1.1.5",
@@ -23,6 +24,7 @@
     "autoprefixer": "^9.7.4",
     "babel-loader": "^8.0.6",
     "clean-webpack-plugin": "^3.0.0",
+    "compression-webpack-plugin": "^3.1.0",
     "cpy": "^8.0.0",
     "css-loader": "^3.4.2",
     "del": "^5.1.0",

packages/kbn-optimizer/src/worker/webpack.config.ts

@@ -28,6 +28,7 @@ import TerserPlugin from 'terser-webpack-plugin';
 import webpackMerge from 'webpack-merge';
 // @ts-ignore
 import { CleanWebpackPlugin } from 'clean-webpack-plugin';
+import CompressionPlugin from 'compression-webpack-plugin';
 import * as UiSharedDeps from '@kbn/ui-shared-deps';
 
 import { Bundle, WorkerConfig, parseDirPath, DisallowedSyntaxPlugin } from '../common';
@@ -134,7 +135,24 @@ export function getWebpackConfig(bundle: Bundle, worker: WorkerConfig) {
       },
     ],
 
-    plugins: [new CleanWebpackPlugin(), new DisallowedSyntaxPlugin()],
+    plugins: [
+      new CleanWebpackPlugin(),
+      new DisallowedSyntaxPlugin(),
+      ...(worker.dist
+        ? [
+            new CompressionPlugin({
+              algorithm: 'brotliCompress',
+              filename: '[path].br',
+              test: /\.(js|css)$/,
+            }),
+            new CompressionPlugin({
+              algorithm: 'gzip',
+              filename: '[path].gz',
+              test: /\.(js|css)$/,
+            }),
+          ]
+        : []),
+    ],
 
     module: {
       // no parse rules for a few known large packages which have no require() statements

packages/kbn-ui-shared-deps/package.json

@@ -14,6 +14,7 @@
     "@kbn/i18n": "1.0.0",
     "abortcontroller-polyfill": "^1.4.0",
     "angular": "^1.7.9",
+    "compression-webpack-plugin": "^3.1.0",
     "core-js": "^3.6.4",
     "custom-event-polyfill": "^0.3.0",
     "elasticsearch-browser": "^16.7.0",

packages/kbn-ui-shared-deps/webpack.config.js

@@ -20,6 +20,7 @@
 const Path = require('path');
 
 const MiniCssExtractPlugin = require('mini-css-extract-plugin');
+const CompressionPlugin = require('compression-webpack-plugin');
 const { REPO_ROOT } = require('@kbn/dev-utils');
 const webpack = require('webpack');
 
@@ -117,5 +118,15 @@ exports.getWebpackConfig = ({ dev = false } = {}) => ({
     new webpack.DefinePlugin({
       'process.env.NODE_ENV': dev ? '"development"' : '"production"',
     }),
+    new CompressionPlugin({
+      algorithm: 'brotliCompress',
+      filename: '[path].br',
+      test: /\.(js|css)$/,
+    }),
+    new CompressionPlugin({
+      algorithm: 'gzip',
+      filename: '[path].gz',
+      test: /\.(js|css)$/,
+    }),
   ],
 });

renovate.json5

@@ -398,6 +398,8 @@
           '@types/good-squeeze',
           'inert',
           '@types/inert',
+          'accept',
+          '@types/accept',
         ],
       },
       {

src/dev/renovate/package_groups.ts

@@ -159,7 +159,17 @@ export const RENOVATE_PACKAGE_GROUPS: PackageGroup[] = [
   {
     name: 'hapi',
     packageWords: ['hapi'],
-    packageNames: ['hapi', 'joi', 'boom', 'hoek', 'h2o2', '@elastic/good', 'good-squeeze', 'inert'],
+    packageNames: [
+      'hapi',
+      'joi',
+      'boom',
+      'hoek',
+      'h2o2',
+      '@elastic/good',
+      'good-squeeze',
+      'inert',
+      'accept',
+    ],
   },
 
   {

src/optimize/bundles_route/dynamic_asset_response.ts

@@ -21,6 +21,7 @@ import { resolve } from 'path';
 import Fs from 'fs';
 import { promisify } from 'util';
 
+import Accept from 'accept';
 import Boom from 'boom';
 import Hapi from 'hapi';
 
@@ -32,6 +33,41 @@ const asyncOpen = promisify(Fs.open);
 const asyncClose = promisify(Fs.close);
 const asyncFstat = promisify(Fs.fstat);
 
+async function tryToOpenFile(filePath: string) {
+  try {
+    return await asyncOpen(filePath, 'r');
+  } catch (e) {
+    if (e.code === 'ENOENT') {
+      return undefined;
+    } else {
+      throw e;
+    }
+  }
+}
+
+async function selectCompressedFile(acceptEncodingHeader: string | undefined, path: string) {
+  let fd: number | undefined;
+  let fileEncoding: string | undefined;
+
+  const supportedEncodings = Accept.encodings(acceptEncodingHeader, ['br', 'gzip']);
+
+  if (supportedEncodings[0] === 'br') {
+    fileEncoding = 'br';
+    fd = await tryToOpenFile(`${path}.br`);
+  }
+  if (!fd && supportedEncodings.includes('gzip')) {
+    fileEncoding = 'gzip';
+    fd = await tryToOpenFile(`${path}.gz`);
+  }
+  if (!fd) {
+    fileEncoding = undefined;
+    // Use raw open to trigger exception if it does not exist
+    fd = await asyncOpen(path, 'r');
+  }
+
+  return { fd, fileEncoding };
+}
+
 /**
  *  Create a Hapi response for the requested path. This is designed
  *  to replicate a subset of the features provided by Hapi's Inert
@@ -67,6 +103,7 @@ export async function createDynamicAssetResponse({
   replacePublicPath: boolean;
 }) {
   let fd: number | undefined;
+  let fileEncoding: string | undefined;
 
   try {
     const path = resolve(bundlesPath, request.params.path);
@@ -79,7 +116,7 @@ export async function createDynamicAssetResponse({
     // we use and manage a file descriptor mostly because
     // that's what Inert does, and since we are accessing
     // the file 2 or 3 times per request it seems logical
-    fd = await asyncOpen(path, 'r');
+    ({ fd, fileEncoding } = await selectCompressedFile(request.headers['accept-encoding'], path));
 
     const stat = await asyncFstat(fd);
     const hash = await getFileHash(fileHashCache, path, stat, fd);
@@ -94,13 +131,21 @@ export async function createDynamicAssetResponse({
     const content = replacePublicPath ? replacePlaceholder(read, publicPath) : read;
     const etag = replacePublicPath ? `${hash}-${publicPath}` : hash;
 
-    return h
+    const response = h
       .response(content)
       .takeover()
       .code(200)
       .etag(etag)
       .header('cache-control', 'must-revalidate')
       .type(request.server.mime.path(path).type);
+
+    // If we manually selected a compressed file, specify the encoding header.
+    // Otherwise, let Hapi automatically gzip the response.
+    if (fileEncoding) {
+      return response.header('content-encoding', fileEncoding);
+    } else {
+      return response;
+    }
   } catch (error) {
     if (fd) {
       try {

test/functional/apps/bundles/index.js

@@ -0,0 +1,65 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/**
+ * These supertest-based tests live in the functional test suite because they depend on the optimizer bundles being built
+ * and served
+ */
+export default function({ getService }) {
+  const supertest = getService('supertest');
+
+  describe('bundle compression', () => {
+    it('returns gzip files when client only supports gzip', () =>
+      supertest
+        // We use the kbn-ui-shared-deps for these tests since they are always built with br compressed outputs,
+        // even in dev. Bundles built by @kbn/optimizer are only built with br compression in dist mode.
+        .get('/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js')
+        .set('Accept-Encoding', 'gzip')
+        .expect(200)
+        .expect('Content-Encoding', 'gzip'));
+
+    it('returns br files when client only supports br', () =>
+      supertest
+        .get('/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js')
+        .set('Accept-Encoding', 'br')
+        .expect(200)
+        .expect('Content-Encoding', 'br'));
+
+    it('returns br files when client only supports gzip and br', () =>
+      supertest
+        .get('/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js')
+        .set('Accept-Encoding', 'gzip, br')
+        .expect(200)
+        .expect('Content-Encoding', 'br'));
+
+    it('returns gzip files when client prefers gzip', () =>
+      supertest
+        .get('/bundles/kbn-ui-shared-deps/kbn-ui-shared-deps.js')
+        .set('Accept-Encoding', 'gzip;q=1.0, br;q=0.5')
+        .expect(200)
+        .expect('Content-Encoding', 'gzip'));
+
+    it('returns gzip files when no brotli version exists', () =>
+      supertest
+        .get('/bundles/commons.style.css') // legacy optimizer does not create brotli outputs
+        .set('Accept-Encoding', 'gzip, br')
+        .expect(200)
+        .expect('Content-Encoding', 'gzip'));
+  });
+}

test/functional/config.js

@@ -25,11 +25,12 @@ export default async function({ readConfigFile }) {
 
   return {
     testFiles: [
+      require.resolve('./apps/bundles'),
       require.resolve('./apps/console'),
-      require.resolve('./apps/getting_started'),
       require.resolve('./apps/context'),
       require.resolve('./apps/dashboard'),
       require.resolve('./apps/discover'),
+      require.resolve('./apps/getting_started'),
       require.resolve('./apps/home'),
       require.resolve('./apps/management'),
       require.resolve('./apps/saved_objects_management'),

test/functional/services/index.ts

@@ -51,6 +51,7 @@ import { ToastsProvider } from './toasts';
 import { PieChartProvider } from './visualizations';
 import { ListingTableProvider } from './listing_table';
 import { SavedQueryManagementComponentProvider } from './saved_query_management_component';
+import { KibanaSupertestProvider } from './supertest';
 
 export const services = {
   ...commonServiceProviders,
@@ -83,4 +84,5 @@ export const services = {
   toasts: ToastsProvider,
   savedQueryManagementComponent: SavedQueryManagementComponentProvider,
   elasticChart: ElasticChartProvider,
+  supertest: KibanaSupertestProvider,
 };

test/functional/services/supertest.ts

@@ -0,0 +1,29 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+import { FtrProviderContext } from 'test/functional/ftr_provider_context';
+import { format as formatUrl } from 'url';
+
+import supertestAsPromised from 'supertest-as-promised';
+
+export function KibanaSupertestProvider({ getService }: FtrProviderContext) {
+  const config = getService('config');
+  const kibanaServerUrl = formatUrl(config.get('servers.kibana'));
+  return supertestAsPromised(kibanaServerUrl);
+}

typings/accept.d.ts

@@ -0,0 +1,23 @@
+/*
+ * Licensed to Elasticsearch B.V. under one or more contributor
+ * license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright
+ * ownership. Elasticsearch B.V. licenses this file to you under
+ * the Apache License, Version 2.0 (the "License"); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *    http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+declare module 'accept' {
+  // @types/accept does not include the `preferences` argument so we override the type to include it
+  export function encodings(encodingHeader?: string, preferences?: string[]): string[];
+}