Use puppeteer-core with custom chromium instead of puppeteer (opendis…

…tro-for-elasticsearch#222) * Use custom chrome binary * Update chromium path * Test workflow for al2 * Disable tests temporarily * Fix workflow Fix workflow * Rename opendistroKibanaReports to opendistroReportsKibana * Update workflows Fix workflow Update workflows Update workflows * Enable tests in workflows * Add windows chromium to workflows * Fix tests * Use path.posix.normalize to accomondate windows * Use @types/puppeteer-core
zhongnansu · Dec 7, 2020 · bdebaa3 · bdebaa3
1 parent 11346d1
commit bdebaa3
Show file tree

Hide file tree

Showing 11 changed files with 723 additions and 5,567 deletions.
diff --git a/.github/workflows/kibana-reports-release-workflow.yml b/.github/workflows/kibana-reports-release-workflow.yml
@@ -3,7 +3,7 @@ name: Release Kibana Reports Artifacts
 on:
   push:
     tags:
-      - "v*"
+      - 'v*'
 
 jobs:
   build:
@@ -30,33 +30,50 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: "10.22.0"
+          node-version: '10.22.0'
 
       - name: Move Kibana Reports to Plugins Dir
-        run: |
-          mkdir kibana/plugins
-          mv kibana-reports kibana/plugins
+        run: mv kibana-reports kibana/plugins/opendistroReportsKibana
 
       - name: Kibana Plugin Bootstrap
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: cd kibana/plugins/kibana-reports; yarn kbn bootstrap
+          command: cd kibana/plugins/opendistroReportsKibana; yarn kbn bootstrap
 
       - name: Test
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: cd kibana/plugins/kibana-reports; yarn test
+          command: cd kibana/plugins/opendistroReportsKibana; yarn test
 
       - name: Build Artifact
         run: |
-          cd kibana/plugins/kibana-reports
+          cd kibana/plugins/opendistroReportsKibana
           yarn build
-          artifact=`ls ./build/*.zip`
 
-          # TODO: rename S3 bucket path after infra team assigns one
-          aws s3 cp $artifact s3://kiabna-reports/kibana-reports-plugin/
+          cd build
+          mkdir -p ./{linux,windows}/kibana/opendistroReportsKibana
+          cp ./${{ env.PLUGIN_NAME }}-*.zip ./linux/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-linux-x64.zip
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ./windows/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.zip
+
+          cd linux
+          wget https://github.com/opendistro-for-elasticsearch/kibana-reports/releases/download/chromium-1.12.0.0/chromium-linux.zip
+          unzip chromium-linux.zip -d ./kibana/opendistroReportsKibana
+          zip -ur ./${{ env.PLUGIN_NAME }}-*.zip ./kibana
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ..
+          cd ..
+
+          cd windows
+          wget https://github.com/opendistro-for-elasticsearch/kibana-reports/releases/download/chromium-1.12.0.0/chromium-windows.zip
+          unzip chromium-windows.zip -d ./kibana/opendistroReportsKibana
+          zip -ur ./${{ env.PLUGIN_NAME }}-*.zip ./kibana
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ..
+          cd ..
+
+          artifact=`ls ./${{ env.PLUGIN_NAME }}-*.zip`
+
+          aws s3 cp $artifact s3://artifacts.opendistroforelasticsearch.amazon.com/downloads/kibana-plugins/opendistro-reports/
           aws cloudfront create-invalidation --distribution-id ${{ secrets.DISTRIBUTION_ID }} --paths "/downloads/*"
diff --git a/.github/workflows/kibana-reports-test-and-build-workflow.yml b/.github/workflows/kibana-reports-test-and-build-workflow.yml
@@ -19,34 +19,65 @@ jobs:
       - name: Setup Node
         uses: actions/setup-node@v1
         with:
-          node-version: "10.22.0"
+          node-version: '10.22.0'
 
       - name: Move Kibana Reports to Plugins Dir
+        run: mv kibana-reports kibana/plugins/opendistroReportsKibana
+
+      - name: Add Chromium Binary to Reporting for Testing
         run: |
-          mkdir kibana/plugins
-          mv kibana-reports kibana/plugins
+          sudo apt install -y libnss3-dev fonts-liberation libfontconfig1
+          cd kibana/plugins/opendistroReportsKibana
+          wget https://github.com/opendistro-for-elasticsearch/kibana-reports/releases/download/chromium-1.12.0.0/chromium-linux.zip
+          unzip chromium-linux.zip
+          rm chromium-linux.zip
 
       - name: Kibana Plugin Bootstrap
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: cd kibana/plugins/kibana-reports; yarn kbn bootstrap
+          command: cd kibana/plugins/opendistroReportsKibana; yarn kbn bootstrap
 
       - name: Test
         uses: nick-invision/retry@v1
         with:
           timeout_minutes: 30
           max_attempts: 3
-          command: cd kibana/plugins/kibana-reports; yarn test
+          command: cd kibana/plugins/opendistroReportsKibana; yarn test
 
       - name: Build Artifact
         run: |
-          cd kibana/plugins/kibana-reports
+          cd kibana/plugins/opendistroReportsKibana
           yarn build
 
-      - name: Upload Artifact
+          cd build
+          mkdir -p ./{linux,windows}/kibana/opendistroReportsKibana
+          cp ./${{ env.PLUGIN_NAME }}-*.zip ./linux/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-linux-x64.zip
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ./windows/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.zip
+
+          cd linux
+          wget https://github.com/opendistro-for-elasticsearch/kibana-reports/releases/download/chromium-1.12.0.0/chromium-linux.zip
+          unzip chromium-linux.zip -d ./kibana/opendistroReportsKibana
+          zip -ur ./${{ env.PLUGIN_NAME }}-*.zip ./kibana
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ..
+          cd ..
+
+          cd windows
+          wget https://github.com/opendistro-for-elasticsearch/kibana-reports/releases/download/chromium-1.12.0.0/chromium-windows.zip
+          unzip chromium-windows.zip -d ./kibana/opendistroReportsKibana
+          zip -ur ./${{ env.PLUGIN_NAME }}-*.zip ./kibana
+          mv ./${{ env.PLUGIN_NAME }}-*.zip ..
+          cd ..
+
+      - name: Upload Artifact For Linux
+        uses: actions/upload-artifact@v1
+        with:
+          name: kibana-reports-linux
+          path: kibana/plugins/opendistroReportsKibana/build/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-linux-x64.zip
+
+      - name: Upload Artifact For Windows
         uses: actions/upload-artifact@v1
         with:
-          name: kibana-reports
-          path: kibana/plugins/kibana-reports/build
+          name: kibana-reports-windows
+          path: kibana/plugins/opendistroReportsKibana/build/${{ env.PLUGIN_NAME }}-${{ env.OD_VERSION }}-windows-x64.zip
diff --git a/kibana-reports/common/index.ts b/kibana-reports/common/index.ts
@@ -1,4 +1,19 @@
-export const PLUGIN_ID = 'opendistroKibanaReports';
+/*
+ * Copyright 2020 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License").
+ * You may not use this file except in compliance with the License.
+ * A copy of the License is located at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * or in the "license" file accompanying this file. This file is distributed
+ * on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either
+ * express or implied. See the License for the specific language governing
+ * permissions and limitations under the License.
+ */
+
+export const PLUGIN_ID = 'opendistroReportsKibana';
 export const PLUGIN_NAME = 'opendistro_kibana_reports';
 
 export const API_PREFIX = '/api/reporting';

diff --git a/kibana-reports/kibana.json b/kibana-reports/kibana.json
@@ -1,5 +1,5 @@
 {
-  "id": "opendistroKibanaReports",
+  "id": "opendistroReportsKibana",
   "version": "1.10.1.0",
   "kibanaVersion": "7.9.1",
   "requiredPlugins": ["navigation", "data"],

diff --git a/kibana-reports/package.json b/kibana-reports/package.json
@@ -76,6 +76,7 @@
     "@types/puppeteer": "^3.0.2",
     "@types/reach__router": "^1.3.5",
     "@types/react": "^16.9.36",
+    "@types/react-addons-test-utils": "^0.14.25",
     "@types/react-dom": "^16.9.8",
     "@types/react-test-renderer": "^16.9.2",
     "@types/set-interval-async": "^1.0.0",

diff --git a/kibana-reports/server/routes/lib/createReport.ts b/kibana-reports/server/routes/lib/createReport.ts
@@ -31,7 +31,7 @@ import { createSavedSearchReport } from '../utils/savedSearchReportHelper';
 import { ReportSchemaType } from '../../model';
 import { CreateReportResultType } from '../utils/types';
 import { createVisualReport } from '../utils/visual_report/visualReportHelper';
-import { SetCookie } from 'puppeteer';
+import { SetCookie } from 'puppeteer-core';
 import { deliverReport } from './deliverReport';
 import { updateReportState } from './updateReportState';
 import { saveReport } from './saveReport';

diff --git a/kibana-reports/server/routes/utils/__tests__/visualReportHelper.test.ts b/kibana-reports/server/routes/utils/__tests__/visualReportHelper.test.ts
@@ -74,7 +74,9 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       queryUrl,
-      mockLogger
+      mockLogger,
+      undefined,
+      './.chromium/headless_shell'
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.png');
@@ -89,7 +91,9 @@ describe('test create visual report', () => {
     const { dataUrl, fileName } = await createVisualReport(
       reportParams as ReportParamsSchemaType,
       queryUrl,
-      mockLogger
+      mockLogger,
+      undefined,
+      './.chromium/headless_shell'
     );
     expect(fileName).toContain(`${reportParams.report_name}`);
     expect(fileName).toContain('.pdf');

diff --git a/kibana-reports/server/routes/utils/constants.ts b/kibana-reports/server/routes/utils/constants.ts
@@ -77,3 +77,6 @@ export const LOCAL_HOST = 'http://localhost:5601';
 export const DEFAULT_REPORT_HEADER = '<h1>Open Distro Kibana Reports</h1>';
 
 export const SECURITY_AUTH_COOKIE_NAME = 'security_authentication';
+
+export const CHROMIUM_PATH =
+  './plugins/opendistroReportsKibana/.chromium/headless_shell';
diff --git a/kibana-reports/server/routes/utils/visual_report/visualReportHelper.ts b/kibana-reports/server/routes/utils/visual_report/visualReportHelper.ts
@@ -13,7 +13,7 @@
  * permissions and limitations under the License.
  */
 
-import puppeteer, { ElementHandle, SetCookie } from 'puppeteer';
+import puppeteer, { ElementHandle, SetCookie } from 'puppeteer-core';
 import createDOMPurify from 'dompurify';
 import { JSDOM } from 'jsdom';
 import { Logger } from '../../../../../../src/core/server';
@@ -22,6 +22,7 @@ import {
   REPORT_TYPE,
   FORMAT,
   SELECTOR,
+  CHROMIUM_PATH,
 } from '../constants';
 import { getFileName } from '../helpers';
 import { CreateReportResultType } from '../types';
@@ -33,7 +34,8 @@ export const createVisualReport = async (
   reportParams: ReportParamsSchemaType,
   queryUrl: string,
   logger: Logger,
-  cookie?: SetCookie
+  cookie?: SetCookie,
+  chromiumPath = CHROMIUM_PATH
 ): Promise<CreateReportResultType> => {
   const {
     core_params,
@@ -66,6 +68,7 @@ export const createVisualReport = async (
      * https://github.com/puppeteer/puppeteer/blob/main/docs/troubleshooting.md#setting-up-chrome-linux-sandbox
      */
     args: ['--no-sandbox', '--disable-setuid-sandbox'],
+    executablePath: chromiumPath,
   });
   const page = await browser.newPage();
   page.setDefaultNavigationTimeout(0);

diff --git a/kibana-reports/server/utils/validationHelper.ts b/kibana-reports/server/utils/validationHelper.ts
@@ -16,7 +16,7 @@
 import path from 'path';
 
 export const isValidRelativeUrl = (relativeUrl: string) => {
-  const normalizedRelativeUrl = path.normalize(relativeUrl);
+  const normalizedRelativeUrl = path.posix.normalize(relativeUrl);
   // check pattern
   // ODFE pattern: /app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d?_g
   // AES pattern: /_plugin/kibana/app/dashboards#/view/7adfa750-4c81-11e8-b3d7-01146121b73d?_g