[ci] Add node scripts/yarn_deduplicate check (#169165)

- Adds an entrypoint, `node scripts/yarn_deduplicate` for fixing
duplicated packages via npm package `yarn-deduplicate`
- Fixes existing duplicated packages
- Adds a CI check verifying no duplicated packages have been added.  

There are currently two exclusions - all types packages which causes
other checks to break, and axe-core, which does not follow semver.

Closes #125712

.buildkite/scripts/steps/checks.sh

@@ -23,3 +23,4 @@ export DISABLE_BOOTSTRAP_VALIDATION=false
 .buildkite/scripts/steps/checks/ftr_configs.sh
 .buildkite/scripts/steps/checks/saved_objects_compat_changes.sh
 .buildkite/scripts/steps/code_generation/security_solution_codegen.sh
+.buildkite/scripts/steps/checks/yarn_deduplicate.sh

.buildkite/scripts/steps/checks/yarn_deduplicate.sh

@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source .buildkite/scripts/common/util.sh
+
+echo "--- Check yarn.lock for duplicated modules"
+node scripts/yarn_deduplicate
+
+check_for_changed_files 'node scripts/yarn_deduplicate && yarn kbn bootstrap' false

package.json

@@ -1617,6 +1617,7 @@
     "webpack-sources": "^1.4.1",
     "xml-crypto": "^3.0.1",
     "xmlbuilder": "13.0.2",
-    "yargs": "^15.4.1"
+    "yargs": "^15.4.1",
+    "yarn-deduplicate": "^6.0.2"
   }
 }

scripts/yarn_deduplicate.js

@@ -0,0 +1,10 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+require('../src/setup_node_env');
+require('../src/dev/yarn_deduplicate');

src/dev/yarn_deduplicate/index.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the Elastic License
+ * 2.0 and the Server Side Public License, v 1; you may not use this file except
+ * in compliance with, at your election, the Elastic License 2.0 or the Server
+ * Side Public License, v 1.
+ */
+
+import { resolve } from 'path';
+import { writeFileSync, readFileSync } from 'fs';
+import { fixDuplicates } from 'yarn-deduplicate';
+import { REPO_ROOT } from '@kbn/repo-info';
+
+const yarnLockFile = resolve(REPO_ROOT, 'yarn.lock');
+const yarnLock = readFileSync(yarnLockFile, 'utf-8');
+const output = fixDuplicates(yarnLock, {
+  useMostCommon: false,
+  excludeScopes: ['@types'],
+  excludePackages: ['axe-core', '@babel/types'],
+});
+
+writeFileSync(yarnLockFile, output);