From 4eb6c00f19522e2baf439cf2f2bf01ff22417302 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Thu, 22 Apr 2021 11:40:06 +0200 Subject: [PATCH 1/3] feat(monorepo-migration): Add script that imports compass deps into the monorepo --- .gitignore | 1 + package-lock.json | 139 ++++++ package.json | 2 + scripts/monorepo/compass-deps-list.js | 400 ++++++++++++++++++ scripts/monorepo/find-missing-repositories.js | 49 +++ scripts/monorepo/index.js | 358 ++++++++++++++++ 6 files changed, 949 insertions(+) create mode 100644 scripts/monorepo/compass-deps-list.js create mode 100644 scripts/monorepo/find-missing-repositories.js create mode 100644 scripts/monorepo/index.js diff --git a/.gitignore b/.gitignore index 5553ba1e12d..371b981995b 100644 --- a/.gitignore +++ b/.gitignore @@ -13,3 +13,4 @@ report.json expansions.yml .nvmrc .vscode +.migration-cache diff --git a/package-lock.json b/package-lock.json index cd73cbb49db..69c474d5802 100644 --- a/package-lock.json +++ b/package-lock.json @@ -3615,6 +3615,29 @@ "integrity": "sha512-jDctJ/IVQbZoJykoeHbhXpOlNBqGNcwXJKJog42E5HDPUwQTSdjCHdihjj0DlnheQ7blbT6dHOafNAiS8ooQKA==", "dev": true }, + "bl": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/bl/-/bl-4.1.0.tgz", + "integrity": "sha512-1W07cM9gS6DcLperZfFSj+bWLtaPGSOHWhPiGzXmvVJbRLdG82sH/Kn8EtW1VqWVA54AKf2h5k5BbnIbwF3h6w==", + "dev": true, + "requires": { + "buffer": "^5.5.0", + "inherits": "^2.0.4", + "readable-stream": "^3.4.0" + }, + "dependencies": { + "buffer": { + "version": "5.7.1", + "resolved": "https://registry.npmjs.org/buffer/-/buffer-5.7.1.tgz", + "integrity": "sha512-EHcyIPBQ4BSGlvjB16k5KgAJ27CIsHY/2JBmCRReo48y9rQ3MaUzWX3KVlBa4U7MyX02HdVj0K7C3WaB3ju7FQ==", + "dev": true, + "requires": { + "base64-js": "^1.3.1", + "ieee754": "^1.1.13" + } + } + } + }, "brace-expansion": { "version": "1.1.11", "resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-1.1.11.tgz", @@ -3945,6 +3968,12 @@ } } }, + "cli-spinners": { + "version": "2.6.0", + "resolved": "https://registry.npmjs.org/cli-spinners/-/cli-spinners-2.6.0.tgz", + "integrity": "sha512-t+4/y50K/+4xcCRosKkA7W4gTr1MySvLV0q+PxmG7FJ5g+66ChKurYjxBCjHggHH3HA5Hh9cy+lcUGWDqVH+4Q==", + "dev": true + }, "cli-truncate": { "version": "0.2.1", "resolved": "https://registry.npmjs.org/cli-truncate/-/cli-truncate-0.2.1.tgz", @@ -7822,6 +7851,12 @@ "is-extglob": "^2.1.1" } }, + "is-interactive": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/is-interactive/-/is-interactive-1.0.0.tgz", + "integrity": "sha512-2HvIEKRoqS62guEC+qBjpvRubdX910WCMuJTZ+I9yvqKU2/12eSL549HMwtabb4oupdj2sMP50k+XJfB/8JE6w==", + "dev": true + }, "is-lambda": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/is-lambda/-/is-lambda-1.0.1.tgz", @@ -7933,6 +7968,12 @@ "unc-path-regex": "^0.1.2" } }, + "is-unicode-supported": { + "version": "0.1.0", + "resolved": "https://registry.npmjs.org/is-unicode-supported/-/is-unicode-supported-0.1.0.tgz", + "integrity": "sha512-knxG2q4UC3u8stRGyAVJCOdxFmv5DZiRcdlIaAQXAbSfJya+OhopNotLQrstBhququ4ZpuKbDc/8S6mgXgPFPw==", + "dev": true + }, "is-url": { "version": "1.2.4", "resolved": "https://registry.npmjs.org/is-url/-/is-url-1.2.4.tgz", @@ -9840,6 +9881,84 @@ "word-wrap": "~1.2.3" } }, + "ora": { + "version": "5.4.0", + "resolved": "https://registry.npmjs.org/ora/-/ora-5.4.0.tgz", + "integrity": "sha512-1StwyXQGoU6gdjYkyVcqOLnVlbKj+6yPNNOxJVgpt9t4eksKjiriiHuxktLYkgllwk+D6MbC4ihH84L1udRXPg==", + "dev": true, + "requires": { + "bl": "^4.1.0", + "chalk": "^4.1.0", + "cli-cursor": "^3.1.0", + "cli-spinners": "^2.5.0", + "is-interactive": "^1.0.0", + "is-unicode-supported": "^0.1.0", + "log-symbols": "^4.1.0", + "strip-ansi": "^6.0.0", + "wcwidth": "^1.0.1" + }, + "dependencies": { + "ansi-regex": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-5.0.0.tgz", + "integrity": "sha512-bY6fj56OUQ0hU1KjFNDQuJFezqKdrAyFdIevADiqrWHwSlbmBNMHp5ak2f40Pm8JTFyM2mqxkG6ngkHO11f/lg==", + "dev": true + }, + "cli-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/cli-cursor/-/cli-cursor-3.1.0.tgz", + "integrity": "sha512-I/zHAwsKf9FqGoXM4WWRACob9+SNukZTd94DWF57E4toouRulbCxcUh6RKUEOQlYTHJnzkPMySvPNaaSLNfLZw==", + "dev": true, + "requires": { + "restore-cursor": "^3.1.0" + } + }, + "log-symbols": { + "version": "4.1.0", + "resolved": "https://registry.npmjs.org/log-symbols/-/log-symbols-4.1.0.tgz", + "integrity": "sha512-8XPvpAA8uyhfteu8pIvQxpJZ7SYYdpUivZpGy6sFsBuKRY/7rQGavedeB8aK+Zkyq6upMFVL/9AW6vOYzfRyLg==", + "dev": true, + "requires": { + "chalk": "^4.1.0", + "is-unicode-supported": "^0.1.0" + } + }, + "mimic-fn": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/mimic-fn/-/mimic-fn-2.1.0.tgz", + "integrity": "sha512-OqbOk5oEQeAZ8WXWydlu9HJjz9WVdEIvamMCcXmuqUYjTknH/sqsWvhQ3vgwKFRR1HpjvNBKQ37nbJgYzGqGcg==", + "dev": true + }, + "onetime": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/onetime/-/onetime-5.1.2.tgz", + "integrity": "sha512-kbpaSSGJTWdAY5KPVeMOKXSrPtr8C8C7wodJbcsd51jRnmD+GZu8Y0VoU6Dm5Z4vWr0Ig/1NKuWRKf7j5aaYSg==", + "dev": true, + "requires": { + "mimic-fn": "^2.1.0" + } + }, + "restore-cursor": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/restore-cursor/-/restore-cursor-3.1.0.tgz", + "integrity": "sha512-l+sSefzHpj5qimhFSE5a8nufZYAM3sBSVMAPtYkmC+4EH2anSGaEMXSD0izRQbu9nfyQ9y5JrVmp7E8oZrUjvA==", + "dev": true, + "requires": { + "onetime": "^5.1.0", + "signal-exit": "^3.0.2" + } + }, + "strip-ansi": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-6.0.0.tgz", + "integrity": "sha512-AuvKTrTfQNYNIctbR1K/YGTR1756GycPsg7b9bdV9Duqur4gv6aKqHXah67Z8ImS7WEz5QVcOtlfW2rZEugt6w==", + "dev": true, + "requires": { + "ansi-regex": "^5.0.0" + } + } + } + }, "os-homedir": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/os-homedir/-/os-homedir-1.0.2.tgz", @@ -10537,6 +10656,17 @@ "read-pkg": "^1.0.0" } }, + "readable-stream": { + "version": "3.6.0", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", + "integrity": "sha512-BViHy7LKeTz4oNnkcLJ+lVSL6vpiFeX6/d3oSH8zCW7UxP2onchk+vTGB143xuFjHS3deTgkKoXXymXqymiIdA==", + "dev": true, + "requires": { + "inherits": "^2.0.3", + "string_decoder": "^1.1.1", + "util-deprecate": "^1.0.1" + } + }, "readdir-scoped-modules": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/readdir-scoped-modules/-/readdir-scoped-modules-1.1.0.tgz", @@ -11043,6 +11173,15 @@ "es-abstract": "^1.17.5" } }, + "string_decoder": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", + "integrity": "sha512-hkRX8U1WjJFd8LsDJ2yQ/wWWxaopEsABU1XfkM8A+j0+85JAGppt16cr1Whg6KIbb4okU6Mql6BOj+uup/wKeA==", + "dev": true, + "requires": { + "safe-buffer": "~5.2.0" + } + }, "strip-ansi": { "version": "3.0.1", "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", diff --git a/package.json b/package.json index ab51e27e872..d98195b9e37 100644 --- a/package.json +++ b/package.json @@ -34,6 +34,7 @@ "cross-env": "^7.0.0", "execa": "^4.1.0", "fs-extra": "^9.0.1", + "glob": "^7.1.6", "lerna": "^4.0.0", "listr": "^0.14.3", "lodash": "^4.17.19", @@ -41,6 +42,7 @@ "mongodb-js-precommit": "^2.2.1", "nock": "^13.0.4", "node-notifier": "^8.0.0", + "ora": "^5.4.0", "semver": "^5.1.0", "sinon": "^8.1.1", "sinon-chai": "^3.4.0" diff --git a/scripts/monorepo/compass-deps-list.js b/scripts/monorepo/compass-deps-list.js new file mode 100644 index 00000000000..dc24aec74d9 --- /dev/null +++ b/scripts/monorepo/compass-deps-list.js @@ -0,0 +1,400 @@ +module.exports = [ + { + name: '@mongodb-js/compass-aggregations', + description: 'Compass Aggregation Pipeline Builder', + repository: 'https://github.com/mongodb-js/compass-aggregations.git' + }, + { + name: '@mongodb-js/compass-app-stores', + description: 'The external stores repo for compass', + repository: 'https://github.com/mongodb-js/compass-app-stores.git' + }, + { + name: '@mongodb-js/compass-auto-updates', + description: 'Compass Auto Updates Plugin', + repository: 'https://github.com/mongodb-js/compass-auto-updates.git' + }, + { + name: '@mongodb-js/compass-collection', + description: 'Compass Collection Plugin', + repository: 'https://github.com/mongodb-js/compass-collection.git' + }, + { + name: '@mongodb-js/compass-collection-stats', + description: 'Compass Collection Stats Plugin', + repository: 'https://github.com/mongodb-js/compass-collection-stats.git' + }, + { + name: '@mongodb-js/compass-collections-ddl', + description: 'Compass Collections DDL Plugin', + repository: 'https://github.com/mongodb-js/compass-collections-ddl.git' + }, + { + name: '@mongodb-js/compass-connect', + description: 'Connection Screen Plugin that supports Compass', + repository: 'https://github.com/mongodb-js/compass-connect.git' + }, + { + name: '@mongodb-js/compass-crud', + description: 'Compass Plugin for CRUD Operations', + repository: 'https://github.com/mongodb-js/compass-crud.git' + }, + { + name: '@mongodb-js/compass-database', + description: 'Compass Database Plugin', + repository: 'https://github.com/mongodb-js/compass-database.git' + }, + { + name: '@mongodb-js/compass-databases-ddl', + description: 'Compass Databases DDL Plugin', + repository: 'https://github.com/mongodb-js/compass-databases-ddl.git' + }, + { + name: '@mongodb-js/compass-deployment-awareness', + description: 'Compass Deployment Awareness Plugin', + repository: 'https://github.com/mongodb-js/compass-deployment-awareness.git' + }, + { + name: '@mongodb-js/compass-explain-plan', + description: 'Evaluate the performance of your quer', + repository: 'https://github.com/mongodb-js/compass-explain-plan.git' + }, + { + name: '@mongodb-js/compass-export-to-language', + description: 'export to language modal', + repository: 'https://github.com/mongodb-js/compass-export-to-language.git' + }, + { + name: '@mongodb-js/compass-field-store', + description: 'FieldStore keeps track of available fields in a collection.', + repository: 'https://github.com/mongodb-js/compass-field-store.git' + }, + { + name: '@mongodb-js/compass-find-in-page', + description: 'cmd-f UI for compass', + repository: 'https://github.com/mongodb-js/compass-find-in-page.git' + }, + { + name: '@mongodb-js/compass-home', + description: 'Home', + repository: 'https://github.com/mongodb-js/compass-home.git' + }, + { + name: '@mongodb-js/compass-import-export', + description: 'Compass Import/Export Plugin', + repository: 'https://github.com/mongodb-js/compass-import-export.git' + }, + { + name: '@mongodb-js/compass-indexes', + description: 'Indexes support for Compass', + repository: 'https://github.com/mongodb-js/compass-indexes.git' + }, + { + name: '@mongodb-js/compass-instance', + description: 'compass instance plugin', + repository: 'https://github.com/mongodb-js/compass-instance.git' + }, + { + name: '@mongodb-js/compass-license', + repository: 'https://github.com/mongodb-js/compass-license.git' + }, + { + name: '@mongodb-js/compass-loading', + description: 'Compass Loading Screen', + repository: 'https://github.com/mongodb-js/compass-loading.git' + }, + { + name: '@mongodb-js/compass-metrics', + description: 'Compass Metrics Plugin', + repository: 'https://github.com/mongodb-js/compass-metrics.git' + }, + { + name: '@mongodb-js/compass-plugin-info', + description: 'Compass Plugin Information Plugin', + repository: 'https://github.com/mongodb-js/compass-plugin-info.git' + }, + { + name: '@mongodb-js/compass-query-bar', + description: + 'Renders a component for executing MongoDB queries through a GUI.', + repository: 'https://github.com/mongodb-js/compass-query-bar.git' + }, + { + name: '@mongodb-js/compass-query-history', + description: 'The query history sidebar.', + repository: 'https://github.com/mongodb-js/compass-query-history.git' + }, + { + name: '@mongodb-js/compass-schema', + description: 'Compass Schema Tab Plugin', + repository: 'https://github.com/mongodb-js/compass-schema.git' + }, + { + name: '@mongodb-js/compass-schema-validation', + description: 'Compass plugin for document JSON schema validation', + repository: 'https://github.com/mongodb-js/compass-schema-validation.git' + }, + { + name: '@mongodb-js/compass-server-version', + description: 'Compass Server Version', + repository: 'https://github.com/mongodb-js/compass-server-version.git' + }, + { + name: '@mongodb-js/compass-serverstats', + description: 'Compass Real Time Server Stats Component.', + repository: 'https://github.com/mongodb-js/compass-serverstats.git' + }, + { + name: '@mongodb-js/compass-shell', + description: 'Compass Shell Plugin', + repository: 'https://github.com/mongodb-js/mongosh.git', + // Because we don't want to get the whole history of mongosh and there is no + // way to extract it in a better way + importStrategy: 'copy' + }, + { + name: '@mongodb-js/compass-sidebar', + description: 'Sidebar external plugin', + repository: 'https://github.com/mongodb-js/compass-sidebar.git' + }, + { + name: '@mongodb-js/compass-ssh-tunnel-status', + description: 'Compass SSH Tunnel Status', + repository: 'https://github.com/mongodb-js/compass-ssh-tunnel-status.git' + }, + { + name: '@mongodb-js/compass-status', + description: 'Compass Status Plugin', + repository: 'https://github.com/mongodb-js/compass-status.git' + }, + { + name: '@mongodb-js/dl-center', + description: 'mongodb download center internal tools', + repository: 'https://github.com/mongodb-js/dl-center.git' + }, + { + name: '@mongodb-js/electron-wix-msi', + description: 'Creates an MSI installer for your Electron app', + repository: 'https://github.com/mongodb-js/electron-wix-msi.git' + }, + { + name: '@mongodb-js/triejs', + description: 'Customizable trie data structure built in JavaScript.', + repository: 'https://github.com/mongodb-js/triejs.git' + }, + { + name: 'app-migrations', + description: 'Helper for application schema migrations.', + repository: 'https://github.com/mongodb-js/app-migrations.git' + }, + { + name: 'compass-preferences-model', + description: 'Compass preferences model.', + repository: 'https://github.com/mongodb-js/compass-preferences-model.git' + }, + { + name: 'compass-user-model', + description: 'MongoDB user model.', + repository: 'https://github.com/mongodb-js/compass-user-model.git' + }, + { + name: 'detect-coordinates', + description: 'Geo coordinates detection based of a mongodb-schema type.', + repository: 'https://github.com/mongodb-js/detect-coordinates.git' + }, + { + name: 'electron-license', + description: 'Tools for electron apps to work with licenses', + repository: 'https://github.com/mongodb-js/electron-license.git' + }, + { + name: 'electron-squirrel-startup', + description: + 'Default Squirrel.Windows event handler for your Electron apps.', + repository: 'https://github.com/mongodb-js/electron-squirrel-startup.git' + }, + { + name: 'hadron-app', + description: 'Hadron Application Singleton', + repository: 'https://github.com/mongodb-js/hadron-app.git' + }, + { + name: 'hadron-app-registry', + description: 'Hadron App Registry', + repository: 'https://github.com/mongodb-js/hadron-app-registry.git' + }, + { + name: 'hadron-auto-update-manager', + description: 'Atoms AutoUpdateManager class as a standalone module.', + repository: 'https://github.com/mongodb-js/hadron-auto-update-manager.git' + }, + { + name: 'hadron-build', + description: 'Tooling for Hadron apps.', + repository: 'https://github.com/mongodb-js/hadron-build.git' + }, + { + name: 'hadron-compile-cache', + description: 'Hadron Compile Cache', + repository: 'https://github.com/mongodb-js/hadron-compile-cache.git' + }, + { + name: 'hadron-document', + description: 'Hadron Document', + repository: 'https://github.com/mongodb-js/hadron-document.git' + }, + { + name: 'hadron-ipc', + description: 'Simplified IPC for electron apps.', + repository: 'https://github.com/mongodb-js/hadron-ipc.git' + }, + { + name: 'hadron-module-cache', + description: 'Hadron Module Cache', + repository: 'https://github.com/mongodb-js/module-cache.git' + }, + { + name: 'hadron-plugin-manager', + description: 'Hadron Plugin Manager', + repository: 'https://github.com/mongodb-js/hadron-plugin-manager.git' + }, + { + name: 'hadron-react-bson', + description: 'Hadron React BSON Components', + repository: 'https://github.com/mongodb-js/hadron-react.git' + }, + { + name: 'hadron-react-buttons', + description: 'Hadron React Button Components', + repository: 'https://github.com/mongodb-js/hadron-react.git' + }, + { + name: 'hadron-react-components', + description: 'Hadron React Components', + repository: 'https://github.com/mongodb-js/hadron-react.git' + }, + { + name: 'hadron-react-utils', + description: 'Hadron React Utils', + repository: 'https://github.com/mongodb-js/hadron-react.git' + }, + { + name: 'hadron-spectron', + description: 'Hadron Spectron Test Support', + repository: 'https://github.com/mongodb-js/hadron-spectron.git' + }, + { + name: 'hadron-style-manager', + description: 'Hadron Style Manager', + repository: 'https://github.com/mongodb-js/hadron-style-manager.git' + }, + { + name: 'hadron-type-checker', + description: 'Hadron Type Checker', + repository: 'https://github.com/mongodb-js/hadron-type-checker.git' + }, + { + name: 'mongodb-ace-autocompleter', + description: 'Ace Editor Autocompleter for MongoDB Queries & Agg Pipelines', + repository: 'https://github.com/mongodb-js/ace-autocompleter.git' + }, + { + name: 'mongodb-ace-mode', + description: 'MongoDB Mode for the ACE Editor', + repository: 'https://github.com/mongodb-js/ace-mode.git' + }, + { + name: 'mongodb-ace-theme', + description: 'MongoDB Theme for the ACE Editor', + repository: 'https://github.com/mongodb-js/ace-theme.git' + }, + { + name: 'mongodb-ace-theme-query', + description: 'MongoDB Theme for the ACE Editor in the Query Bar', + repository: 'https://github.com/mongodb-js/ace-theme-query.git' + }, + { + name: 'mongodb-collection-model', + description: 'MongoDB collection model.', + repository: 'https://github.com/mongodb-js/collection-model.git' + }, + { + name: 'mongodb-connection-fixture', + repository: 'https://github.com/mongodb-js/connection-fixture.git' + }, + { + name: 'mongodb-connection-model', + description: 'MongoDB connection model.', + repository: 'https://github.com/mongodb-js/connection-model.git' + }, + { + name: 'mongodb-data-service', + description: 'MongoDB Data Service', + repository: 'https://github.com/mongodb-js/data-service.git' + }, + { + name: 'mongodb-database-model', + description: 'MongoDB database model.', + repository: 'https://github.com/mongodb-js/database-model.git' + }, + { + name: 'mongodb-explain-compat', + repository: 'https://github.com/mongodb-js/mongodb-explain-compat.git' + }, + { + name: 'mongodb-explain-plan-model', + description: 'Ampersand model abstraction for MongoDB explain plans (3.0+)', + repository: 'https://github.com/mongodb-js/explain-plan-model.git' + }, + { + name: 'mongodb-index-model', + description: 'MongoDB index model.', + repository: 'https://github.com/mongodb-js/index-model.git' + }, + { + name: 'mongodb-instance-model', + description: 'MongoDB instance model.', + repository: 'https://github.com/mongodb-js/instance-model.git' + }, + { + name: 'mongodb-js-metrics', + description: 'Shareable metric recording.', + repository: 'https://github.com/mongodb-js/metrics.git' + }, + { + name: 'mongodb-language-model', + description: + 'Parses MongoDB query language and returns an abstract syntax tree', + repository: 'https://github.com/mongodb-js/mongodb-language-model.git' + }, + { + name: 'mongodb-notary-service-client', + description: 'A client for our notary-service: an API for codesigning.', + repository: 'https://github.com/mongodb-js/notary-service-client.git' + }, + { + name: 'mongodb-redux-common', + repository: 'https://github.com/mongodb-js/redux-common.git' + }, + { + name: 'mongodb-reflux-store', + repository: 'https://github.com/mongodb-js/reflux-store.git' + }, + { + name: 'mongodb-security', + description: 'Portable business logic of MongoDB security model', + repository: 'https://github.com/mongodb-js/security.git' + }, + { + name: 'reflux-state-mixin', + description: + "Reflux stores mixin adding 'state' syntax similar to React components. es6 classes supported", + repository: 'https://github.com/mongodb-js/reflux-state-mixin.git' + }, + { + name: 'storage-mixin', + description: + 'Ampersand model mixin to persist data via various storage backends', + repository: 'https://github.com/mongodb-js/storage-mixin.git' + } +]; diff --git a/scripts/monorepo/find-missing-repositories.js b/scripts/monorepo/find-missing-repositories.js new file mode 100644 index 00000000000..873baa45a7a --- /dev/null +++ b/scripts/monorepo/find-missing-repositories.js @@ -0,0 +1,49 @@ +/* eslint-disable no-console */ +const { request } = require('https'); + +const exists = (url) => { + return new Promise((resolve, reject) => { + try { + request(url, { method: 'HEAD' }, (res) => { + if (res.statusCode >= 200 && res.statusCode <= 399) { + resolve(res.headers.location || url); + } else { + const e = new Error(res.statusMessage); + e.code = res.statusCode; + reject(e); + } + }).end(); + } catch (e) { + reject(e); + } + }); +}; + +(async() => { + const compassDepsList = await Promise.all( + require('./compass-deps-list').map(async(pkg) => { + if (!pkg.repository) { + try { + // eslint-disable-next-line no-nested-ternary + const pkgPath = /^@mongodb-js/.test(pkg.name) + ? pkg.name.replace('@', '') + : /^mongodb-/.test(pkg.name) + ? pkg.name.replace('mongodb-', 'mongodb-js/') + : `mongodb-js/${pkg.name}`; + + const repo = (await exists(`https://github.com/${pkgPath}`)) + '.git'; + + pkg.repository = repo; + } catch (e) { + console.error( + `Failed to resolve package ${pkg.name}: ${e.code} ${e.message}` + ); + } + } + + return pkg; + }) + ); + + console.log(compassDepsList); +})(); diff --git a/scripts/monorepo/index.js b/scripts/monorepo/index.js new file mode 100644 index 00000000000..1493618b0e3 --- /dev/null +++ b/scripts/monorepo/index.js @@ -0,0 +1,358 @@ +/* eslint-disable no-console */ +const { promisify } = require('util'); +const { exec } = require('child_process'); +const fs = require('fs-extra'); +const path = require('path'); +const ora = require('ora'); +const glob = require('glob'); +const readline = require('readline'); + +async function questionAsync(query) { + return new Promise((resolve) => { + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + terminal: false + }); + + rl.question(`${query} `, (answer) => { + rl.close(); + resolve(answer); + }); + }); +} + +const globAsync = promisify(glob); + +const compassDepsList = require('./compass-deps-list'); + +const ROOT = path.resolve(__dirname, '..', '..'); + +const LERNA_BIN = path.join(ROOT, 'node_modules', '.bin', 'lerna'); + +const { IMPORT_STRATEGY, TARGET_DIR, CLONE_DIR } = process.env; + +let [ + , + , + // Directory we create the lerna monorepo in. + // Eventually we'll run this script with the main compass + // folder as target. For now putting things in one nested place + // helps development and teardown. + targetDir = TARGET_DIR || ROOT, + // The CLONE_DIR is where we clone repos before we import them in lerna. + // We use it to cache the clones while we develop as well. + cloneDir = CLONE_DIR || path.join(ROOT, '.migration-cache'), + // "lerna" or "copy" + defaultImportStrategy = IMPORT_STRATEGY || 'copy' +] = process.argv; + +targetDir = path.resolve(process.cwd(), targetDir); + +cloneDir = path.resolve(process.cwd(), cloneDir); + +const COMPASS = { + name: 'compass', + repository: 'https://github.com/mongodb-js/compass.git' +}; + +function sleep(ms) { + return new Promise((resolve) => setTimeout(resolve, ms)); +} + +const ONE_HOUR = 1000 * 60 * 60; + +async function runInDir(command, cwd = process.cwd(), timeout = ONE_HOUR) { + const execPromise = promisify(exec)(command, { + stdio: 'pipe', + cwd, + timeout + }); + return await execPromise; +} + +function lerna(command, dir) { + return runInDir(`${LERNA_BIN} ${command}`, dir); +} + +function git(command, dir) { + return runInDir(`git ${command}`, dir); +} + +async function withProgress(text, fn, ...args) { + spinner = ora(text).start(); + try { + const result = await fn.call(spinner, ...args); + spinner.succeed(); + return result; + } catch (e) { + if (spinner.isSpinning) { + spinner.fail(); + } + throw e; + } +} + +async function initializeMonorepo(cloneCacheDir, monorepoTargetDir) { + await fs.mkdirp(cloneCacheDir); + + // If path already exists check that user is sure that they want to continue + if ( + (await fs.pathExists(monorepoTargetDir)) && + (await fs.readdir(monorepoTargetDir)).length > 0 + ) { + let turnSpinnerBack = false; + + if (this && this.isSpinning) { + this.stop(); + turnSpinnerBack = true; + } + + const answer = await questionAsync( + `Directory ${prettyPath( + process.cwd(), + monorepoTargetDir + )} already exists and is populated. Are you sure you want to continue? [y/N]` + ); + + if (!/^(y|yes)$/i.test(answer)) { + throw new Error('Migration interrupted'); + } + + if (turnSpinnerBack) { + console.log(); + this.start(); + } + } else { + await fs.mkdirp(monorepoTargetDir); + } + + // If lerna config exists in the dir, lerna is already initialized + if (!(await fs.pathExists(path.join(monorepoTargetDir, 'lerna.json')))) { + // https://github.com/lerna/lerna/blob/main/commands/init + await lerna('init --independent', monorepoTargetDir); + } + + // Lerna needs git to be initialized for the import to happen correctly + try { + await git('rev-parse HEAD', monorepoTargetDir); + } catch (e) { + // If rev-parse failed, we know that there is not git repo here or first + // commit wasn't made yet + if (!(await fs.pathExists(path.join(monorepoTargetDir, '.git')))) { + await git('init', monorepoTargetDir); + } + await git('add -A', monorepoTargetDir); + await git('commit -m "Initial commit" --no-verify', monorepoTargetDir); + } +} + +async function importSinglePackage( + repoUrl, + importStrategy, + monorepoTargetDir, + cloneCacheDir +) { + const clonedDir = await cloneOrUpdate(repoUrl, cloneCacheDir); + const clonedRepoDirname = path.basename(clonedDir); + const destDir = path.join(monorepoTargetDir, 'packages', clonedRepoDirname); + + if (importStrategy === 'lerna') { + try { + // https://github.com/lerna/lerna/tree/main/commands/import + await lerna( + `import -y --flatten --preserve-commit ${clonedDir}`, + monorepoTargetDir + ); + } catch (e) { + // Lerna will exit with a code > 0 sometimes while actually succeding in + // importing a package. We can identify this based on error message + // (basically an stderr in this case) and continue the import + if (!/^lerna success import finished/m.test(e.message)) { + throw e; + } + } + } else if (importStrategy === 'copy') { + await fs.copy(clonedDir, destDir, { recursive: true, errorOnExist: true }); + await fs.remove(path.join(destDir, '.git'), { recursive: true }); + await git('add -A', monorepoTargetDir); + await runInDir( + `git commit -m "chore(monorepo): Import ${repoUrl} into main monorepo" --no-verify`, + monorepoTargetDir + ); + } + + await sleep(100); +} + +async function importCompassReposIntoLerna( + repositories, + monorepoTargetDir, + cloneCacheDir +) { + for (const { name, repository, importStrategy } of repositories) { + const repoPath = new URL(repository).pathname + .replace(/^\//, '') + .replace(/\.git$/, ''); + + const repoGitUrl = `git@github.com:${repoPath}.git`; + + const packagesToImport = Array.isArray(name) ? name.join(', ') : name; + + await withProgress( + `Importing ${packagesToImport} from ${repoGitUrl} using "${importStrategy}" strategy`, + importSinglePackage, + repoGitUrl, + importStrategy, + monorepoTargetDir, + cloneCacheDir + ); + } +} + +function prettyPath(from, to) { + return path.relative(from, to) || '.'; +} + +async function cloneOrUpdate(repoUrl, cloneCacheDir) { + const repoName = path.basename(repoUrl, '.git'); + const repoDir = path.join(cloneCacheDir, repoName); + + if (!(await fs.pathExists(repoDir))) { + await runInDir(`git clone ${repoUrl} ${repoDir}`); + await sleep(1000); + } else { + await git('pull', repoDir); + } + + return repoDir; +} + +async function unwrapNestedMonorepos(monorepoTargetDir, packagesToImport) { + const monoreposToProcess = ( + await globAsync('packages/*/lerna.json', { + cwd: monorepoTargetDir + }) + ).map((configPath) => path.resolve(monorepoTargetDir, configPath)); + + for (const configPath of monoreposToProcess) { + const monorepoRoot = path.dirname(configPath); + const monorepoPackageJson = path.join(monorepoRoot, 'package.json'); + const { name: monorepoName } = require(monorepoPackageJson); + + await withProgress( + `Unwrapping nested package(s) in monorepo ${monorepoName}`, + async () => { + const { packages } = require(configPath); + + if (!packages || packages.length === 0) { + throw new Error( + `Couldn't resolve packages in nested monorepo at ${monorepoRoot}` + ); + } + + const pattern = + packages.length > 1 ? `{${packages.join(',')}}` : packages[0]; + + const packagesToMove = await globAsync(pattern, { + cwd: monorepoRoot + }); + + let unwrapped = []; + + for (const packagePath of packagesToMove) { + const pkgPath = path.join(monorepoRoot, packagePath); + const pkgInfo = require(path.join(pkgPath, 'package.json')); + + if (packagesToImport.has(pkgInfo.name)) { + unwrapped.push(pkgInfo.name); + await fs.move( + pkgPath, + path.resolve(monorepoRoot, '..', path.basename(packagePath)), + { overwrite: false } + ); + } + } + + await fs.rmdir(monorepoRoot, { recursive: true }); + + unwrapped = unwrapped.join(', '); + + await git('add -A', monorepoTargetDir); + await runInDir( + `git commit -m "chore(monorepo): Unwrap packages ${unwrapped} in nested monorepo at ${prettyPath( + monorepoTargetDir, + monorepoRoot + )}" --no-verify`, + monorepoTargetDir + ); + } + ); + } +} + +async function runMonorepo() { + console.log(); + console.log('Migrating Compass to monorepo ...'); + console.log(); + + // 1. Setup lerna and the monorepo structure. + await withProgress( + `Initializing monorepo at ${prettyPath(process.cwd(), targetDir)}`, + initializeMonorepo, + cloneDir, + targetDir + ); + + // Some of the repositories are monorepos, we will import based on repository + // URL and then flatten them after we are done (that way we can import with + // git history preserved when possible) + const mergedRepoImportConfig = (() => { + // const allDeps = [COMPASS].concat(compassDepsList); + const allDeps = compassDepsList; + // .filter(({ name }) => /(compass-shell|hadron-react)/.test(name)); + const normalized = new Map(); + for (const { name, repository, importStrategy: strategy } of allDeps) { + if (normalized.has(repository)) { + const repo = normalized.get(repository); + repo.name.push(name); + // `copy` import strategy should be the main one, otherwise latest + // found one takes the priority + if (repo.importStrategy !== 'copy' && Boolean(strategy)) { + repo.importStrategy = strategy; + } + } else { + normalized.set(repository, { + name: [name], + repository, + importStrategy: strategy ?? defaultImportStrategy + }); + } + } + return Array.from(normalized.values()); + })(); + + // 2. Move the packages into lerna. + await importCompassReposIntoLerna( + mergedRepoImportConfig, + targetDir, + cloneDir + ); + + const packagesToImport = new Set( + mergedRepoImportConfig.map((pkg) => pkg.name).flat() + ); + + // 3. Move nested monorepos to the top-level packages/ folder + await unwrapNestedMonorepos(targetDir, packagesToImport); + + console.log(); + console.log('All done'); +} + +process.on('unhandledRejection', (err) => { + console.error(); + console.error(err.stack || err.message || err); +}); + +runMonorepo(); From fd579d4f7fc5e9bb41c12cedceb3d0297201fd10 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Thu, 22 Apr 2021 15:11:43 +0200 Subject: [PATCH 2/3] chore(migration): Remove dl-center and ace-autocompleter from the list --- scripts/monorepo/compass-deps-list.js | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/scripts/monorepo/compass-deps-list.js b/scripts/monorepo/compass-deps-list.js index dc24aec74d9..5926672b5fe 100644 --- a/scripts/monorepo/compass-deps-list.js +++ b/scripts/monorepo/compass-deps-list.js @@ -167,11 +167,6 @@ module.exports = [ description: 'Compass Status Plugin', repository: 'https://github.com/mongodb-js/compass-status.git' }, - { - name: '@mongodb-js/dl-center', - description: 'mongodb download center internal tools', - repository: 'https://github.com/mongodb-js/dl-center.git' - }, { name: '@mongodb-js/electron-wix-msi', description: 'Creates an MSI installer for your Electron app', @@ -293,11 +288,6 @@ module.exports = [ description: 'Hadron Type Checker', repository: 'https://github.com/mongodb-js/hadron-type-checker.git' }, - { - name: 'mongodb-ace-autocompleter', - description: 'Ace Editor Autocompleter for MongoDB Queries & Agg Pipelines', - repository: 'https://github.com/mongodb-js/ace-autocompleter.git' - }, { name: 'mongodb-ace-mode', description: 'MongoDB Mode for the ACE Editor', From 3246ad0fb6caf65593ca35b882557292f5ab6498 Mon Sep 17 00:00:00 2001 From: Sergey Petushkov Date: Mon, 26 Apr 2021 09:27:04 +0200 Subject: [PATCH 3/3] fix(monorepo-migration-script): Add missing script dependencies; Replace ?? with || --- package-lock.json | 28 ++++++++++++++++++++++++++++ package.json | 1 + scripts/monorepo/index.js | 2 +- 3 files changed, 30 insertions(+), 1 deletion(-) diff --git a/package-lock.json b/package-lock.json index 90a3225a7ee..1fb95e20e46 100644 --- a/package-lock.json +++ b/package-lock.json @@ -5345,6 +5345,18 @@ "mime-types": "^2.1.12" } }, + "fs-extra": { + "version": "9.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-9.1.0.tgz", + "integrity": "sha512-hcg3ZmepS30/7BSFqRvoo3DOMQu7IjqxO5nCDt+zM9XWjb33Wg7ziNT+Qvqbuc3+gWpzO02JubVyk2G4Zvo1OQ==", + "dev": true, + "requires": { + "at-least-node": "^1.0.0", + "graceful-fs": "^4.2.0", + "jsonfile": "^6.0.1", + "universalify": "^2.0.0" + } + }, "fs-minipass": { "version": "1.2.7", "resolved": "https://registry.npmjs.org/fs-minipass/-/fs-minipass-1.2.7.tgz", @@ -6814,6 +6826,16 @@ "integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus=", "dev": true }, + "jsonfile": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-6.1.0.tgz", + "integrity": "sha512-5dgndWOriYSm5cnYaJNhalLNDKOqFwyDB/rr1E9ZsGciGvKPs8R2xYGCacuf3z6K1YKDz182fd+fY3cn3pMqXQ==", + "dev": true, + "requires": { + "graceful-fs": "^4.1.6", + "universalify": "^2.0.0" + } + }, "jsonparse": { "version": "1.3.1", "resolved": "https://registry.npmjs.org/jsonparse/-/jsonparse-1.3.1.tgz", @@ -9644,6 +9666,12 @@ "integrity": "sha512-isyNax3wXoKaulPDZWHQqbmIx1k2tb9fb3GGDBRxCscfYV2Ch7WxPArBsFEG8s/safwXTT7H4QGhaIkTp9447w==", "dev": true }, + "universalify": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", + "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==", + "dev": true + }, "upath": { "version": "2.0.1", "resolved": "https://registry.npmjs.org/upath/-/upath-2.0.1.tgz", diff --git a/package.json b/package.json index 4361a6b29b2..675d9d0a05e 100644 --- a/package.json +++ b/package.json @@ -25,6 +25,7 @@ ], "dependencies": {}, "devDependencies": { + "fs-extra": "^9.1.0", "glob": "^7.1.6", "lerna": "^4.0.0", "mongodb-js-precommit": "^2.2.1", diff --git a/scripts/monorepo/index.js b/scripts/monorepo/index.js index 1493618b0e3..8d087d1434d 100644 --- a/scripts/monorepo/index.js +++ b/scripts/monorepo/index.js @@ -325,7 +325,7 @@ async function runMonorepo() { normalized.set(repository, { name: [name], repository, - importStrategy: strategy ?? defaultImportStrategy + importStrategy: strategy || defaultImportStrategy }); } }