Merge branch 'master' into harmony/main

teambit · Feb 6, 2020 · f3fc0b4 · f3fc0b4
2 parents 6eef5f0 + 642f6d4
commit f3fc0b4
Show file tree

Hide file tree

Showing 8 changed files with 120 additions and 36 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [unreleased]
 
+## [[14.7.4] - 2020-02-06](https://github.com/teambit/bit/releases/tag/v14.7.4)
+
+- [#2300](https://github.com/teambit/bit/issues/2300) improve `bit export` performance by pushing new tags only
+
 ## [[14.7.3] - 2020-02-02](https://github.com/teambit/bit/releases/tag/v14.7.3)
 
 ### New

diff --git a/e2e/commands/export.e2e.1.ts b/e2e/commands/export.e2e.1.ts
@@ -600,6 +600,7 @@ describe('bit export command', function() {
     describe('some components were exported to one scope and other to another scope', () => {
       let localScopeBefore;
       let remoteScopeBefore;
+      let anotherRemoteScopeBefore;
       let anotherRemote;
       let anotherRemotePath;
       before(() => {
@@ -618,6 +619,7 @@ describe('bit export command', function() {
         helper.command.tagScope('2.0.0');
         localScopeBefore = helper.scopeHelper.cloneLocalScope();
         remoteScopeBefore = helper.scopeHelper.cloneRemoteScope();
+        anotherRemoteScopeBefore = helper.scopeHelper.cloneScope(anotherRemotePath);
       });
       describe('export with no ids, no remote and no flags', () => {
         let output;
@@ -644,7 +646,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           output = helper.command.exportToCurrentScope('foo1 foo2');
         });
         it('should export successfully all ids, each to its own remote', () => {
@@ -667,7 +669,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           helper.fs.outputFile('foo1.js', "require('./foo2');");
           helper.command.tagScope('3.0.0');
           helper.scopeHelper.addRemoteScope(anotherRemotePath, helper.scopes.remotePath);
@@ -686,7 +688,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           helper.fs.outputFile('foo1.js', "require('./foo2');");
           helper.fs.outputFile('foo2.js', "require('./foo1');");
           helper.command.tagScope('3.0.0');
@@ -704,7 +706,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           helper.fs.outputFile('foo1.js', "require('./foo2');");
 
           helper.command.tagScope('3.0.0');
@@ -727,7 +729,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           helper.fs.outputFile('foo3.js', '');
           helper.command.addComponent('foo3.js');
           helper.command.tagAllComponents();
@@ -748,7 +750,7 @@ describe('bit export command', function() {
         before(() => {
           helper.scopeHelper.getClonedLocalScope(localScopeBefore);
           helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-          helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
           helper.fs.outputFile('foo3.js');
           helper.command.addComponent('foo3.js');
           helper.command.tagAllComponents();
@@ -769,7 +771,7 @@ describe('bit export command', function() {
           before(() => {
             helper.scopeHelper.getClonedLocalScope(beforeExportScope);
             helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
-            helper.scopeHelper.reInitRemoteScope(anotherRemotePath);
+            helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
             helper.bitJson.addKeyVal(undefined, 'defaultScope', helper.scopes.remote);
             output = helper.command.export();
           });
@@ -827,7 +829,7 @@ describe('bit export command', function() {
             helper.scopeHelper.reInitRemoteScope(forkScopePath);
             helper.fs.createFile('utils', 'is-string.js', ''); // remove the is-type dependency
             helper.command.tagAllComponents();
-            helper.command.exportAllComponents();
+            helper.command.export('--all-versions');
 
             helper.command.export(`${forkScope} utils/is-string --include-dependencies`);
             const forkScopeList = helper.command.listScopeParsed(forkScope);
@@ -852,7 +854,7 @@ describe('bit export command', function() {
             helper.fs.createFile('utils', 'is-string.js', ''); // remove the is-type dependency
             helper.fs.createFile('utils', 'is-type.js', ''); // add another version for is-type
             helper.command.tagAllComponents();
-            helper.command.exportAllComponents();
+            helper.command.export('--all-versions');
 
             helper.scopeHelper.reInitLocalScope();
             helper.scopeHelper.addRemoteScope();

diff --git a/package.json b/package.json
@@ -1,6 +1,6 @@
 {
   "name": "bit-bin",
-  "version": "14.7.3",
+  "version": "14.7.4",
   "license": "Apache-2.0",
   "main": "./dist/api.js",
   "preferGlobal": true,

diff --git a/src/api/consumer/lib/export.ts b/src/api/consumer/lib/export.ts
@@ -32,6 +32,7 @@ export default (async function exportAction(params: {
   eject: boolean;
   includeDependencies: boolean;
   setCurrentScope: boolean;
+  allVersions: boolean;
   includeNonStaged: boolean;
   codemod: boolean;
   force: boolean;
@@ -52,6 +53,7 @@ async function exportComponents({
   setCurrentScope,
   includeNonStaged,
   codemod,
+  allVersions,
   force
 }: {
   ids: string[];
@@ -60,6 +62,7 @@ async function exportComponents({
   setCurrentScope: boolean;
   includeNonStaged: boolean;
   codemod: boolean;
+  allVersions: boolean;
   force: boolean;
 }): Promise<{ updatedIds: BitId[]; nonExistOnBitMap: BitId[]; missingScope: BitId[]; exported: BitId[] }> {
   const consumer: Consumer = await loadConsumer();
@@ -79,6 +82,7 @@ async function exportComponents({
     includeDependencies,
     changeLocallyAlthoughRemoteIsDifferent: setCurrentScope,
     codemod,
+    allVersions,
     idsWithFutureScope
   });
   const { updatedIds, nonExistOnBitMap } = _updateIdsOnBitMap(consumer.bitMap, updatedLocally);

diff --git a/src/cli/commands/public-cmds/export-cmd.ts b/src/cli/commands/public-cmds/export-cmd.ts
@@ -36,6 +36,7 @@ export default class Export extends Command {
       'rewire',
       'EXPERIMENTAL. when exporting to a different scope, replace import/require statements in the source code to the new scope'
     ],
+    ['', 'all-versions', 'export not only staged versions but all of them'],
     ['f', 'force', 'force changing a component remote without asking for a confirmation']
   ];
   loader = true;
@@ -49,6 +50,7 @@ export default class Export extends Command {
       includeDependencies = false,
       setCurrentScope = false,
       all = false,
+      allVersions = false,
       force = false,
       rewire = false
     }: any
@@ -74,6 +76,7 @@ export default class Export extends Command {
       includeDependencies,
       setCurrentScope,
       includeNonStaged: all,
+      allVersions,
       codemod: rewire,
       force
     }).then(results => ({

diff --git a/src/e2e-helper/e2e-scope-helper.ts b/src/e2e-helper/e2e-scope-helper.ts
@@ -181,21 +181,25 @@ export default class ScopeHelper {
   }
 
   cloneRemoteScope() {
+    return this.cloneScope(this.scopes.remotePath);
+  }
+
+  cloneScope(scopePath: string) {
     const clonedScope = generateRandomStr();
     const clonedScopePath = path.join(this.scopes.e2eDir, clonedScope);
-    if (this.debugMode) console.log(`cloning a scope from ${this.scopes.remotePath} to ${clonedScopePath}`);
-    fs.copySync(this.scopes.remotePath, clonedScopePath);
+    if (this.debugMode) console.log(`cloning a scope from ${scopePath} to ${clonedScopePath}`);
+    fs.copySync(scopePath, clonedScopePath);
     this.clonedScopes.push(clonedScopePath);
     return clonedScopePath;
   }
 
-  getClonedRemoteScope(clonedScopePath: string, deleteCurrentScope = true) {
-    if (deleteCurrentScope) {
-      fs.removeSync(this.scopes.remotePath);
-    } else {
-      this.getNewBareScope();
-    }
-    if (this.debugMode) console.log(`cloning a scope from ${clonedScopePath} to ${this.scopes.remotePath}`);
-    fs.copySync(clonedScopePath, this.scopes.remotePath);
+  getClonedScope(clonedScopePath: string, scopePath: string) {
+    fs.removeSync(scopePath);
+    if (this.debugMode) console.log(`cloning a scope from ${clonedScopePath} to ${scopePath}`);
+    fs.copySync(clonedScopePath, scopePath);
+  }
+
+  getClonedRemoteScope(clonedScopePath: string) {
+    return this.getClonedScope(clonedScopePath, this.scopes.remotePath);
   }
 }
diff --git a/src/scope/component-ops/export-scope-components.ts b/src/scope/component-ops/export-scope-components.ts
@@ -59,6 +59,7 @@ export async function exportMany({
   includeDependencies = false, // kind of fork. by default dependencies only cached, with this, their scope-name is changed
   changeLocallyAlthoughRemoteIsDifferent = false, // by default only if remote stays the same the component is changed from staged to exported
   codemod = false,
+  allVersions,
   idsWithFutureScope
 }: {
   scope: Scope;
@@ -68,6 +69,7 @@ export async function exportMany({
   includeDependencies: boolean;
   changeLocallyAlthoughRemoteIsDifferent: boolean;
   codemod: boolean;
+  allVersions: boolean;
   idsWithFutureScope: BitIds;
 }): Promise<{ exported: BitIds; updatedLocally: BitIds }> {
   logger.debugAndAddBreadCrumb('scope.exportMany', 'ids: {ids}', { ids: ids.toString() });
@@ -100,9 +102,21 @@ export async function exportMany({
     const componentsAndObjects = [];
     const processComponentObjects = async (componentObject: ComponentObjects) => {
       const componentAndObject = componentObject.toObjects(scope.objects);
+      const localVersions = componentAndObject.component.getLocalVersions();
       componentAndObject.component.clearStateData();
-      await convertToCorrectScope(scope, componentAndObject, remoteNameStr, includeDependencies, bitIds, codemod);
-      await changePartialNamesToFullNamesInDists(scope, componentAndObject.component, componentAndObject.objects);
+      const didConvertScope = await convertToCorrectScope(
+        scope,
+        componentAndObject,
+        remoteNameStr,
+        includeDependencies,
+        bitIds,
+        codemod
+      );
+      const didChangeDists = await changePartialNamesToFullNamesInDists(
+        scope,
+        componentAndObject.component,
+        componentAndObject.objects
+      );
       const remoteObj = { url: remote.host, name: remote.name, date: Date.now().toString() };
       componentAndObject.component.addScopeListItem(remoteObj);
 
@@ -116,8 +130,18 @@ export async function exportMany({
         // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
         componentsAndObjects.push(componentAndObjectCloned);
       }
+
       const componentBuffer = await componentAndObject.component.compress();
-      const objectsBuffer = await Promise.all(componentAndObject.objects.map(obj => obj.compress()));
+      const getObjectsBuffer = () => {
+        if (allVersions || includeDependencies || didConvertScope || didChangeDists) {
+          // only when really needed (e.g. fork or version changes), collect all versions objects
+          return Promise.all(componentAndObject.objects.map(obj => obj.compress()));
+        }
+        // when possible prefer collecting only new/local versions. the server has already
+        // the rest, so no point of sending them.
+        return componentAndObject.component.collectVersionsObjects(scope.objects, localVersions);
+      };
+      const objectsBuffer = await getObjectsBuffer();
       return new ComponentObjects(componentBuffer, objectsBuffer);
     };
     // don't use Promise.all, otherwise, it'll throw "JavaScript heap out of memory" on a large set of data
@@ -287,16 +311,19 @@ async function convertToCorrectScope(
   fork: boolean,
   exportingIds: BitIds,
   codemod: boolean
-): Promise<void> {
+): Promise<boolean> {
   // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   const versionsObjects: Version[] = componentsObjects.objects.filter(object => object instanceof Version);
-  await Promise.all(
+  const haveVersionsChanged = await Promise.all(
     versionsObjects.map(async (objectVersion: Version) => {
       const hashBefore = objectVersion.hash().toString();
-      if (codemod) await _replaceSrcOfVersionIfNeeded(objectVersion);
-      changeDependencyScope(objectVersion);
+      const didCodeMod = codemod ? await _replaceSrcOfVersionIfNeeded(objectVersion) : false;
+      const didDependencyChange = changeDependencyScope(objectVersion);
       const hashAfter = objectVersion.hash().toString();
       if (hashBefore !== hashAfter) {
+        if (!didCodeMod && !didDependencyChange) {
+          throw new Error('hash should not be changed if there was not any dependency scope changes nor codemod');
+        }
         logger.debugAndAddBreadCrumb(
           'scope._convertToCorrectScope',
           `switching {id} version hash from ${hashBefore} to ${hashAfter}`,
@@ -309,18 +336,29 @@ async function convertToCorrectScope(
           }
         });
       }
+      return didCodeMod || didDependencyChange;
     })
   );
+  const hasComponentChanged = remoteScope !== componentsObjects.component.scope;
   componentsObjects.component.scope = remoteScope;
 
-  function changeDependencyScope(version: Version): void {
+  // return true if one of the versions has changed or the component itself
+  return haveVersionsChanged.some(x => x) || hasComponentChanged;
+
+  function changeDependencyScope(version: Version): boolean {
+    let hasChanged = false;
     version.getAllDependencies().forEach(dependency => {
-      dependency.id = getIdWithUpdatedScope(dependency.id);
+      const updatedScope = getIdWithUpdatedScope(dependency.id);
+      if (!updatedScope.isEqual(dependency.id)) {
+        hasChanged = true;
+        dependency.id = updatedScope;
+      }
     });
     version.flattenedDependencies = getBitIdsWithUpdatedScope(version.flattenedDependencies);
     version.flattenedDevDependencies = getBitIdsWithUpdatedScope(version.flattenedDevDependencies);
     version.flattenedCompilerDependencies = getBitIdsWithUpdatedScope(version.flattenedCompilerDependencies);
     version.flattenedTesterDependencies = getBitIdsWithUpdatedScope(version.flattenedTesterDependencies);
+    return hasChanged;
   }
 
   function getIdWithUpdatedScope(dependencyId: BitId): BitId {
@@ -342,18 +380,21 @@ async function convertToCorrectScope(
     const updatedIds = bitIds.map(id => getIdWithUpdatedScope(id));
     return BitIds.fromArray(updatedIds);
   }
-  async function _replaceSrcOfVersionIfNeeded(version: Version) {
+  async function _replaceSrcOfVersionIfNeeded(version: Version): Promise<boolean> {
+    let hasVersionChanged = false;
     const files = [...version.files, ...(version.dists || [])];
     await Promise.all(
       files.map(async file => {
         const newFileObject = await _createNewFileIfNeeded(version, file);
         if (newFileObject) {
           file.file = newFileObject.hash();
           componentsObjects.objects.push(newFileObject);
+          hasVersionChanged = true;
         }
         return null;
       })
     );
+    return hasVersionChanged;
   }
   async function _createNewFileIfNeeded(
     version: Version,
@@ -395,24 +436,29 @@ async function changePartialNamesToFullNamesInDists(
   scope: Scope,
   component: ModelComponent,
   objects: BitObject[]
-): Promise<void> {
+): Promise<boolean> {
   // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   const versions: Version[] = objects.filter(object => object instanceof Version);
-  await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));
+  const haveVersionsChanged = await Promise.all(versions.map(version => _replaceDistsOfVersionIfNeeded(version)));
+
+  return haveVersionsChanged.some(x => x);
 
-  async function _replaceDistsOfVersionIfNeeded(version: Version) {
+  async function _replaceDistsOfVersionIfNeeded(version: Version): Promise<boolean> {
     const dists = version.dists;
-    if (!dists) return;
-    await Promise.all(
+    if (!dists) return false;
+    const hasDistsChanged = await Promise.all(
       dists.map(async dist => {
         const newDistObject = await _createNewDistIfNeeded(version, dist);
         if (newDistObject) {
           dist.file = newDistObject.hash();
           objects.push(newDistObject);
+          return true;
         }
-        return null;
+        return false;
       })
     );
+    // return true if one of the dists has changed
+    return hasDistsChanged.some(x => x);
   }
 
   async function _createNewDistIfNeeded(