Harmony/merge master (#2314)

* fix #2211 and #2308, when exporting to multiple scopes, make sure to not export their dependencies when these dependencies themselves are not export pending (#2309)

* resolve #2268, prevent logger from holding the terminal once a command is completed by adding an error handler. Also, changed some "logger.debug" to "logger.silly". Also, support configuring the logger level by running "bit config set log_level <level>". Also, limit the number of concurrent files write to 100 (#2310)

* fix typo in the BitId README

* fix "bit status" and "bit tag" when new components require each other by module paths (#2313)

Co-authored-by: David First <david@bit.dev>

CHANGELOG.md

@@ -7,6 +7,14 @@ and this project adheres to [Semantic Versioning](http://semver.org/).
 
 ## [unreleased]
 
+- support configuring the logger level by running `bit config set log_level <level>`.
+- [#2268](https://github.com/teambit/bit/issues/2268) prevent logger from holding the terminal once a command is completed
+
+## [[14.7.5-dev.1] - 2020-02-06]
+
+- [#2211](https://github.com/teambit/bit/issues/2211) fix bit export to not export non-staged dependencies
+- [#2308](https://github.com/teambit/bit/issues/2308) fix "Cannot read property 'scope' of undefined" error on bit export
+
 ## [[14.7.4] - 2020-02-06](https://github.com/teambit/bit/releases/tag/v14.7.4)
 
 - [#2300](https://github.com/teambit/bit/issues/2300) improve `bit export` performance by pushing new tags only

CONTRIBUTING.md

@@ -79,6 +79,8 @@ In some cases, you might get very helpful info by prefixing Bit command with `BI
 
 To print the log messages on the console, prefix your command with `BIT_LOG=<debug-level>`, e.g. `BIT_LOG=error`.
 
+The log level written to the log file is by default "debug". To change it, run `bit config set log_level <level>`, e.g. `bit config set log_level silly`.
+
 ### Lint
 
 Run eslint and tsc (for type checking)

e2e/commands/export.e2e.1.ts

@@ -641,6 +641,39 @@ describe('bit export command', function() {
           expect(output).to.have.string(`${anotherRemote}/foo2`);
         });
       });
+      // fixes https://github.com/teambit/bit/issues/2308
+      // here, the component foo1 has a new dependency "bar", this dependency has been exported
+      // already, so we expect "bit export" to not attempt to export it.
+      describe('export with no ids, no remote and no flags when a dependency is from another collection', () => {
+        let output;
+        before(() => {
+          helper.scopeHelper.getClonedLocalScope(localScopeBefore);
+          helper.scopeHelper.getClonedRemoteScope(remoteScopeBefore);
+          helper.scopeHelper.getClonedScope(anotherRemoteScopeBefore, anotherRemotePath);
+          const { scopeName, scopePath } = helper.scopeHelper.getNewBareScope();
+          helper.scopeHelper.addRemoteScope(scopePath);
+          helper.scopeHelper.addRemoteScope(scopePath, helper.scopes.remotePath);
+          helper.fs.outputFile('bar.js', '');
+          helper.command.addComponent('bar.js');
+          helper.fs.outputFile('foo1.js', 'require("./bar");');
+          helper.command.tagAllComponents();
+          helper.command.exportComponent('bar', scopeName);
+          output = helper.command.runCmd('bit export');
+        });
+        it('should export successfully all ids, each to its own remote', () => {
+          const remoteList = helper.command.listRemoteScopeParsed();
+          expect(remoteList).to.have.lengthOf(1);
+          expect(remoteList[0].id).to.have.string('foo1');
+
+          const anotherRemoteListJson = helper.command.runCmd(`bit list ${anotherRemote} --json`);
+          const anotherRemoteList = JSON.parse(anotherRemoteListJson);
+          expect(anotherRemoteList).to.have.lengthOf(1);
+          expect(anotherRemoteList[0].id).to.have.string('foo2');
+        });
+        it('should not export the dependency that was not intended to be exported', () => {
+          expect(output).to.not.have.string('bar');
+        });
+      });
       describe('export with ids, no remote and the flag --last-scope', () => {
         let output;
         before(() => {

e2e/commands/link.e2e.1.ts

@@ -50,7 +50,7 @@ describe('bit link', function() {
           ).to.be.a.directory();
         });
       });
-      describe('when scopeDefault is overridden for this component', () => {
+      describe('when defaultScope is overridden for this component', () => {
         let linkOutput;
         before(() => {
           helper.scopeHelper.getClonedLocalScope(beforeLink);

e2e/functionalities/default-scope.e2e.2.ts

@@ -0,0 +1,51 @@
+import chai, { expect } from 'chai';
+import Helper from '../../src/e2e-helper/e2e-helper';
+
+chai.use(require('chai-fs'));
+
+describe('default scope functionality', function() {
+  this.timeout(0);
+  const helper = new Helper();
+  after(() => {
+    helper.scopeHelper.destroy();
+  });
+  describe('basic flow', () => {
+    helper.scopeHelper.setNewLocalAndRemoteScopes();
+    helper.fixtures.populateWorkspaceWithThreeComponentsAndModulePath();
+    helper.bitJson.addDefaultScope();
+    helper.command.runCmd('bit link');
+  });
+  it('bit status should not break', () => {
+    const status = helper.command.statusJson();
+    expect(status.newComponents).have.lengthOf(3);
+    expect(status.invalidComponents).have.lengthOf(0);
+  });
+  describe('tagging the components', () => {
+    let tagOutput;
+    before(() => {
+      tagOutput = helper.command.tagAllComponents();
+    });
+    it('should be able to to tag them successfully', () => {
+      expect(tagOutput).to.have.string('tagged');
+    });
+    it('bit status should not show any issue', () => {
+      const status = helper.command.statusJson();
+      expect(status.stagedComponents).have.lengthOf(3);
+      expect(status.newComponents).have.lengthOf(0);
+      expect(status.modifiedComponent).have.lengthOf(0);
+      expect(status.invalidComponents).have.lengthOf(0);
+    });
+    describe('exporting the components', () => {
+      before(() => {
+        helper.command.exportAllComponents();
+      });
+      it('should be able to export them all successfully', () => {
+        const status = helper.command.statusJson();
+        expect(status.stagedComponents).have.lengthOf(0);
+        expect(status.newComponents).have.lengthOf(0);
+        expect(status.modifiedComponent).have.lengthOf(0);
+        expect(status.invalidComponents).have.lengthOf(0);
+      });
+    });
+  });
+});

package.json

@@ -1,6 +1,6 @@
 {
   "name": "bit-bin",
-  "version": "14.7.4",
+  "version": "14.7.5-dev.1",
   "license": "Apache-2.0",
   "main": "./dist/api.js",
   "preferGlobal": true,
@@ -203,6 +203,7 @@
     "@babel/preset-env": "^7.7.7",
     "@babel/preset-typescript": "^7.7.7",
     "@babel/register": "^7.4.4",
+    "@types/bluebird": "^3.5.29",
     "@types/chai": "^4.2.5",
     "@types/chai-arrays": "^1.0.3",
     "@types/chai-fs": "^2.0.2",

src/api/consumer/lib/migrate.ts

@@ -15,12 +15,12 @@ export default (async function migrate(
   scopePath: string,
   verbose: boolean
 ): Promise<MigrationResult | null | undefined> {
-  logger.debug('migrate.migrate, starting migration process');
+  logger.silly('migrate.migrate, starting migration process');
   if (verbose) console.log('starting migration process'); // eslint-disable-line no-console
   let scope;
   // If a scope path provided we will run the migrate only for the scope
   if (scopePath) {
-    logger.debug(`migrate.migrate, running migration process for scope in path ${scopePath}`);
+    logger.silly(`migrate.migrate, running migration process for scope in path ${scopePath}`);
     if (verbose) console.log(`running migration process for scope in path ${scopePath}`); // eslint-disable-line no-console
     scope = await loadScope(scopePath);
     return scope.migrate(verbose);
@@ -34,7 +34,7 @@ export default (async function migrate(
   await consumer.migrate(verbose);
   // const consumerMigrationResult = await consumer.migrate(verbose);
   // if (!consumerMigrationResult)
-  logger.debug('migrate.migrate, running migration process for scope in consumer');
+  logger.silly('migrate.migrate, running migration process for scope in consumer');
   if (verbose) console.log('running migration process for scope in consumer'); // eslint-disable-line no-console
   return scope.migrate(verbose);
 });

src/app.ts

@@ -1,5 +1,5 @@
 import 'reflect-metadata';
-import * as BPromise from 'bluebird';
+import Bluebird from 'bluebird';
 import { Harmony } from './harmony';
 import HooksManager from './hooks';
 import { BitCliExt } from './extensions/cli';
@@ -12,7 +12,7 @@ process.env.MEMFS_DONT_WARN = 'true'; // suppress fs experimental warnings from
 
 // removing this, default to longStackTraces also when env is `development`, which impacts the
 // performance dramatically. (see http://bluebirdjs.com/docs/api/promise.longstacktraces.html)
-BPromise.config({
+Bluebird.config({
   longStackTraces: true
 });
 

src/bit-id/README.md

@@ -9,7 +9,8 @@ That's the preferable representation of BitId. Wherever possible, use this forma
 A string representation of BitId. `BitId.toString()` generates this string.
 For example: `my-scope/my-name@0.0.1`.
 When an ID is entered by the end user it's always a string.
-The problem with the string representation is that since the dynamic-namespace introduced, it's not clear from the string whether an ID has a scope or not. In the previous example, `my-scope/my-name` could be interpreted as a scopereadonly name or only a name.
+The problem with the string representation is that since the dynamic-namespace introduced, it's not clear from the string whether an ID has a scope or not.
+In the previous example, `my-scope/my-name` could be interpreted as an id with scope (`{ scope: 'my-scope', name: 'my-name' }`) or an id without a scope (`{ scope: null, name: 'my-scope/my-name' }`).
 See the next section how to safely parse the string.
 
 ## How to transform a Bit ID string to a BitId instance

src/constants.ts

@@ -208,6 +208,8 @@ export const CFG_GIT_EXECUTABLE_PATH = 'git_path';
 
 export const CFG_LOG_JSON_FORMAT = 'log_json_format';
 
+export const CFG_LOG_LEVEL = 'log_level';
+
 export const CFG_NO_WARNINGS = 'no_warnings';
 
 export const CFG_INTERACTIVE = 'interactive';
@@ -403,3 +405,5 @@ export const DEPENDENCIES_FIELDS = ['dependencies', 'devDependencies', 'peerDepe
 const MISSING_DEPS_SPACE_COUNT = 10;
 export const MISSING_DEPS_SPACE = ' '.repeat(MISSING_DEPS_SPACE_COUNT);
 export const MISSING_NESTED_DEPS_SPACE = ' '.repeat(MISSING_DEPS_SPACE_COUNT + 2);
+
+export const CONCURRENT_IO_LIMIT = 100; // limit number of files to read/write/delete/symlink at the same time

src/consumer/component/package-json-vinyl.ts

@@ -30,7 +30,6 @@ export default class PackageJsonVinyl extends AbstractVinyl {
     // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     logger.debug(`package-json-vinyl.write, path ${this.path}`);
     // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
-    // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     await fs.outputFile(this.path, this.contents);
     // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
     return this.path;

src/consumer/component/sources/data-to-persist.ts

@@ -1,11 +1,13 @@
 import * as path from 'path';
+import Bluebird from 'bluebird';
 import fs from 'fs-extra';
 import { ComponentCapsule } from '../../../extensions/capsule-ext';
 import AbstractVinyl from './abstract-vinyl';
 import Symlink from '../../../links/symlink';
 import logger from '../../../logger/logger';
 import RemovePath from './remove-path';
 import removeFilesAndEmptyDirsRecursively from '../../../utils/fs/remove-files-and-empty-dirs-recursively';
+import { CONCURRENT_IO_LIMIT as concurrency } from '../../../constants';
 
 export default class DataToPersist {
   files: AbstractVinyl[];
@@ -161,18 +163,18 @@ export default class DataToPersist {
     return dataToPersist;
   }
   async _persistFilesToFS() {
-    return Promise.all(this.files.map(file => file.write()));
+    return Bluebird.map(this.files, file => file.write(), { concurrency });
   }
   async _persistSymlinksToFS() {
-    return Promise.all(this.symlinks.map(symlink => symlink.write()));
+    return Bluebird.map(this.symlinks, symlink => symlink.write(), { concurrency });
   }
   async _deletePathsFromFS() {
     const pathWithRemoveItsDirIfEmptyEnabled = this.remove.filter(p => p.removeItsDirIfEmpty).map(p => p.path);
     const restPaths = this.remove.filter(p => !p.removeItsDirIfEmpty);
     if (pathWithRemoveItsDirIfEmptyEnabled.length) {
       await removeFilesAndEmptyDirsRecursively(pathWithRemoveItsDirIfEmptyEnabled);
     }
-    return Promise.all(restPaths.map(removePath => removePath.persistToFS()));
+    return Bluebird.map(restPaths, removePath => removePath.persistToFS(), { concurrency });
   }
   _validateAbsolute() {
     // it's important to make sure that all paths are absolute before writing them to the

src/consumer/consumer.ts

@@ -72,6 +72,7 @@ import { AutoTagResult } from '../scope/component-ops/auto-tag';
 import ShowDoctorError from '../error/show-doctor-error';
 import { EnvType } from '../legacy-extensions/env-extension-types';
 import loadFlattenedDependenciesForCapsule from './component-ops/load-flattened-dependencies';
+import { packageNameToComponentId } from '../utils/bit/package-name-to-component-id';
 
 type ConsumerProps = {
   projectPath: string;
@@ -240,7 +241,7 @@ export default class Consumer {
     const bitmapVersion = this.bitMap.version || '0.10.9';
 
     if (semver.gte(bitmapVersion, BIT_VERSION)) {
-      logger.debug('bit.map version is up to date');
+      logger.silly('bit.map version is up to date');
       return {
         run: false
       };
@@ -711,34 +712,19 @@ export default class Consumer {
 
   getComponentIdFromNodeModulesPath(requirePath: string, bindingPrefix: string): BitId {
     requirePath = pathNormalizeToLinux(requirePath);
-    // Temp fix to support old components before the migration has been running
-    bindingPrefix = bindingPrefix === 'bit' ? '@bit' : bindingPrefix;
-    const prefix = requirePath.includes('node_modules') ? `node_modules/${bindingPrefix}/` : `${bindingPrefix}/`;
-    const withoutPrefix = requirePath.substr(requirePath.indexOf(prefix) + prefix.length);
-    const componentName = withoutPrefix.includes('/')
-      ? withoutPrefix.substr(0, withoutPrefix.indexOf('/')) // the part after the first slash is the path inside the package
-      : withoutPrefix;
-    const pathSplit = componentName.split(NODE_PATH_COMPONENT_SEPARATOR);
-    if (pathSplit.length < 2) throw new GeneralError(`component has an invalid require statement: ${requirePath}`);
-    // since the dynamic namespaces feature introduced, the require statement doesn't have a fixed
-    // number of separators.
-    // also, a scope name may or may not include a dot. depends whether it's on bitHub or self hosted.
-    // we must check against BitMap to get the correct scope and name of the id.
-    if (pathSplit.length === 2) {
-      return new BitId({ scope: pathSplit[0], name: pathSplit[1] });
+    const prefix = requirePath.includes('node_modules') ? 'node_modules/' : '';
+    const withoutPrefix = prefix ? requirePath.substr(requirePath.indexOf(prefix) + prefix.length) : requirePath;
+
+    if (!withoutPrefix.includes('/')) {
+      throw new GeneralError(
+        'getComponentIdFromNodeModulesPath expects the path to have at least one slash for the scoped package, such as @bit/'
+      );
     }
-    const mightBeScope = R.head(pathSplit);
-    const mightBeName = R.tail(pathSplit).join('/');
-    const mightBeId = new BitId({ scope: mightBeScope, name: mightBeName });
-    const allBitIds = this.bitMap.getAllBitIds();
-    if (allBitIds.searchWithoutVersion(mightBeId)) return mightBeId;
-    // only bit hub has the concept of having the username in the scope name.
-    if (bindingPrefix !== 'bit' && bindingPrefix !== '@bit') return mightBeId;
-    // pathSplit has 3 or more items. the first two are the scope, the rest is the name.
-    // for example "user.scopeName.utils.is-string" => scope: user.scopeName, name: utils/is-string
-    const scope = pathSplit.splice(0, 2).join('.');
-    const name = pathSplit.join('/');
-    return new BitId({ scope, name });
+    const packageSplitBySlash = withoutPrefix.split('/');
+    // the part after the second slash is the path inside the package, just ignore it.
+    // (e.g. @bit/my-scope.my-name/internal-path.js).
+    const packageName = `${packageSplitBySlash[0]}/${packageSplitBySlash[1]}`;
+    return packageNameToComponentId(this, packageName, bindingPrefix);
   }
 
   composeRelativeComponentPath(bitId: BitId): string {

src/e2e-helper/e2e-bit-json-helper.ts

@@ -27,6 +27,9 @@ export default class BitJsonHelper {
     bitJson.overrides = overrides;
     this.write(bitJson);
   }
+  addDefaultScope(scope = this.scopes.remote) {
+    this.addKeyVal(undefined, 'defaultScope', scope);
+  }
   getEnvByType(bitJson: Record<string, any>, envType: 'compiler' | 'tester') {
     const basePath = ['env', envType];
     const env = R.path(basePath, bitJson);

src/e2e-helper/e2e-fixtures-helper.ts

@@ -89,6 +89,15 @@ export default class FixtureHelper {
     this.addComponentBarFoo();
   }
 
+  populateWorkspaceWithThreeComponentsAndModulePath() {
+    this.fs.createFile('utils', 'is-type.js', fixtures.isType);
+    this.addComponentUtilsIsType();
+    this.fs.createFile('utils', 'is-string.js', fixtures.isStringModulePath(this.scopes.remote));
+    this.addComponentUtilsIsString();
+    this.createComponentBarFoo(fixtures.barFooModulePath(this.scopes.remote));
+    this.addComponentBarFoo();
+  }
+
   /**
    * @deprecated use populateWorkspaceWithThreeComponents()
    */

src/jsdoc/jsdoc/jsdoc-parser.ts

@@ -25,7 +25,7 @@ export default async function parse(data: string, filePath?: PathOsBased): Promi
     docs.forEach(doc => extractDataRegex(doc, doclets, filePath));
   } catch (e) {
     // never mind, ignore the doc of this source
-    logger.debug(`failed parsing docs using on path ${filePath} with error`, e);
+    logger.silly(`failed parsing docs using on path ${filePath} with error`, e);
   }
   // @ts-ignore AUTO-ADDED-AFTER-MIGRATION-PLEASE-FIX!
   return doclets.filter(doclet => doclet.access === 'public');

src/jsdoc/react/react-parser.ts

@@ -110,7 +110,7 @@ export default async function parse(data: string, filePath?: PathOsBased): Promi
       return formatted;
     }
   } catch (err) {
-    logger.debug(`failed parsing docs using docgen on path ${filePath} with error`, err);
+    logger.silly(`failed parsing docs using docgen on path ${filePath} with error`, err);
   }
   return undefined;
 }