search command is working on a local scope

package.json

@@ -32,6 +32,7 @@
     "ramda": "^0.23.0",
     "requestify": "^0.2.5",
     "roadrunner": "^1.1.0",
+    "search-index": "^0.9.15",
     "semver": "^5.3.0",
     "sequest": "^0.9.0",
     "serialize-error": "^2.1.0",

src/cli/command-registrar-builder.js

@@ -3,6 +3,7 @@ import CommandRegistrar from './command-registrar';
 import { BIT_VERSION, BIT_USAGE, BIT_DESCRIPTION } from '../constants';
 import Init from './commands/public-cmds/init-cmd';
 import ScopeList from './commands/private-cmds/_list-cmd';
+import ScopeSearch from './commands/private-cmds/_search-cmd';
 import ScopeShow from './commands/private-cmds/_show-cmd';
 import Create from './commands/public-cmds/create-cmd';
 import Export from './commands/public-cmds/export-cmd';
@@ -44,6 +45,7 @@ export default function registerCommands(): CommandRegistrar {
     new Test(),
     new Put(),
     new ScopeList(),
+    new ScopeSearch(),
     new ScopeShow(),
     new Fetch(),
     new Build(),

src/cli/commands/private-cmds/_search-cmd.js

@@ -0,0 +1,20 @@
+/** @flow */
+import Command from '../../command';
+import { fromBase64 } from '../../../utils';
+import { searchAdapter } from '../../../search';
+
+export default class Search extends Command {
+  name = '_search <path> <query> <reindex>';
+  private = true;
+  description = 'search for bits on a remote scope';
+  alias = '';
+  opts = [];
+
+  action([path, query, reindex]: [string, string, string]): Promise<any> {
+    return searchAdapter.scopeSearch(fromBase64(path), fromBase64(query), fromBase64(reindex) === 'true');
+  }
+
+  report(searchResults: string): string {
+    return searchResults;
+  }
+}

src/cli/commands/public-cmds/search-cmd.js

@@ -1,19 +1,31 @@
 /** @flow */
+import chalk from 'chalk';
 import Command from '../../command';
+import { searchAdapter } from '../../../search';
 
 export default class Search extends Command {
-  name = 'search <query> [remote]';
-  description = 'search for bits in configured remote(s)';
+  name = 'search [query...]';
+  description = 'search for bits';
   alias = '';
-  opts = [];
+  opts = [
+    ['s', 'scope <scopename>', 'search in scope'],
+    ['r', 'reindex', 're-index all components']
+  ];
 
-  action(): Promise<any> {
-    const m = this.alias;
-    console.log('searching bit...');
-    return new Promise(resolve => resolve(m));
+  action([query, ]: [string[], ], { scope, reindex }) {
+    const queryStr = query.join(' ');
+    console.log(`searching bits in ${scope ? scope : 'local scope'} for "${queryStr}"`);
+    if (scope) {
+      return searchAdapter.searchRemotely(queryStr, scope, reindex);
+    }
+    return searchAdapter.searchLocally(queryStr, reindex);
   }
 
-  report(data: {string: any}): string {
-    return '';
+  report(searchResults: string): string {
+    const parsedResults = JSON.parse(searchResults);
+    if (!parsedResults.length) {
+      return chalk.red('No Results');
+    }
+    return chalk.green(parsedResults.join('\n'));
   }
 }

src/remotes/remote.js

@@ -40,13 +40,19 @@ export default class Remote {
       return network.describeScope();
     });
   }
-  
+
   list(): Promise<[]> {
     return this
       .connect()
       .then(network => network.list());
   }
 
+  search(query: string, reindex: boolean): Promise<any> {
+    return this
+      .connect()
+      .then(network => network.search(query, reindex));
+  }
+
   show(bitId: BitId): Promise<> {
     return this
       .connect()
@@ -78,6 +84,6 @@ export default class Remote {
     const primary = isPrimary(name);
     if (primary) name = cleanBang(name);
 
-    return new Remote(name, host, primary); 
+    return new Remote(name, host, primary);
   }
 }

src/scope/network/fs/fs.js

@@ -44,11 +44,15 @@ export default class Fs {
   fetchOnes(bitIds: BitIds): Promise<ComponentObjects[]> {
     return this.getScope().manyOneObjects(bitIds);
   }
-  
+
   list(): Promise<[]> {
     return this.getScope().list();
   }
 
+  search(): Promise<[]> {
+    throw new Error('not implemented yet');
+  }
+
   show(bitId: BitId): Promise<> {
     return this.getScope()
     .loadComponent(bitId);

src/scope/network/network.js

@@ -11,6 +11,7 @@ export interface Network {
   describeScope(): Promise<ScopeDescriptor>;
   fetch(bitIds: BitIds): Promise<ComponentObjects[]>;
   list(): Promise<ComponentObjects[]>;
+  search(query: string, reindex: boolean): Promise<string>;
   show(bitId: BitId): Promise<>;
   fetchOnes(bitIds: BitIds): Promise<ComponentObjects[]>;
 }

src/scope/network/ssh/ssh.js

@@ -77,7 +77,7 @@ export default class SSH {
         throw new RemoteScopeNotFound();
       });
   }
-  
+
   list() {
     return this.exec('_list')
     .then((str: string) => {
@@ -117,7 +117,7 @@ export default class SSH {
   composeConnectionUrl() {
     return `${this.username}@${this.host}:${this.port}`;
   }
- 
+
   connect(sshUrl: SSHUrl, key: ?string): Promise<SSH> {
     this.connection = sequest.connect(this.composeConnectionUrl(), {
       privateKey: keyGetter(key)

src/search/index.js

@@ -0,0 +1,10 @@
+/** @flow */
+import indexer from './indexer';
+import searcher from './searcher';
+import searchAdapter from './search-adapter';
+
+export {
+  indexer,
+  searcher,
+  searchAdapter
+};

src/search/indexer.js

@@ -0,0 +1,69 @@
+/** @flow */
+import path from 'path';
+import { Readable } from 'stream';
+import { parser } from '../jsdoc';
+import Component from '../consumer/component';
+import serverlessIndex from './serverless-index';
+
+let localIndex;
+
+function tokenizeStr(str: string) {
+  return str.trim().split(/(?=[A-Z])/).join(' ').toLowerCase().split(/ |_|-/).join(' ');
+}
+
+function prepareDoc(docs: Object, component: Component): Object {
+  const name = component.name;
+  const box = component.box;
+  const functionNames = docs.map(doc => doc.name).join(' ');
+  return {
+    id: `${box}_${name}`,
+    name,
+    tokenizedName: tokenizeStr(name),
+    box,
+    tokenizedBox: tokenizeStr(box),
+    functionNames,
+    tokenizedFunctionNames: tokenizeStr(functionNames),
+    description: docs.map(doc => doc.description).join(' ')
+  };
+}
+
+function addToLocalIndex(component: Component): Promise<any> {
+  return new Promise((resolve, reject) => {
+    const doc = prepareDoc(component.docs, component);
+    localIndex.then((indexInstance) => {
+      const docStream = new Readable({ objectMode: true });
+      docStream.push(doc);
+      docStream.push(null);
+      docStream
+        .pipe(indexInstance.defaultPipeline())
+        .pipe(indexInstance.add())
+        .on('data', (d) => {
+          // this function needs to be called if you want to listen for the end event
+        })
+        .on('end', () => {
+          resolve('The indexing has been completed');
+        });
+    });
+  });
+}
+
+function index(component: Component, scopePath: string) {
+  localIndex = serverlessIndex.initializeIndex(scopePath);
+  return addToLocalIndex(component);
+}
+
+function indexAll(path: string, components: Component[]): Promise<any> {
+  return new Promise((resolve, reject) => {
+    if (!components) return reject('The scope is empty');
+    serverlessIndex.deleteDb(path);
+    localIndex = serverlessIndex.initializeIndex(path);
+    const results = components.map(component => addToLocalIndex(component));
+    return resolve(Promise.all(results));
+  });
+}
+
+module.exports = {
+  index,
+  indexAll,
+  tokenizeStr
+};

src/search/search-adapter.js

@@ -0,0 +1,74 @@
+/** @flow */
+import serverlessIndex from './serverless-index';
+import indexer from './indexer';
+import searcher from './searcher';
+import { loadConsumer } from '../consumer';
+import { loadScope } from '../scope';
+
+function searchLocally(queryStr: string, reindex: boolean = false): Promise<any> {
+  return new Promise((resolve, reject) => {
+    let scopePath;
+    if (reindex) {
+      loadConsumer()
+        .then(consumer => {
+          scopePath = consumer.scope.path;
+          return consumer.scope.list();
+        })
+        .then((components) => {
+          return indexer.indexAll(scopePath, components);
+        })
+        .then(() => {
+          resolve(searcher.search(queryStr, scopePath));
+        });
+    }
+    else {
+      loadConsumer()
+        .then(consumer => {
+          scopePath = consumer.scope.path;
+          resolve(searcher.search(queryStr, scopePath));
+        });
+    }
+  });
+}
+
+function searchRemotely(queryStr: string, scope: string, reindex: boolean = false): Promise<any> {
+  return new Promise((resolve, reject) => {
+    loadConsumer()
+      .then(consumer => {
+        return consumer.scope.remotes()
+          .then(remotes =>
+            // $FlowFixMe
+            remotes.resolve(scope, consumer.scope.name)
+              .then(remote => {
+                resolve(remote.search(queryStr, reindex));
+              })
+          );
+      });
+  });
+}
+
+function scopeSearch(path: string, query: string, reindex: boolean): Promise<any> {
+  return new Promise((resolve, reject) => {
+    if (reindex) {
+      loadScope(path)
+        .then(scope => {
+          return scope.list();
+        })
+        .then((components) => {
+          return indexer.indexAll(path, components);
+        })
+        .then(() => {
+          resolve(searcher.search(query, path));
+        });
+    }
+    else {
+      resolve(searcher.search(query, path));
+    }
+  });
+}
+
+module.exports = {
+  searchLocally,
+  searchRemotely,
+  scopeSearch,
+};

src/search/searcher.js

@@ -0,0 +1,85 @@
+/** @flow */
+import serverlessIndex from './serverless-index';
+import indexer from './indexer';
+
+let localIndex;
+
+function totalHits(query: string) {
+  return new Promise((resolve, reject) => {
+    return localIndex.then((indexInstance) => {
+      indexInstance.totalHits({
+        query: buildQuery(query)
+      }, (err, count) => {
+        if (err) reject(err);
+        resolve(JSON.stringify(count));
+      });
+    });
+  });
+}
+
+function countDocs() {
+  return new Promise((resolve, reject) => {
+    return localIndex.then((indexInstance) => {
+      indexInstance.countDocs((err, info) => {
+        if (err) reject(err);
+        resolve(info);
+      });
+    });
+  });
+}
+
+function getDoc(docIds: string[]) {
+  return new Promise((resolve, reject) => {
+    return localIndex.then((indexInstance) => {
+      indexInstance.get(docIds).on('data', function (doc) {
+        console.log(doc);
+      });
+    });
+  });
+}
+
+function formatSearchResult(searchResult: Object): string {
+  const doc = searchResult.document;
+  return `> ${doc.box}/${doc.name}`;
+}
+
+function queryItem(field, query, boost = 1) {
+  return {
+    AND: { [field]: query.toLowerCase().split(' ') },
+    BOOST: boost
+  };
+}
+
+function buildQuery(queryStr: string) {
+  const tokenizedQuery = indexer.tokenizeStr(queryStr);
+  const query = [];
+  query.push(queryItem('box', queryStr, 4));
+  query.push(queryItem('tokenizedBox', queryStr, 3));
+  query.push(queryItem('name', queryStr, 4));
+  query.push(queryItem('tokenizedName', tokenizedQuery, 3));
+  query.push(queryItem('functionNames', queryStr, 3));
+  query.push(queryItem('tokenizedFunctionNames', tokenizedQuery, 2));
+  query.push(queryItem('description', queryStr));
+  return query;
+}
+
+function search(queryStr: string, path: string) {
+  return new Promise((resolve, reject) => {
+    localIndex = serverlessIndex.initializeIndex(path);
+    const searchResults = [];
+    const query = buildQuery(queryStr);
+    return localIndex.then((indexInstance) => {
+      indexInstance.search({
+        query,
+      }).on('data', function (data) {
+        searchResults.push(formatSearchResult(data));
+      }).on('end', function () {
+          return resolve(JSON.stringify(searchResults));
+        });
+    });
+  });
+}
+
+module.exports = {
+  search,
+};