INT-203 arrays

README.md

@@ -1,17 +1,19 @@
-# mongodb-schema
-
-[![build status](https://secure.travis-ci.org/mongodb-js/mongodb-schema.png)](http://travis-ci.org/mongodb-js/mongodb-schema)
-[![Coverage Status](https://coveralls.io/repos/mongodb-js/mongodb-schema/badge.svg)](https://coveralls.io/r/mongodb-js/mongodb-schema)
-[![Gitter](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/mongodb-js/mongodb-js?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
+# mongodb-schema [![][npm_img]][npm_url] [![][travis_img]][travis_url] [![][coverage_img]][coverage_url] [![][gitter_img]][gitter_url]
 
 Infer a probabilistic schema for a MongoDB collection.
 
+A high-level view of the class interactions is as follows:
+
+![](./docs/mongodb-schema_diagram.png)
+
 ## Example
 
-`mongodb-schema` doesn't do anything directly with `mongodb` so to try the examples we'll install the node.js driver.  As well, we'll need some data
-in a collection to derive the schema of:
+`mongodb-schema` doesn't do anything directly with `mongodb` so to try the examples we'll install the node.js driver.  
+As well, we'll need some data in a collection to derive the schema of.
+
+Make sure you have a `mongod` running on localhost on port 27017 (or change the example accordingly). Then, do:
 
-1. `npm install mongodb mongodb-schema`.
+1. `npm install mongodb mongodb-schema`
 2. `mongo --eval "db.test.insert([{_id: 1, a: true}, {_id: 2, a: 'true'}, {_id: 3, a: 1}, {_id: 4}])" localhost:27017/test`
 3. Create a new file `parse-schema.js` and paste in the following code:
   ```javascript
@@ -30,63 +32,92 @@ in a collection to derive the schema of:
   });
   ```
 4. When we run the above with `node parse-schema.js`, we'll see something
-  like the following:
+  like the following (some fields not present here for clarity):
 
   ```javascript
   {
-    ns: 'test.test',
-    count: 4, // The number of documents sampled
-    fields: [ // A collection of Field objects @see lib/field.js
+    "count": 4,                   // parsed 4 documents
+    "ns": "test.test",            // namespace
+    "fields": [                   // an array of Field objects, @see `./lib/field.js`
       {
-        name: "_id",
-        probability: 1, // Just as we expected, all 4 documents had `_id`
-        unique: 4, // All 4 values for `_id` were unique
-        types: [
+        "name": "_id",
+        "count": 4,               // 4 documents counted with _id
+        "type": "Number",         // the type of _id is `Number`
+        "probability": 1,         // all documents had an _id field
+        "unique": 4,              // 4 unique values found
+        "has_duplicates": false,  // therefore no duplicates
+        "types": [                // an array of Type objects, @see `./lib/types/`
           {
-            name: "Number", // The only type seen was a Number
-            probability: 1,
-            unique: 4
+            "name": "Number",     // name of the type
+            "count": 4,           // 4 numbers counted
+            "probability": 1,
+            "unique": 4,
+            "values": [           // array of encountered values
+              1,
+              2,
+              3,
+              4
+            ]
           }
         ]
       },
       {
-        name: "a", // Unlike `_id`, `a` was present in only 3 of 4 documents
-        probability: 0.75,
-        unique: 3, // Of the 3 values seen, all 3 were unique
-        // As expected, Boolean, String, and Number values were seen.
-        // A handy instance of `Undefined` is also provided to represent missing data",
+        "name": "a",
+        "count": 3,               // only 3 documents with field `a` counted
+        "probability": 0.75,      // hence probability 0.75
+        "type": [                 // found these types
+          "Boolean",
+          "String",
+          "Number",
+          "Undefined"             // for convenience, we treat Undefined as its own type
+        ],
+        "unique": 3,
+        "has_duplicates": false,   // there were no duplicate values
         "types": [
           {
-            name: "Boolean",
-            probability: 0.25,
-            unique: 1
+            "name": "Boolean",
+            "count": 1,
+            "probability": 0.25,  // probabilities for types are calculated factoring in Undefined
+            "unique": 1,
+            "values": [
+              true
+            ]
           },
           {
-            name: "String",
-            probability: 0.25,
-            unique: 1
+            "name": "String",
+            "count": 1,
+            "probability": 0.25,
+            "unique": 1,
+            "values": [
+              "true"
+            ]
           },
           {
-            name: "Number",
-            probability: 0.25,
-            unique: 1
+            "name": "Number",
+            "count": 1,
+            "probability": 0.25,
+            "unique": 1,
+            "values": [
+              1
+            ]
           },
           {
-            name: "Undefined",
-            probability: 0.25
+            "name": "Undefined",
+            "count": 1,
+            "probability": 0.25,
+            "unique": 0
           }
         ]
       }
     ]
   }
-  ```
+```
 
 ### More Examples
 
 `mongodb-schema` supports all [BSON types][bson-types].
 Checkout [the tests][tests] for more usage examples.
 
-
 ## Installation
 
 ```
@@ -108,9 +139,6 @@ Apache 2.0
 Under the hood, `mongodb-schema` uses [ampersand-state][ampersand-state] and
 [ampersand-collection][ampersand-collection] for modeling [Schema][schema], [Field][field]'s, and [Type][type]'s.
 
-A high-level view of the class interactions is as follows:
-
-![](./docs/mongodb-schema_diagram.png)
 
 
 
@@ -121,3 +149,12 @@ A high-level view of the class interactions is as follows:
 [schema]: https://github.com/mongodb-js/mongodb-language-model/blob/master/lib/schema.js
 [field]: https://github.com/mongodb-js/mongodb-language-model/blob/master/lib/field.js
 [type]: https://github.com/mongodb-js/mongodb-language-model/blob/master/lib/type.js
+
+[travis_img]: https://secure.travis-ci.org/mongodb-js/mongodb-schema.svg?branch=master
+[travis_url]: https://travis-ci.org/mongodb-js/mongodb-schema
+[npm_img]: https://img.shields.io/npm/v/mongodb-schema.svg
+[npm_url]: https://www.npmjs.org/package/mongodb-schema
+[coverage_img]: https://coveralls.io/repos/mongodb-js/mongodb-schema/badge.svg
+[coverage_url]: https://coveralls.io/r/mongodb-js/mongodb-schema
+[gitter_img]: https://badges.gitter.im/Join%20Chat.svg
+[gitter_url]: https://gitter.im/mongodb-js/mongodb-js

examples/parse-schema.js

@@ -4,7 +4,7 @@ var connect = require('mongodb');
 connect('mongodb://localhost:27017/test', function(err, db){
   if(err) return console.error(err);
 
-  parseSchema('test', db.collection('test').find(), function(err, schema){
+  parseSchema('test.test', db.collection('test').find(), function(err, schema){
     if(err) return console.error(err);
 
     console.log(JSON.stringify(schema, null, 2));

lib/collection-parent-mixin.js

@@ -0,0 +1,14 @@
+var _ = require('lodash');
+var AmpersandCollection = require('ampersand-collection');
+
+/**
+ * Collection with this mixin passes its parent down to its values
+ */
+module.exports = {
+  set: function (models, options) {
+    options = _.defaults({
+      parent: this.parent
+    }, options || {});
+    return AmpersandCollection.prototype.set.call(this, models, options);
+  }
+};

lib/field-collection.js

@@ -1,11 +1,15 @@
 var Collection = require('ampersand-collection');
 var lodashMixin = require('ampersand-collection-lodash-mixin');
+var parentMixin = require('./collection-parent-mixin');
+var Field = require('./field');
 
 /**
  * Container for a list of Fields.
  */
-var FieldCollection = Collection.extend(lodashMixin, {
+var FieldCollection = Collection.extend(lodashMixin, parentMixin, {
+  modelType: 'FieldCollection',
   mainIndex: 'name',
+  model: Field,
   comparator: function (a, b) {
     // make sure _id is always at top, even in presence of uppercase fields
     var aId = a.getId();
@@ -15,8 +19,41 @@ var FieldCollection = Collection.extend(lodashMixin, {
     // otherwise sort case-insensitively
     return (aId.toLowerCase() < bId.toLowerCase()) ? -1 : 1;
   },
-  model: function(attrs, options) {
-    return new attrs.klass(attrs, options);
+  /**
+   * adds a new name/value pair to the correct field, and creates the
+   * field first if it doesn't exist yet. Leave it to field.types to
+   * add the value.
+   *
+   * @param {String} name   name of the field
+   * @param {Any} value     value to be added
+   */
+  addToField: function (name, value) {
+    // get or create field
+    var field = this.get(name);
+    if (!field) {
+      field = this.add({
+        name: name,
+        parent: this.parent
+      });
+      if (this.parent) this.parent.trigger('change:fields.length');
+      /**
+       * first time we see this field. We need to compensate for
+       * the Undefined values we missed so far for this field,
+       * by setting the count to the parent count and adjusting
+       * total_count as well.
+       */
+      if (this.parent && this.parent.count > 0) {
+        var undef = field.types.add({name: 'Undefined'});
+        undef.count += this.parent.count;
+        field.total_count += undef.count;
+      }
+    }
+    // undefined are not counted towards the field's count
+    if(value !== undefined) field.count += 1;
+    // but they are counted towards the field's total_count
+    field.total_count += 1;
+    field.types.addToType(value);
   }
 });
+
 module.exports = FieldCollection;