Test and document nested objects

also warn if you reduce by an object field

README.md

@@ -162,6 +162,9 @@ In this case, Juttle will issue a query against ES that matches documents whose
 * `reduce by fieldname` (other than reduce by document type)
 * `reduce -every :interval:`
 
+##### Optimization and nested objects
+There are a few fundamental incompatibilities between [Elasticsearch's model](https://www.elastic.co/guide/en/elasticsearch/guide/current/complex-core-fields.html) for nested object and array fields and [Juttle's](http://juttle.github.io/juttle/concepts/fields/#fields-with-object-or-array-values). This can lead to some odd results for optimized programs. For objects, an optimized `reduce by some_object_field` will return `null` as the only value for `some_object_field`. For arrays, an optimized `reduce by some_array_field` will return a separate value for `some_array_field` for every element in every array stored in `some_array_field`. For results conforming to Juttle's `reduce` behavior, disable optimization with `read elastic -optimize false`.
+
 In case of unexpected behavior with optimized reads, add `-optimize false` option to `read elastic` to disable optimizations, and kindly report the problem as a GitHub issue.
 
 ## Contributing

lib/read.js

@@ -96,6 +96,7 @@ class ReadElastic extends AdapterRead {
                         reduce_by_fields.forEach((field) => {
                             this._warn_if_analyzed(properties, field, index, type);
                             this._warn_if_missing(properties, field, index, type);
+                            this._warn_if_object(properties, field, index, type);
                         });
                     });
                 });
@@ -118,6 +119,13 @@ class ReadElastic extends AdapterRead {
         }
     }
 
+    _warn_if_object(properties, field, index, type) {
+        if (properties && properties[field] && properties[field].properties) {
+            this.warn(`field "${field}" is an object in type "${type}" in index ` +
+                `"${index}", results may be unexpected`);
+        }
+    }
+
     _setup_index(options) {
         this.index = options.index || this._default_config('readIndex');
         this.interval = options.indexInterval || this._default_config('indexInterval');

test/elastic-adapter.spec.js

@@ -194,19 +194,6 @@ describe('elastic source', function() {
                     });
             });
 
-            it('writes a point with an array and object field', function() {
-                var point = {
-                    time: new Date().toISOString(),
-                    array: [1,2,3],
-                    object: { key: "val", arr: [1,2,3] }
-                };
-
-                return test_utils.write([point], {id: type})
-                    .then(function(result) {
-                        return test_utils.verify_import([point], type);
-                    });
-            });
-
             it('errors if you write a point without time', function() {
                 var timeless = {value: 1, name: 'dave'};
 

test/nested-objects.spec.js

@@ -0,0 +1,101 @@
+var _ = require('underscore');
+var expect = require('chai').expect;
+
+var test_utils = require('./elastic-test-utils');
+var points = test_utils.generate_sample_data({
+    tags: {
+        nest: [{tag: 'a', arr: ['a']}, {tag: 'b', arr: ['b']}],
+        array: [['a', 'b', 'c'], ['d', 'e', 'f'], ['a', 'e', 'g']]
+    }
+});
+
+var modes = test_utils.modes;
+
+modes.forEach(function(mode) {
+    describe(`nested objects -- ${mode}`, function() {
+        this.timeout(30000);
+        before(function() {
+            return test_utils.write(points, {id: mode})
+                .then(function() {
+                    return test_utils.verify_import(points, mode);
+                });
+        });
+
+        after(function() {
+            return test_utils.clear_data(mode);
+        });
+
+        it('read', function() {
+            return test_utils.read({id: mode})
+                .then(function(result) {
+                    expect(result.sinks.table).deep.equal(points);
+                });
+        });
+
+        // depends on https://github.com/juttle/juttle/issues/320
+        it.skip('object property access filter', function() {
+            return test_utils.read({id: mode}, 'nest["tag"] = "a"')
+                .then(function(result) {
+                    expect(result.errors).deep.equal([]);
+                    var expected = points.filter(function(pt) {
+                        return pt.nest.tag === 'a';
+                    });
+
+                    expect(result.sinks.table).deep.equal(expected);
+                });
+        });
+
+        // depends on https://github.com/juttle/juttle/issues/320
+        it.skip('object filter', function() {
+            return test_utils.read({id: mode}, 'nest = {tag: "a"}')
+                .then(function(result) {
+                    expect(result.errors).deep.equal([]);
+                    var expected = points.filter(function(pt) {
+                        return pt.nest.tag === 'a';
+                    });
+
+                    expect(result.sinks.table).deep.equal(expected);
+                });
+        });
+
+        it('optimized reduce by object: warns', function() {
+            return test_utils.read({id: mode}, '| reduce count() by nest')
+                .then(function(result) {
+                    var expected = [{nest: null, count: points.length}];
+                    expect(result.sinks.table).deep.equal(expected);
+                    expect(result.warnings).match(/"nest" is an object/);
+                });
+        });
+
+        // inconsistent with Juttle which can't reduce by object field, but useful
+        it('optimized reduce by object field', function() {
+            return test_utils.read({id: mode}, '| reduce count() by "nest.tag"')
+                .then(function(result) {
+                    var counts = _.countBy(points, function(pt) {
+                        return pt.nest.tag;
+                    });
+
+                    var expected = _.chain(counts).map(function(count, tag) {
+                        return {count: count, 'nest.tag': tag};
+                    }).sortBy('nest.tag').value();
+
+                    var received = _.sortBy(result.sinks.table, 'nest.tag');
+
+                    expect(expected).deep.equal(received);
+                });
+        });
+
+        // depends on https://github.com/juttle/juttle/issues/320
+        it.skip('array literal filter', function() {
+            return test_utils.read({id: mode}, 'array = ["a", "b", "c"]')
+                .then(function(result) {
+                    var expected = points.filter(function(pt) {
+                        return pt.array[0] === 'a' && pt.array[1] === 'b' &&
+                            pt.array[2] === 'c';
+                    });
+
+                    expect(result.sinks.table).deep.equal(expected);
+                });
+        });
+    });
+});