Fix PigValueWriter not excluding fields when writing tuples.

Fixes #1015
elastic · Jul 24, 2017 · 66b8c17 · 66b8c17
1 parent 173c09d
commit 66b8c17
Show file tree

Hide file tree

Showing 2 changed files with 38 additions and 7 deletions.
diff --git a/pig/src/main/java/org/elasticsearch/hadoop/pig/PigValueWriter.java b/pig/src/main/java/org/elasticsearch/hadoop/pig/PigValueWriter.java
@@ -229,15 +229,24 @@ private Result writeTuple(Object object, ResourceFieldSchema field, Generator ge
         }
 
         for (int i = 0; i < nestedFields.length; i++) {
+            // Do this based on if the field name is included.
             if (writeAsObject) {
                 String name = nestedFields[i].getName();
                 // handle schemas without names
-                name = (StringUtils.hasText(name) ? alias.toES(name) : Integer.toString(i));
-                generator.writeFieldName(name);
-            }
-            Result res = write(tuples.get(i), nestedFields[i], generator);
-            if (!res.isSuccesful()) {
-                return res;
+                boolean fieldHasName = StringUtils.hasText(name);
+                String actualName = (fieldHasName ? alias.toES(name) : Integer.toString(i));
+                if (shouldKeep(generator.getParentPath(), actualName)) {
+                    generator.writeFieldName(actualName);
+                    Result res = write(tuples.get(i), nestedFields[i], generator);
+                    if (!res.isSuccesful()) {
+                        return res;
+                    }
+                }
+            } else {
+                Result res = write(tuples.get(i), nestedFields[i], generator);
+                if (!res.isSuccesful()) {
+                    return res;
+                }
             }
         }
         if (writeAsObject) {

diff --git a/pig/src/test/java/org/elasticsearch/hadoop/serialization/PigNamedTypeToJsonTest.java b/pig/src/test/java/org/elasticsearch/hadoop/serialization/PigNamedTypeToJsonTest.java
@@ -28,10 +28,13 @@
 import org.apache.pig.data.Tuple;
 import org.apache.pig.data.TupleFactory;
 import org.apache.pig.impl.util.Utils;
+import org.elasticsearch.hadoop.cfg.ConfigurationOptions;
+import org.elasticsearch.hadoop.cfg.Settings;
 import org.elasticsearch.hadoop.pig.PigTuple;
 import org.elasticsearch.hadoop.pig.PigValueWriter;
 import org.elasticsearch.hadoop.serialization.builder.ContentBuilder;
 import org.elasticsearch.hadoop.util.FastByteArrayOutputStream;
+import org.elasticsearch.hadoop.util.TestSettings;
 import org.junit.After;
 import org.junit.AfterClass;
 import org.junit.Before;
@@ -242,6 +245,15 @@ public void testBagWithAnonTuple() {
                 createSchema("{t:(chararray, chararray)}"))), is(expected));
     }
 
+    @Test
+    public void testNamedTupleWithExclusion() {
+        Settings settings = new TestSettings();
+        settings.setProperty(ConfigurationOptions.ES_MAPPING_EXCLUDE, "namedtuple.second");
+        String expected = "{\"namedtuple\":[{\"first\":\"one\"}]}";
+        assertThat(pigTypeToJson(createTuple(TupleFactory.getInstance().newTuple(Arrays.asList(new String[] { "one", "two" })),
+                createSchema("namedtuple: (first:chararray, second:chararray)")), settings), is(expected));
+    }
+
     private ResourceSchema createSchema(String schema) {
         try {
             return new ResourceSchema(Utils.getSchemaFromString(schema));
@@ -257,7 +269,17 @@ private PigTuple createTuple(Object obj, ResourceSchema schema) {
     }
 
     private String pigTypeToJson(PigTuple obj) {
-        ContentBuilder.generate(out, new PigValueWriter(true)).value(obj).flush().close();
+        return pigTypeToJson(obj, null);
+    }
+
+    private String pigTypeToJson(PigTuple obj, Settings settings) {
+        PigValueWriter writer = new PigValueWriter(true);
+        if (settings != null) {
+            // Make sure to write with use.field.names as true
+            settings.setProperty("es.mapping.pig.tuple.use.field.names", "true");
+            writer.setSettings(settings);
+        }
+        ContentBuilder.generate(out, writer).value(obj).flush().close();
         return out.bytes().toString();
     }
 }