Skip to content
This repository

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
  • 2 commits
  • 4 files changed
  • 0 comments
  • 1 contributor
2  TODO
... ...
@@ -1,5 +1,5 @@
1 1
 Arrays
2  
- - implement Avro maps
  2
+ - test & debug Avro maps
3 3
  - add tests for ToJson tool
4 4
 
5 5
 Integration with other tools:
14  java/avro/src/main/java/org/apache/trevni/avro/AvroColumnReader.java
@@ -141,9 +141,20 @@ private Object read(Schema s) throws IOException {
141 141
     if (isSimple(s))
142 142
       return nextValue(s, column++);
143 143
 
  144
+    final int startColumn = column;
  145
+
144 146
     switch (s.getType()) {
145 147
     case MAP: 
146  
-      throw new TrevniRuntimeException("Unknown schema: "+s);
  148
+      int size = values[column].nextLength();
  149
+      Map map = (Map)new HashMap(size);
  150
+      for (int i = 0; i < size; i++) {
  151
+        this.column = startColumn;
  152
+        values[column++].nextValue();                      // null in parent
  153
+        String key = (String)values[column++].nextValue(); // key
  154
+        map.put(key, read(s.getValueType()));              // value
  155
+      }
  156
+      column = startColumn + arrayWidths[startColumn];
  157
+      return map;
147 158
     case RECORD: 
148 159
       Object record = model.newRecord(null, s);
149 160
       for (Field f : s.getFields())
@@ -152,7 +163,6 @@ private Object read(Schema s) throws IOException {
152 163
     case ARRAY: 
153 164
       int length = values[column].nextLength();
154 165
       List elements = (List)new GenericData.Array(length, s);
155  
-      int startColumn = column;
156 166
       for (int i = 0; i < length; i++) {
157 167
         this.column = startColumn;
158 168
         Object value = nextValue(s, column++);
11  java/avro/src/main/java/org/apache/trevni/avro/AvroColumnWriter.java
@@ -21,6 +21,7 @@
21 21
 import java.io.File;
22 22
 import java.io.OutputStream;
23 23
 import java.util.Collection;
  24
+import java.util.Map;
24 25
 
25 26
 import org.apache.trevni.ColumnFileMetaData;
26 27
 import org.apache.trevni.ColumnFileWriter;
@@ -84,7 +85,15 @@ private int write(Object o, Schema s, int column) throws IOException {
84 85
     }
85 86
     switch (s.getType()) {
86 87
     case MAP: 
87  
-      throw new TrevniRuntimeException("Unknown schema: "+s);
  88
+      Map<?,?> map = (Map)o;
  89
+      writer.writeLength(map.size(), column);
  90
+      for (Map.Entry e : map.entrySet()) {
  91
+        writer.writeValue(null, column);
  92
+        writer.writeValue(e.getKey(), column+1);
  93
+        int c = write(e.getValue(), s.getValueType(), column+2);
  94
+        assert(c == column+arrayWidths[column]);
  95
+      }
  96
+      return column+arrayWidths[column];
88 97
     case RECORD: 
89 98
       for (Field f : s.getFields())
90 99
         column = write(model.getField(o,f.name(),f.pos()), f.schema(), column);
6  java/avro/src/main/java/org/apache/trevni/avro/AvroColumnator.java
@@ -74,7 +74,11 @@ private void columnize(String path, Schema s,
74 74
     
75 75
     switch (s.getType()) {
76 76
     case MAP: 
77  
-      throw new TrevniRuntimeException("Can't shred maps yet: "+s);
  77
+      path = path == null ? ">" : path+">";
  78
+      ColumnMetaData p = addColumn(path, ValueType.NULL, parent, true);
  79
+      addColumn(p(path,"key", ">"), ValueType.STRING, p, false);
  80
+      columnize(p(path,"value", ">"), s.getValueType(), p, false);
  81
+      break;
78 82
     case RECORD:
79 83
       for (Field field : s.getFields())           // flatten fields to columns
80 84
         columnize(p(path, field.name(), "#"), field.schema(), parent, isArray);

No commit comments for this range

Something went wrong with that request. Please try again.