Merge oficial/master into trunk-AVRO-1723

Conflicts:
	lang/java/avro/src/test/java/org/apache/avro/TestFixed.java
	lang/ruby/lib/avro.rb

.gitignore

@@ -15,3 +15,4 @@ test-output
 /dist
 /lang/java/compiler/nbactions.xml
 /lang/java/compiler/nb-configuration.xml
+/lang/java/compiler/nbproject/

lang/java/avro/src/test/java/org/apache/avro/TestFixed.java

@@ -16,4 +16,4 @@ public void testFixedDefaultValueDrop() {
     Assert.assertArrayEquals(new byte[16], (byte[]) field.defaultVal());
   }
 
-}
+}

lang/ruby/Manifest

@@ -11,6 +11,7 @@ lib/avro/io.rb
 lib/avro/ipc.rb
 lib/avro/protocol.rb
 lib/avro/schema.rb
+lib/avro/schema_compatibility.rb
 lib/avro/schema_normalization.rb
 lib/avro/schema_validator.rb
 test/case_finder.rb
@@ -25,6 +26,7 @@ test/test_help.rb
 test/test_io.rb
 test/test_protocol.rb
 test/test_schema.rb
+test/test_schema_compatibility.rb
 test/test_schema_normalization.rb
 test/test_schema_validator.rb
 test/test_socket_transport.rb

lang/ruby/lib/avro.rb

@@ -41,3 +41,4 @@ def initialize(schm=nil, datum=nil, msg=nil)
 require 'avro/ipc'
 require 'avro/schema_normalization'
 require 'avro/schema_validator'
+require 'avro/schema_compatibility'

lang/ruby/lib/avro/io.rb

@@ -221,46 +221,7 @@ def write(datum)
 
     class DatumReader
       def self.match_schemas(writers_schema, readers_schema)
-        w_type = writers_schema.type_sym
-        r_type = readers_schema.type_sym
-
-        # This conditional is begging for some OO love.
-        if w_type == :union || r_type == :union
-          return true
-        end
-
-        if w_type == r_type
-          return true if Schema::PRIMITIVE_TYPES_SYM.include?(r_type)
-
-          case r_type
-          when :record
-            return writers_schema.fullname == readers_schema.fullname
-          when :error
-            return writers_schema.fullname == readers_schema.fullname
-          when :request
-            return true
-          when :fixed
-            return writers_schema.fullname == readers_schema.fullname &&
-                   writers_schema.size == readers_schema.size
-          when :enum
-            return writers_schema.fullname == readers_schema.fullname
-          when :map
-            return writers_schema.values.type == readers_schema.values.type
-          when :array
-            return writers_schema.items.type == readers_schema.items.type
-          end
-        end
-
-        # Handle schema promotion
-        if w_type == :int && [:long, :float, :double].include?(r_type)
-          return true
-        elsif w_type == :long && [:float, :double].include?(r_type)
-          return true
-        elsif w_type == :float && r_type == :double
-          return true
-        end
-
-        return false
+        Avro::SchemaCompatibility.match_schemas(writers_schema, readers_schema)
       end
 
       attr_accessor :writers_schema, :readers_schema
@@ -393,11 +354,11 @@ def read_record(writers_schema, readers_schema, decoder)
           writers_fields_hash = writers_schema.fields_hash
           readers_fields_hash.each do |field_name, field|
             unless writers_fields_hash.has_key? field_name
-              if !field.default.nil?
+              if field.default?
                 field_val = read_default_value(field.type, field.default)
                 read_record[field.name] = field_val
               else
-                # FIXME(jmhodges) another 'unset' here
+                raise AvroError, "Missing data for #{field.type} with no default"
               end
             end
           end
@@ -407,10 +368,6 @@ def read_record(writers_schema, readers_schema, decoder)
       end
 
       def read_default_value(field_schema, default_value)
-        if default_value == :no_default
-          raise AvroError, "Missing data for #{field_schema} with no default"
-        end
-
         # Basically a JSON Decoder?
         case field_schema.type_sym
         when :null

lang/ruby/lib/avro/protocol.rb

@@ -20,7 +20,7 @@ class Protocol
     VALID_TYPE_SCHEMA_TYPES_SYM = Set.new(VALID_TYPE_SCHEMA_TYPES.map(&:to_sym))
     class ProtocolParseError < Avro::AvroError; end
 
-    attr_reader :name, :namespace, :types, :messages, :md5
+    attr_reader :name, :namespace, :types, :messages, :md5, :doc
     def self.parse(protocol_string)
       json_data = MultiJson.load(protocol_string)
 
@@ -29,13 +29,14 @@ def self.parse(protocol_string)
         namespace = json_data['namespace']
         types = json_data['types']
         messages = json_data['messages']
-        Protocol.new(name, namespace, types, messages)
+        doc = json_data['doc']
+        Protocol.new(name, namespace, types, messages, doc)
       else
         raise ProtocolParseError, "Not a JSON object: #{json_data}"
       end
     end
 
-    def initialize(name, namespace=nil, types=nil, messages=nil)
+    def initialize(name, namespace=nil, types=nil, messages=nil, doc=nil)
       # Ensure valid ctor args
       if !name
         raise ProtocolParseError, 'Protocols must have a non-empty name.'
@@ -55,6 +56,7 @@ def initialize(name, namespace=nil, types=nil, messages=nil)
       @types = parse_types(types, type_names)
       @messages = parse_messages(messages, type_names)
       @md5 = Digest::MD5.digest(to_s)
+      @doc = doc
     end
 
     def to_s
@@ -92,7 +94,8 @@ def parse_messages(messages, names)
         request  = body['request']
         response = body['response']
         errors   = body['errors']
-        message_objects[name] = Message.new(name, request, response, errors, names, namespace)
+        doc      = body['doc']
+        message_objects[name] = Message.new(name, request, response, errors, names, namespace, doc)
       end
       message_objects
     end
@@ -111,14 +114,15 @@ def to_avro(names=Set.new)
     end
 
     class Message
-      attr_reader :name, :request, :response, :errors, :default_namespace
+      attr_reader :name, :request, :response, :errors, :default_namespace, :doc
 
-      def initialize(name, request, response, errors=nil, names=nil, default_namespace=nil)
+      def initialize(name, request, response, errors=nil, names=nil, default_namespace=nil, doc=nil)
         @name = name
         @default_namespace = default_namespace
         @request = parse_request(request, names)
         @response = parse_response(response, names)
         @errors = parse_errors(errors, names) if errors
+        @doc = doc
       end
 
       def to_avro(names=Set.new)
@@ -127,6 +131,7 @@ def to_avro(names=Set.new)
           'response' => response.to_avro(names)
         }.tap do |hash|
           hash['errors'] = errors.to_avro(names) if errors
+          hash['doc'] = @doc if @doc
         end
       end
 

lang/ruby/lib/avro/schema.rb

@@ -61,10 +61,12 @@ def self.real_parse(json_obj, names=nil, default_namespace=nil)
             return FixedSchema.new(name, namespace, size, names)
           when :enum
             symbols = json_obj['symbols']
-            return EnumSchema.new(name, namespace, symbols, names)
+            doc     = json_obj['doc']
+            return EnumSchema.new(name, namespace, symbols, names, doc)
           when :record, :error
             fields = json_obj['fields']
-            return RecordSchema.new(name, namespace, fields, names, type_sym)
+            doc    = json_obj['doc']
+            return RecordSchema.new(name, namespace, fields, names, type_sym, doc)
           else
             raise SchemaParseError.new("Unknown named type: #{type}")
           end
@@ -120,6 +122,18 @@ def sha256_fingerprint
       Digest::SHA256.hexdigest(parsing_form).to_i(16)
     end
 
+    def read?(writers_schema)
+      SchemaCompatibility.can_read?(writers_schema, self)
+    end
+
+    def be_read?(other_schema)
+      other_schema.read?(self)
+    end
+
+    def mutual_read?(other_schema)
+      SchemaCompatibility.mutual_read?(other_schema, self)
+    end
+
     def ==(other, seen=nil)
       other.is_a?(Schema) && type_sym == other.type_sym
     end
@@ -152,9 +166,10 @@ def to_s
 
     class NamedSchema < Schema
       attr_reader :name, :namespace
-      def initialize(type, name, namespace=nil, names=nil)
+      def initialize(type, name, namespace=nil, names=nil, doc=nil)
         super(type)
         @name, @namespace = Name.extract_namespace(name, namespace)
+        @doc  = doc
         names = Name.add_name(names, self)
       end
 
@@ -165,6 +180,7 @@ def to_avro(names=Set.new)
         end
         props = {'name' => @name}
         props.merge!('namespace' => @namespace) if @namespace
+        props.merge!('doc' => @doc) if @doc
         super.merge props
       end
 
@@ -174,7 +190,7 @@ def fullname
     end
 
     class RecordSchema < NamedSchema
-      attr_reader :fields
+      attr_reader :fields, :doc
 
       def self.make_field_objects(field_data, names, namespace=nil)
         field_objects, field_names = [], Set.new
@@ -184,7 +200,8 @@ def self.make_field_objects(field_data, names, namespace=nil)
             name = field['name']
             default = field.key?('default') ? field['default'] : :no_default
             order = field['order']
-            new_field = Field.new(type, name, default, order, names, namespace)
+            doc = field['doc']
+            new_field = Field.new(type, name, default, order, names, namespace, doc)
             # make sure field name has not been used yet
             if field_names.include?(new_field.name)
               raise SchemaParseError, "Field name #{new_field.name.inspect} is already in use"
@@ -198,14 +215,18 @@ def self.make_field_objects(field_data, names, namespace=nil)
         field_objects
       end
 
-      def initialize(name, namespace, fields, names=nil, schema_type=:record)
+      def initialize(name, namespace, fields, names=nil, schema_type=:record, doc=nil)
         if schema_type == :request || schema_type == 'request'
           @type_sym = schema_type.to_sym
           @namespace = namespace
         else
-          super(schema_type, name, namespace, names)
+          super(schema_type, name, namespace, names, doc)
         end
-        @fields = RecordSchema.make_field_objects(fields, names, self.namespace)
+        @fields = if fields
+                    RecordSchema.make_field_objects(fields, names, self.namespace)
+                  else
+                    {}
+                  end
       end
 
       def fields_hash
@@ -256,8 +277,7 @@ class UnionSchema < Schema
       def initialize(schemas, names=nil, default_namespace=nil)
         super(:union)
 
-        schema_objects = []
-        schemas.each_with_index do |schema, i|
+        @schemas = schemas.each_with_object([]) do |schema, schema_objects|
           new_schema = subparse(schema, names, default_namespace)
           ns_type = new_schema.type_sym
 
@@ -270,7 +290,6 @@ def initialize(schemas, names=nil, default_namespace=nil)
           else
             schema_objects << new_schema
           end
-          @schemas = schema_objects
         end
       end
 
@@ -280,13 +299,14 @@ def to_avro(names=Set.new)
     end
 
     class EnumSchema < NamedSchema
-      attr_reader :symbols
-      def initialize(name, space, symbols, names=nil)
+      attr_reader :symbols, :doc
+
+      def initialize(name, space, symbols, names=nil, doc=nil)
         if symbols.uniq.length < symbols.length
           fail_msg = 'Duplicate symbol: %s' % symbols
           raise Avro::SchemaParseError, fail_msg
         end
-        super(:enum, name, space, names)
+        super(:enum, name, space, names, doc)
         @symbols = symbols
       end
 
@@ -332,19 +352,25 @@ def to_avro(names=Set.new)
     end
 
     class Field < Schema
-      attr_reader :type, :name, :default, :order
+      attr_reader :type, :name, :default, :order, :doc
 
-      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil)
+      def initialize(type, name, default=:no_default, order=nil, names=nil, namespace=nil, doc=nil)
         @type = subparse(type, names, namespace)
         @name = name
         @default = default
         @order = order
+        @doc = doc
+      end
+
+      def default?
+        @default != :no_default
       end
 
       def to_avro(names=Set.new)
         {'name' => name, 'type' => type.to_avro(names)}.tap do |avro|
-          avro['default'] = default unless default == :no_default
+          avro['default'] = default if default?
           avro['order'] = order if order
+          avro['doc'] = doc if doc
         end
       end
     end