working towards references

lib/avro_ex/decode.ex

@@ -3,7 +3,7 @@ defmodule AvroEx.Decode do
 
   require Bitwise
   alias AvroEx.Schema
-  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Union}
+  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Reference, Union}
   alias AvroEx.Schema.Record.Field
 
   @type reason :: term
@@ -16,7 +16,7 @@ defmodule AvroEx.Decode do
     {:ok, value}
   end
 
-  defp do_decode(name, %Context{} = context, data) when is_binary(name) do
+  defp do_decode(%Reference{type: name}, %Context{} = context, data) do
     do_decode(Context.lookup(context, name), context, data)
   end
 

lib/avro_ex/encode.ex

@@ -5,7 +5,7 @@ defmodule AvroEx.Encode do
 
   alias AvroEx.EncodeError
   alias AvroEx.{Schema}
-  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Union}
+  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Reference, Union}
   alias AvroEx.Schema.Enum, as: AvroEnum
   alias AvroEx.Schema.Record.Field
 
@@ -21,8 +21,9 @@ defmodule AvroEx.Encode do
     end
   end
 
-  defp do_encode(name, %Context{} = context, data) when is_binary(name),
-    do: do_encode(Context.lookup(context, name), context, data)
+  defp do_encode(%Reference{type: type}, %Context{} = context, data) do
+    do_encode(Context.lookup(context, type), context, data)
+  end
 
   defp do_encode(%Primitive{type: :boolean}, %Context{}, true), do: <<1::8>>
   defp do_encode(%Primitive{type: :boolean}, %Context{}, false), do: <<0::8>>

lib/avro_ex/schema.ex

@@ -3,7 +3,7 @@ defmodule AvroEx.Schema do
   alias AvroEx.Schema.Enum, as: AvroEnum
   alias AvroEx.Schema.Map, as: AvroMap
   alias AvroEx.Schema.Record.Field
-  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Union}
+  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Reference, Union}
 
   defstruct [:context, :schema]
 
@@ -153,7 +153,7 @@ defmodule AvroEx.Schema do
     AvroEnum.match?(schema, context, data)
   end
 
-  def encodable?(name, %Context{} = context, data) when is_binary(name) do
+  def encodable?(%Reference{type: name}, %Context{} = context, data) do
     schema = Context.lookup(context, name)
     encodable?(schema, context, data)
   end
@@ -263,7 +263,11 @@ defmodule AvroEx.Schema do
   @spec full_name(schema_types()) :: nil | String.t()
   def full_name(%struct{}) when struct in [Array, AvroMap, Primitive, Union], do: nil
 
-  def full_name(%struct{name: name, namespace: namespace}) when struct in [Fixed, Record, AvroEnum] do
+  def full_name(%Record.Field{name: name}) do
+    name
+  end
+
+  def full_name(%struct{name: name, namespace: namespace}) when struct in [AvroEnum, Fixed, Record] do
     full_name(namespace, name)
   end
 
@@ -312,6 +316,7 @@ defmodule AvroEx.Schema do
   def type_name(%Array{items: type}), do: "Array<items=#{type_name(type)}>"
   def type_name(%Union{possibilities: types}), do: "Union<possibilities=#{Enum.map_join(types, "|", &type_name/1)}>"
   def type_name(%Record{} = record), do: "Record<name=#{full_name(record)}>"
+  def type_name(%Record.Field{} = field), do: "Field<name=#{full_name(field)}>"
   def type_name(%Fixed{size: size} = fixed), do: "Fixed<name=#{full_name(fixed)}, size=#{size}>"
   def type_name(%AvroEnum{} = enum), do: "Enum<name=#{full_name(enum)}>"
   def type_name(%AvroMap{values: values}), do: "Map<values=#{type_name(values)}>"

lib/avro_ex/schema/parser.ex

@@ -1,9 +1,11 @@
 defmodule AvroEx.Schema.Parser do
+  @doc false
   alias AvroEx.Schema
-  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Union}
+  alias AvroEx.Schema.{Array, Context, Fixed, Primitive, Record, Reference, Union}
   alias AvroEx.Schema.Enum, as: AvroEnum
   alias AvroEx.Schema.Map, as: AvroMap
 
+  # TODO convert to atom
   @primitives [
     "null",
     "boolean",
@@ -21,22 +23,27 @@ defmodule AvroEx.Schema.Parser do
 
   def primitive?(_), do: false
 
-  # Parses a schema from an elixir (string) map representation
-  #
-  # Will throw DecodeError on structural and type issues,
-  # but will not do any validation of semantic information,
-  # i.e. is this logicalType valid?
   @spec parse!(term()) :: AvroEx.Schema.t()
   def parse!(data) do
     try do
       type = do_parse(data)
+      context = build_context(type, %Context{})
 
-      %Schema{schema: type, context: %Context{}}
+      %Schema{schema: type, context: context}
     catch
       :throw, %AvroEx.Schema.DecodeError{} = err -> raise err
     end
   end
 
+  # do_parse_ref/1 handles types that might be a %Reference{}
+  defp do_parse_ref(term) do
+    if is_binary(term) and not primitive?(term) do
+      Reference.new(term)
+    else
+      do_parse(term)
+    end
+  end
+
   defp do_parse(nil), do: %Primitive{type: :null}
 
   for p <- @primitives do
@@ -48,7 +55,7 @@ defmodule AvroEx.Schema.Parser do
   defp do_parse(list) when is_list(list) do
     {possibilities, _} =
       Enum.map_reduce(list, MapSet.new(), fn type, seen ->
-        %struct{} = parsed = do_parse(type)
+        %struct{} = parsed = do_parse_ref(type)
 
         if match?(%Union{}, parsed) do
           error({:nested_union, parsed, list})
@@ -88,9 +95,9 @@ defmodule AvroEx.Schema.Parser do
       |> validate_required([:values])
       |> drop([:type])
       |> extract_data()
-      |> update_in([:values], &do_parse/1)
+      |> update_in([:values], &do_parse_ref/1)
 
-    struct!(AvroMap, data) |> validate_default()
+    struct!(AvroMap, data)
   end
 
   defp do_parse(%{"type" => "enum", "symbols" => symbols} = enum) when is_list(symbols) do
@@ -115,7 +122,7 @@ defmodule AvroEx.Schema.Parser do
       MapSet.put(set, symbol)
     end)
 
-    struct!(AvroEnum, data) |> validate_default()
+    struct!(AvroEnum, data)
   end
 
   defp do_parse(%{"type" => "array"} = array) do
@@ -125,9 +132,9 @@ defmodule AvroEx.Schema.Parser do
       |> drop([:type])
       |> validate_required([:items])
       |> extract_data()
-      |> update_in([:items], &do_parse/1)
+      |> update_in([:items], &do_parse_ref/1)
 
-    struct!(Array, data) |> validate_default()
+    struct!(Array, data)
   end
 
   defp do_parse(%{"type" => "fixed"} = fixed) do
@@ -168,9 +175,9 @@ defmodule AvroEx.Schema.Parser do
       |> cast(Record.Field, [:aliases, :doc, :default, :name, :namespace, :order, :type])
       |> validate_required([:name, :type])
       |> extract_data()
-      |> put_in([:type], do_parse(type))
+      |> put_in([:type], do_parse_ref(type))
 
-    struct!(Record.Field, data) |> validate_default()
+    struct!(Record.Field, data)
   end
 
   defp cast(data, type, keys) do
@@ -261,6 +268,68 @@ defmodule AvroEx.Schema.Parser do
     {data, Map.drop(rest, Enum.map(keys, &to_string/1)), info}
   end
 
+  defp build_context(type, context) do
+    context = capture_context(type, context)
+
+    type
+    |> validate_default()
+    |> do_build_context(context)
+  end
+
+  defp do_build_context(%Union{} = union, context) do
+    build_inner_context(union, :possibilities, context)
+  end
+
+  defp do_build_context(%Record{} = record, context) do
+    build_inner_context(record, :fields, context)
+  end
+
+  defp do_build_context(%Record.Field{} = field, context) do
+    build_inner_context(field, :type, context)
+  end
+
+  defp do_build_context(%Array{} = array, context) do
+    build_inner_context(array, :items, context)
+  end
+
+  defp do_build_context(%AvroMap{} = map, context) do
+    build_inner_context(map, :values, context)
+  end
+
+  defp do_build_context(%Reference{} = ref, context) do
+    unless Map.has_key?(context.names, ref.type) do
+      error({:missing_ref, ref, context})
+    end
+
+    context
+  end
+
+  defp do_build_context(_schema, context), do: context
+
+  defp build_inner_context(type, field, context) do
+    %{^field => inner} = type
+
+    if is_list(inner) do
+      Enum.reduce(inner, context, &build_context/2)
+    else
+      build_context(inner, context)
+    end
+  end
+
+  defp capture_context(%{name: name} = schema, context) do
+    name = AvroEx.Schema.full_name(schema)
+
+    if Map.has_key?(context.names, name) do
+      error({:duplicate_name, name, schema})
+    end
+
+    # TODO aliases and namespace propagation
+
+    put_in(context.names[name], schema)
+  end
+
+  defp capture_context(_type, context), do: context
+
   defp error(info) do
     info |> AvroEx.Schema.DecodeError.new() |> throw()
   end

lib/avro_ex/schema/record.ex

@@ -12,7 +12,9 @@ defmodule AvroEx.Schema.Record do
     field(:doc, :string)
     field(:name, :string)
     field(:namespace, :string)
+    # TODO remove all of these
     field(:qualified_names, {:array, :string}, default: [])
+    # TODO remove
     field(:metadata, :map, default: %{})
 
     embeds_many(:fields, Field)

lib/avro_ex/schema/reference.ex

@@ -0,0 +1,7 @@
+defmodule AvroEx.Schema.Reference do
+  defstruct [:type]
+
+  def new(type) when is_binary(type) do
+    %__MODULE__{type: type}
+  end
+end

lib/avro_ex/schema/schema_decode_error.ex

@@ -39,6 +39,12 @@ defmodule AvroEx.Schema.DecodeError do
     %__MODULE__{message: message}
   end
 
+  def new({:duplicate_name, name, schema}) do
+    type = AvroEx.Schema.type_name(schema)
+    message = "Duplicate name #{surround(name)} found in #{type}"
+    %__MODULE__{message: message}
+  end
+
   def new({:invalid_name, {field, name}, context}) do
     message = "Invalid name #{surround(name)} for #{surround(field)} in #{inspect(context)}"
     %__MODULE__{message: message}
@@ -61,6 +67,18 @@ defmodule AvroEx.Schema.DecodeError do
     %__MODULE__{message: message}
   end
 
+  def new({:missing_ref, ref, context}) do
+    known =
+      if context.names == %{} do
+        "empty"
+      else
+        context.names |> Map.keys() |> Enum.map_join(", ", &surround/1)
+      end
+
+    message = "Found undeclared reference #{surround(ref.type)}. Known references are #{known}"
+    %__MODULE__{message: message}
+  end
+
   defp surround(string, value \\ "`") do
     value <> to_string(string) <> value
   end