Skip to content

Commit

Permalink
aliases
Browse files Browse the repository at this point in the history
  • Loading branch information
@davydog187
davydog187 committed Mar 3, 2022
1 parent 9f9eb0f commit 7077e6c
Show file tree
Hide file tree
Showing 2 changed files with 188 additions and 63 deletions.
56 changes: 53 additions & 3 deletions lib/avro_ex/schema/parser.ex
View file
Expand Up @@ -108,6 +108,7 @@ defmodule AvroEx.Schema.Parser do
|> validate_required([:name, :symbols])
|> validate_name()
|> validate_namespace()
|> validate_aliases()
|> extract_data()

Enum.reduce(symbols, MapSet.new(), fn symbol, set ->
Expand Down Expand Up @@ -146,6 +147,7 @@ defmodule AvroEx.Schema.Parser do
|> validate_integer(:size)
|> validate_name()
|> validate_namespace()
|> validate_aliases()
|> extract_data()

struct!(Fixed, data)
Expand All @@ -159,6 +161,7 @@ defmodule AvroEx.Schema.Parser do
|> validate_required([:name, :fields])
|> validate_name()
|> validate_namespace()
|> validate_aliases()
|> extract_data()
|> update_in([:fields], fn fields -> Enum.map(fields, &parse_fields/1) end)

Expand All @@ -174,6 +177,7 @@ defmodule AvroEx.Schema.Parser do
field
|> cast(Record.Field, [:aliases, :doc, :default, :name, :namespace, :order, :type])
|> validate_required([:name, :type])
|> validate_aliases()
|> extract_data()
|> put_in([:type], do_parse_ref(type))

Expand Down Expand Up @@ -233,6 +237,14 @@ defmodule AvroEx.Schema.Parser do
end)
end

defp validate_aliases({_data, _rest, {_type, raw}} = input) do
validate_field(input, :aliases, fn aliases ->
unless is_list(aliases) and Enum.all?(aliases, &valid_name?/1) do
error({:invalid_name, {:aliases, aliases}, raw})
end
end)
end

defp validate_namespace({_data, _rest, {_type, raw}} = input) do
validate_field(input, :namespace, fn value ->
unless valid_namespace?(value) do
Expand All @@ -258,6 +270,8 @@ defmodule AvroEx.Schema.Parser do

defp extract_data({data, rest, {type, raw}}) do
if rest != %{} do
# TODO this violates the spec
# where typeName is either a primitive or derived type name, as defined below. Attributes not defined in this document are permitted as metadata, but must not affect the format of serialized data.
error({:unrecognized_fields, Map.keys(rest), type, raw})
end

Expand Down Expand Up @@ -316,20 +330,56 @@ defmodule AvroEx.Schema.Parser do
end
end

defp capture_context(%{name: name} = schema, context) do
defp capture_context(%Record.Field{}, context), do: context

defp capture_context(%{name: _name} = schema, context) do
name = AvroEx.Schema.full_name(schema)

if Map.has_key?(context.names, name) do
error({:duplicate_name, name, schema})
end

# TODO aliases and namespace propagation
if match?(%Record{}, schema) do
Enum.reduce(schema.fields, MapSet.new(), fn field, set ->
if MapSet.member?(set, field.name) do
error({:duplicate_name, field.name, schema})
end

put_in(context.names[name], schema)
MapSet.put(set, field.name)
end)
end

# TODO needs to propagate
parent_namespace = nil

context =
schema
|> aliases(parent_namespace)
|> Enum.reduce(context, fn name, context ->
put_context(context, name, schema)
end)

put_context(context, name, schema)
end

defp capture_context(_type, context), do: context

defp put_context(context, name, schema) do
put_in(context.names[name], schema)
end

defp aliases(%{aliases: aliases, namespace: namespace} = record, parent_namespace)
when is_list(aliases) do
full_aliases =
Enum.map(aliases, fn name ->
AvroEx.Schema.full_name(namespace || parent_namespace, name)
end)

[AvroEx.Schema.full_name(namespace || parent_namespace, record.name) | full_aliases]
end

defp aliases(_schema, _parent_namespace), do: []

defp error(info) do
info |> AvroEx.Schema.DecodeError.new() |> throw()
end
Expand Down
195 changes: 135 additions & 60 deletions test/schema_parser_test.exs
View file
Expand Up @@ -165,7 +165,7 @@ defmodule AvroEx.Schema.ParserTest do
end

test "field names must be unique" do
message = "Duplicate name `key` found in Field<name=key>"
message = "Duplicate name `key` found in Record<name=duplicate_names>"

assert_raise AvroEx.Schema.DecodeError, message, fn ->
Parser.parse!(%{
Expand Down Expand Up @@ -443,7 +443,7 @@ defmodule AvroEx.Schema.ParserTest do
"name" => "double",
"namespace" => "one.two.three",
"doc" => "two numbers",
"aliases" => ["dos nums"],
"aliases" => ["dos_nums"],
"type" => "fixed",
"size" => 2
})
Expand All @@ -453,18 +453,13 @@ defmodule AvroEx.Schema.ParserTest do
namespace: "one.two.three",
size: 2,
doc: "two numbers",
aliases: ["dos nums"]
aliases: ["dos_nums"]
}

assert context == %Context{
names: %{
"one.two.three.double" => %AvroEx.Schema.Fixed{
aliases: ["dos nums"],
doc: "two numbers",
name: "double",
namespace: "one.two.three",
size: 2
}
"one.two.three.double" => schema,
"one.two.three.dos_nums" => schema
}
}
end
Expand Down Expand Up @@ -612,23 +607,22 @@ defmodule AvroEx.Schema.ParserTest do
fields: [
%Record.Field{
name: "favorite_pet",
type:
%Record{
name: "Pet",
fields: [
%Record.Field{
name: "type",
type: %AvroEnum{name: "PetType", symbols: ["cat", "dog"]} = pet_type
},
%Record.Field{name: "name", type: %Primitive{type: :string}}
]
} = pet
type: %Record{
name: "Pet",
fields: [
%Record.Field{
name: "type",
type: %AvroEnum{name: "PetType", symbols: ["cat", "dog"]}
},
%Record.Field{name: "name", type: %Primitive{type: :string}}
]
}
},
%Record.Field{name: "first_pet", type: %Reference{type: "Pet"}}
]
} = schema

assert %Context{names: %{"Pet" => pet, "PetType" => pet_type}}
assert Map.keys(context.names) == ["Pet", "PetType", "pets"]
end

test "types can be referred by an alias" do
Expand Down Expand Up @@ -673,53 +667,134 @@ defmodule AvroEx.Schema.ParserTest do
name: "top"
}

assert context ==
%AvroEx.Schema.Context{
names: %{
"a" => %AvroEx.Schema.Enum{
aliases: ["b", "c"],
name: "a",
symbols: ["x"]
},
"top" => %AvroEx.Schema.Record{
fields: [
%AvroEx.Schema.Record.Field{
name: "one",
type: %AvroEx.Schema.Enum{
aliases: ["b", "c"],
name: "a",
symbols: ["x"]
}
},
%AvroEx.Schema.Record.Field{
name: "two",
type: %AvroEx.Schema.Reference{type: "a"}
},
%AvroEx.Schema.Record.Field{
name: "three",
type: %AvroEx.Schema.Reference{type: "b"}
},
%AvroEx.Schema.Record.Field{
name: "four",
type: %AvroEx.Schema.Reference{type: "c"}
}
],
name: "top",
qualified_names: []
}
}
}
assert Map.keys(context.names) == ["a", "b", "c", "top"]
end

test "can create recursive types" do
flunk()
assert %Schema{schema: schema} =
Parser.parse!(%{
"type" => "record",
"name" => "recursive",
"fields" => [
%{"name" => "nested", "type" => ["null", "recursive"]}
]
})

assert schema == %Record{
name: "recursive",
fields: [
%Record.Field{
name: "nested",
type: %Union{possibilities: [%Primitive{type: :null}, %Reference{type: "recursive"}]}
}
]
}
end

test "aliases must be valid" do
message =
"Invalid name `` for `aliases` in %{\"aliases\" => \"\", \"fields\" => [%{\"name\" => \"one\", \"type\" => \"string\"}], \"name\" => \"invalid_aliases\", \"type\" => \"record\"}"

assert_raise AvroEx.Schema.DecodeError, message, fn ->
Parser.parse!(%{
"type" => "record",
"name" => "invalid_aliases",
"aliases" => "",
"fields" => [%{"name" => "one", "type" => "string"}]
})
end

message =
"Invalid name `bad name` for `aliases` in %{\"aliases\" => [\"bad name\"], \"fields\" => [%{\"name\" => \"one\", \"type\" => \"string\"}], \"name\" => \"invalid_aliases\", \"type\" => \"record\"}"

assert_raise AvroEx.Schema.DecodeError, message, fn ->
Parser.parse!(%{
"type" => "record",
"name" => "invalid_aliases",
"aliases" => ["bad name"],
"fields" => [%{"name" => "one", "type" => "string"}]
})
end
end

test "must refer to types previously defined" do
flunk()
message = "Found undeclared reference `callback`. Known references are `invalid_ref`"

assert_raise AvroEx.Schema.DecodeError, message, fn ->
Parser.parse!(%{
"type" => "record",
"name" => "invalid_ref",
"fields" => [
%{"name" => "one", "type" => "callback"},
%{"name" => "two", "type" => %{"name" => "callback", "type" => "fixed", "size" => 2}}
]
})
end
end

test "namespaces are inherited" do
assert %Schema{schema: schema} =
Parser.parse!(%{
"type" => "record",
"name" => "inferred_reference",
"namespace" => "beam.community",
"fields" => [
%{"name" => "one", "type" => %{"name" => "callback", "type" => "fixed", "size" => 2}},
%{"name" => "two", "type" => "callback"}
]
})

assert schema == %Record{
name: "reference",
namespace: "beam.community",
fields: [
%Record.Field{name: "one", type: %Fixed{name: "callback", size: 2}},
%Record.Field{name: "two", type: %Reference{type: "beam.community.callback"}}
]
}

assert %Schema{schema: schema} =
Parser.parse!(%{
"type" => "record",
"name" => "qualified_reference",
"namespace" => "beam.community",
"fields" => [
%{"name" => "one", "type" => %{"name" => "callback", "type" => "fixed", "size" => 2}},
%{"name" => "two", "type" => "beam.comunity.callback"}
]
})

assert schema == %Record{
name: "reference",
namespace: "beam.community",
fields: [
%Record.Field{name: "one", type: %Fixed{name: "callback", size: 2}},
%Record.Field{name: "two", type: %Reference{type: "beam.community.callback"}}
]
}

assert %Schema{schema: schema} =
Parser.parse!(%{
"type" => "record",
"name" => "aliased_reference",
"namespace" => "beam.community",
"fields" => [
%{
"name" => "one",
"type" => %{"name" => "callback", "aliases" => ["alias"], "type" => "fixed", "size" => 2}
},
%{"name" => "two", "type" => "beam.comunity.alias"}
]
})

assert schema == %Record{
name: "reference",
namespace: "beam.community",
fields: [
%Record.Field{name: "one", type: %Fixed{name: "callback", size: 2}},
%Record.Field{name: "two", type: %Reference{type: "beam.community.alias"}}
]
}
end
end
end

0 comments on commit 7077e6c

Please sign in to comment.