/
test_datafile.rb
121 lines (109 loc) · 4.12 KB
/
test_datafile.rb
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership. The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
require 'test_help'
class TestDataFile < Test::Unit::TestCase
HERE = File.expand_path File.dirname(__FILE__)
def setup
if File.exists?(HERE + '/data.avr')
File.unlink(HERE + '/data.avr')
end
end
def teardown
if File.exists?(HERE + '/data.avr')
File.unlink(HERE + '/data.avr')
end
end
def test_differing_schemas_with_primitives
writer_schema = <<-JSON
{ "type": "record",
"name": "User",
"fields" : [
{"name": "username", "type": "string"},
{"name": "age", "type": "int"},
{"name": "verified", "type": "boolean", "default": "false"}
]}
JSON
data = [{"username" => "john", "age" => 25, "verified" => true},
{"username" => "ryan", "age" => 23, "verified" => false}]
Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
data.each{|h| dw << h }
end
# extract the username only from the avro serialized file
reader_schema = <<-JSON
{ "type": "record",
"name": "User",
"fields" : [
{"name": "username", "type": "string"}
]}
JSON
Avro::DataFile.open('data.avr', 'r', reader_schema) do |dr|
dr.each_with_index do |record, i|
assert_equal data[i]['username'], record['username']
end
end
end
def test_differing_schemas_with_complex_objects
writer_schema = <<-JSON
{ "type": "record",
"name": "something",
"fields": [
{"name": "something_fixed", "type": {"name": "inner_fixed",
"type": "fixed", "size": 3}},
{"name": "something_enum", "type": {"name": "inner_enum",
"type": "enum",
"symbols": ["hello", "goodbye"]}},
{"name": "something_array", "type": {"type": "array", "items": "int"}},
{"name": "something_map", "type": {"type": "map", "values": "int"}},
{"name": "something_record", "type": {"name": "inner_record",
"type": "record",
"fields": [
{"name": "inner", "type": "int"}
]}},
{"name": "username", "type": "string"}
]}
JSON
data = [{"username" => "john",
"something_fixed" => "foo",
"something_enum" => "hello",
"something_array" => [1,2,3],
"something_map" => {"a" => 1, "b" => 2},
"something_record" => {"inner" => 2},
"something_error" => {"code" => 403}
},
{"username" => "ryan",
"something_fixed" => "bar",
"something_enum" => "goodbye",
"something_array" => [1,2,3],
"something_map" => {"a" => 2, "b" => 6},
"something_record" => {"inner" => 1},
"something_error" => {"code" => 401}
}]
Avro::DataFile.open('data.avr', 'w', writer_schema) do |dw|
data.each{|d| dw << d }
end
%w[fixed enum record error array map union].each do |s|
reader = Yajl.load(writer_schema)
reader['fields'] = reader['fields'].reject{|f| f['type']['type'] == s}
Avro::DataFile.open('data.avr', 'r', Yajl.dump(reader)) do |dr|
dr.each_with_index do |obj, i|
reader['fields'].each do |field|
assert_equal data[i][field['name']], obj[field['name']]
end
end
end
end
end
end