/
from_json.cr
511 lines (464 loc) · 12.9 KB
/
from_json.cr
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
# Deserializes the given JSON in *string_or_io* into
# an instance of `self`. This simply creates a `parser = JSON::PullParser`
# and invokes `new(parser)`: classes that want to provide JSON
# deserialization must provide an `def initialize(parser : JSON::PullParser)`
# method.
#
# ```
# Int32.from_json("1") # => 1
# Array(Int32).from_json("[1, 2, 3]") # => [1, 2, 3]
# ```
def Object.from_json(string_or_io)
parser = JSON::PullParser.new(string_or_io)
new parser
end
# Deserializes the given JSON in *string_or_io* into
# an instance of `self`, assuming the JSON consists
# of an JSON object with key *root*, and whose value is
# the value to deserialize.
#
# ```
# Int32.from_json(%({"main": 1}), root: "main") # => 1
# ```
def Object.from_json(string_or_io, root : String)
parser = JSON::PullParser.new(string_or_io)
parser.on_key!(root) do
new parser
end
end
# Parses a `String` or `IO` denoting a JSON array, yielding
# each of its elements to the given block. This is useful
# for decoding an array and processing its elements without
# creating an Array in memory, which might be expensive.
#
# ```
# require "json"
#
# Array(Int32).from_json("[1, 2, 3]") do |element|
# puts element
# end
# ```
#
# Output:
#
# ```text
# 1
# 2
# 3
# ```
#
# To parse and get an `Array`, use the block-less overload.
def Array.from_json(string_or_io, &) : Nil
parser = JSON::PullParser.new(string_or_io)
new(parser) do |element|
yield element
end
nil
end
def Deque.from_json(string_or_io, &) : Nil
parser = JSON::PullParser.new(string_or_io)
new(parser) do |element|
yield element
end
end
module Iterator(T)
# Reads the content of a JSON array into an iterator in a lazy way.
# With this method it should be possible to process a huge JSON array, without
# the requirement that the whole array fits into memory.
#
# The following example produces a huge file, uses a lot of CPU but should not require much memory.
#
# ```
# File.open("/tmp/test.json", "w+") do |f|
# (0..1_000_000_000).each.to_json(f)
# end
#
# File.open("/tmp/test.json", "r") do |f|
# p Iterator(Int32).from_json(f).skip(1_000_000_000).to_a
# end
# ```
#
# WARNING: The `string_or_io` can't be used by anything else until the iterator is fully consumed.
def self.from_json(string_or_io)
Iterator(T).new(JSON::PullParser.new(string_or_io))
end
# Creates a new iterator which iterates over a JSON array. See also `Iterator#from_json`.
#
# WARNING: The `JSON::PullParser` can't be used by anything else until the iterator is fully consumed.
def self.new(pull : JSON::PullParser)
FromJson(T).new(pull)
end
private class FromJson(T)
include Iterator(T)
def initialize(@pull : JSON::PullParser)
@pull.read_begin_array
@end = false
end
def next
if @end
stop
elsif @pull.kind.end_array?
@pull.read_next
@end = true
stop
else
T.new(@pull)
end
end
end
end
def Nil.new(pull : JSON::PullParser)
pull.read_null
end
def Bool.new(pull : JSON::PullParser)
pull.read_bool
end
{% for type, method in {
"Int8" => "i8",
"Int16" => "i16",
"Int32" => "i32",
"Int64" => "i64",
"Int128" => "i128",
"UInt8" => "u8",
"UInt16" => "u16",
"UInt32" => "u32",
"UInt64" => "u64",
"UInt128" => "u128",
} %}
def {{type.id}}.new(pull : JSON::PullParser)
location = pull.location
value =
{% if type == "UInt64" || type == "UInt128" || type == "Int128" %}
pull.read_raw
{% else %}
pull.read_int
{% end %}
begin
value.to_{{method.id}}
rescue ex : OverflowError | ArgumentError
raise JSON::ParseException.new("Can't read {{type.id}}", *location, ex)
end
end
def {{type.id}}.from_json_object_key?(key : String)
key.to_{{method.id}}?
end
{% end %}
def Float32.new(pull : JSON::PullParser)
case pull.kind
when .int?
value = pull.int_value.to_f32
pull.read_next
value
else
pull.read_float.to_f32
end
end
def Float32.from_json_object_key?(key : String) : Float32?
key.to_f32?
end
def Float64.new(pull : JSON::PullParser)
case pull.kind
when .int?
value = pull.int_value.to_f
pull.read_next
value
else
pull.read_float.to_f
end
end
def Float64.from_json_object_key?(key : String) : Float64?
key.to_f64?
end
def String.new(pull : JSON::PullParser)
pull.read_string
end
def Path.new(pull : JSON::PullParser)
new(pull.read_string)
end
def String.from_json_object_key?(key : String) : String
key
end
def Array.new(pull : JSON::PullParser)
ary = new
new(pull) do |element|
ary << element
end
ary
end
def Array.new(pull : JSON::PullParser, &)
pull.read_array do
yield T.new(pull)
end
end
def Deque.new(pull : JSON::PullParser)
ary = new
new(pull) do |element|
ary << element
end
ary
end
def Deque.new(pull : JSON::PullParser, &)
pull.read_array do
yield T.new(pull)
end
end
def Set.new(pull : JSON::PullParser)
set = new
pull.read_array do
set << T.new(pull)
end
set
end
# Reads a Hash from the given pull parser.
#
# Keys are read by invoking `from_json_object_key?` on this hash's
# key type (`K`), which must return a value of type `K` or `nil`.
# If `nil` is returned a `JSON::ParseException` is raised.
#
# Values are parsed using the regular `new(pull : JSON::PullParser)` method.
def Hash.new(pull : JSON::PullParser)
hash = new
pull.read_object do |key, key_location|
parsed_key = K.from_json_object_key?(key)
unless parsed_key
raise JSON::ParseException.new("Can't convert #{key.inspect} into #{K}", *key_location)
end
hash[parsed_key] = V.new(pull)
end
hash
end
def Tuple.new(pull : JSON::PullParser)
{% begin %}
pull.read_begin_array
value = Tuple.new(
{% for i in 0...T.size %}
(self[{{i}}].new(pull)),
{% end %}
)
pull.read_end_array
value
{% end %}
end
def NamedTuple.new(pull : JSON::PullParser)
{% begin %}
{% for key, type in T %}
{% if type.nilable? %}
%var{key.id} = nil
{% else %}
%var{key.id} = uninitialized typeof(element_type({{ key.symbolize }}))
%found{key.id} = false
{% end %}
{% end %}
location = pull.location
pull.read_object do |key|
case key
{% for key, type in T %}
when {{key.stringify}}
%var{key.id} = self[{{ key.symbolize }}].new(pull)
{% unless type.nilable? %}
%found{key.id} = true
{% end %}
{% end %}
else
pull.skip
end
end
{% for key, type in T %}
{% unless type.nilable? %}
unless %found{key.id}
raise JSON::ParseException.new("Missing json attribute: #{ {{ key.id.stringify }} }", *location)
end
{% end %}
{% end %}
NamedTuple.new(
{% for key in T.keys %}
{{ key.id.stringify }}: %var{key.id},
{% end %}
)
{% end %}
end
# Reads a serialized enum member by name from *pull*.
#
# See `#to_json` for reference.
#
# Raises `JSON::ParseException` if the deserialization fails.
def Enum.new(pull : JSON::PullParser)
{% if @type.annotation(Flags) %}
value = {{ @type }}::None
pull.read_array do
value |= parse?(pull.read_string) || pull.raise "Unknown enum #{self} value: #{pull.string_value.inspect}"
end
value
{% else %}
parse?(pull.read_string) || pull.raise "Unknown enum #{self} value: #{pull.string_value.inspect}"
{% end %}
end
# Converter for value-based serialization and deserialization of enum type `T`.
#
# The serialization format of `Enum#to_json` and `Enum.from_json` is based on
# the member name. This converter offers an alternative based on the member value.
#
# This converter can be used for its standalone serialization methods as a
# replacement of the default strategy of `Enum`. It also works as a serialization
# converter with `JSON::Field` and `YAML::Field`
#
# ```
# require "json"
# require "yaml"
#
# enum MyEnum
# ONE = 1
# TWO = 2
# end
#
# class Foo
# include JSON::Serializable
# include YAML::Serializable
#
# @[JSON::Field(converter: Enum::ValueConverter(MyEnum))]
# @[YAML::Field(converter: Enum::ValueConverter(MyEnum))]
# property foo : MyEnum = MyEnum::ONE
#
# def initialize(@foo)
# end
# end
#
# foo = Foo.new(MyEnum::ONE)
# foo.to_json # => %({"foo":1})
# foo.to_yaml # => %(---\nfoo: 1\n)
# ```
#
# NOTE: Automatically assigned enum values are subject to change when the order
# of members by adding, removing or reordering them. This can affect the integrity
# of serialized data between two instances of a program based on different code
# versions. A way to avoid this is to explicitly assign fixed values to enum
# members.
module Enum::ValueConverter(T)
def self.new(pull : JSON::PullParser) : T
from_json(pull)
end
# Reads a serialized enum member by value from *pull*.
#
# See `.to_json` for reference.
#
# Raises `JSON::ParseException` if the deserialization fails.
def self.from_json(pull : JSON::PullParser) : T
T.from_value?(pull.read_int) || pull.raise "Unknown enum #{T} value: #{pull.int_value}"
end
end
def Union.new(pull : JSON::PullParser)
location = pull.location
{% begin %}
case pull.kind
{% if T.includes? Nil %}
when .null?
return pull.read_null
{% end %}
{% if T.includes? Bool %}
when .bool?
return pull.read_bool
{% end %}
{% if T.includes? String %}
when .string?
return pull.read_string
{% end %}
when .int?
{% type_order = [Int64, UInt64, Int32, UInt32, Int16, UInt16, Int8, UInt8, Float64, Float32] %}
{% for type in type_order.select { |t| T.includes? t } %}
value = pull.read?({{type}})
return value unless value.nil?
{% end %}
when .float?
{% type_order = [Float64, Float32] %}
{% for type in type_order.select { |t| T.includes? t } %}
value = pull.read?({{type}})
return value unless value.nil?
{% end %}
else
# no priority type
end
{% end %}
{% begin %}
{% primitive_types = [Nil, Bool, String] + Number::Primitive.union_types %}
{% non_primitives = T.reject { |t| primitive_types.includes? t } %}
# If after traversing all the types we are left with just one
# non-primitive type, we can parse it directly (no need to use `read_raw`)
{% if non_primitives.size == 1 %}
return {{non_primitives[0]}}.new(pull)
{% else %}
string = pull.read_raw
{% for type in non_primitives %}
begin
return {{type}}.from_json(string)
rescue JSON::ParseException
# Ignore
end
{% end %}
raise JSON::ParseException.new("Couldn't parse #{self} from #{string}", *location)
{% end %}
{% end %}
end
# Reads a string from JSON parser as a time formatted according to [RFC 3339](https://tools.ietf.org/html/rfc3339)
# or other variations of [ISO 8601](http://xml.coverpages.org/ISO-FDIS-8601.pdf).
#
# The JSON format itself does not specify a time data type, this method just
# assumes that a string holding a ISO 8601 time format can be interpreted as a
# time value.
#
# See `#to_json` for reference.
def Time.new(pull : JSON::PullParser)
Time::Format::ISO_8601_DATE_TIME.parse(pull.read_string)
end
struct Time::Format
def from_json(pull : JSON::PullParser) : Time
string = pull.read_string
parse(string, Time::Location::UTC)
end
end
module JSON::ArrayConverter(Converter)
private struct WithInstance(T)
def from_json(pull : JSON::PullParser)
ary = Array(typeof(@converter.from_json(pull))).new
pull.read_array do
ary << @converter.from_json(pull)
end
ary
end
end
def self.from_json(pull : JSON::PullParser)
WithInstance.new(Converter).from_json(pull)
end
end
module JSON::HashValueConverter(Converter)
private struct WithInstance(T)
def from_json(pull : JSON::PullParser)
hash = Hash(String, typeof(@converter.from_json(pull))).new
pull.read_object do |key, key_location|
parsed_key = String.from_json_object_key?(key)
unless parsed_key
raise JSON::ParseException.new("Can't convert #{key.inspect} into String", *key_location)
end
hash[parsed_key] = @converter.from_json(pull)
end
hash
end
end
def self.from_json(pull : JSON::PullParser)
WithInstance.new(Converter).from_json(pull)
end
end
module Time::EpochConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix(value.read_int)
end
end
module Time::EpochMillisConverter
def self.from_json(value : JSON::PullParser) : Time
Time.unix_ms(value.read_int)
end
end
module String::RawConverter
def self.from_json(value : JSON::PullParser) : String
value.read_raw
end
end