-
Notifications
You must be signed in to change notification settings - Fork 50
/
P.py
420 lines (370 loc) · 15.6 KB
/
P.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
#!/usr/bin/env python
# Copyright (C) 2014 Open Data ("Open Data" refers to
# one or more of the following companies: Open Data Partners LLC,
# Open Data Research LLC, or Open Data Capital LLC.)
#
# This file is part of Hadrian.
#
# Licensed under the Hadrian Personal Use and Evaluation License (PUEL);
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://raw.githubusercontent.com/opendatagroup/hadrian/master/LICENSE
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from titus.datatype import Type
from titus.datatype import FcnType
from titus.datatype import ExceptionType
from titus.datatype import AvroType
from titus.datatype import AvroCompiled
from titus.datatype import AvroNull
from titus.datatype import AvroBoolean
from titus.datatype import AvroInt
from titus.datatype import AvroLong
from titus.datatype import AvroFloat
from titus.datatype import AvroDouble
from titus.datatype import AvroBytes
from titus.datatype import AvroFixed
from titus.datatype import AvroString
from titus.datatype import AvroEnum
from titus.datatype import AvroArray
from titus.datatype import AvroMap
from titus.datatype import AvroRecord
from titus.datatype import AvroField
from titus.datatype import AvroUnion
class Pattern(object):
"""Trait for a type pattern.
A type pattern is something that a titus.datatype.AvroType is matched against when determining if a PFA function signature can be applied to a given set of arguments.
It could be as simple as the type itself (e.g. P.Int() matches AvroInt()) or it could be a complex wildcard.
"""
pass
class Null(Pattern):
"""Matches titus.datatype.AvroNull."""
def __repr__(self):
return "P.Null()"
class Boolean(Pattern):
"""Matches titus.datatype.AvroBoolean."""
def __repr__(self):
return "P.Boolean()"
class Int(Pattern):
"""Matches titus.datatype.AvroInt."""
def __repr__(self):
return "P.Int()"
class Long(Pattern):
"""Matches titus.datatype.AvroLong."""
def __repr__(self):
return "P.Long()"
class Float(Pattern):
"""Matches titus.datatype.AvroFloat."""
def __repr__(self):
return "P.Float()"
class Double(Pattern):
"""Matches titus.datatype.AvroDouble."""
def __repr__(self):
return "P.Double()"
class Bytes(Pattern):
"""Matches titus.datatype.AvroBytes."""
def __repr__(self):
return "P.Bytes()"
class String(Pattern):
"""Matches titus.datatype.AvroString."""
def __repr__(self):
return "P.String()"
class Array(Pattern):
"""Matches titus.datatype.AvroArray with a given items pattern."""
def __init__(self, items):
""":type items: titus.P
:param items: pattern for the array items
"""
self._items = items
@property
def items(self):
return self._items
def __repr__(self):
return "P.Array(" + repr(self.items) + ")"
class Map(Pattern):
"""Matches titus.datatype.AvroMap with a given values pattern."""
def __init__(self, values):
""":type values: titus.P
:param values: pattern for the map values
"""
self._values = values
@property
def values(self):
return self._values
def __repr__(self):
return "P.Map(" + repr(self.values) + ")"
class Union(Pattern):
"""Matches titus.datatype.AvroUnion with a given set of sub-patterns."""
def __init__(self, types):
""":type types: list of titus.P
:param types: patterns for the subtypes
"""
self._types = types
@property
def types(self):
return self._types
def __repr__(self):
return "P.Union(" + repr(self.types) + ")"
class Fixed(Pattern):
"""Matches titus.datatype.AvroFixed with a given size and an optional name.
Note: not used for any PFA patterns, and probably shouldn't be, even in the future. titus.P.WildFixed is used instead.
"""
def __init__(self, size, fullName=None):
""":type size: positive integer
:param size: width of the fixed-width byte array
:type fullName: string or ``None``
:param fullName: optional name of the fixed pattern (if not provided, fixed types of any name would match)
"""
self._size = size
self._fullName = fullName
@property
def size(self):
return self._size
@property
def fullName(self):
return self._fullName
def __repr__(self):
return "P.Fixed(" + repr(self.size) + ", " + repr(self.fullName) + ")"
class Enum(Pattern):
"""Matches titus.datatype.AvroEnum with a given symbols and an optional name.
Note: not used for any PFA patterns, and probably shouldn't be, even in the future. titus.P.WildEnum is used instead.
"""
def __init__(self, symbols, fullName=None):
""":type symbols: list of strings
:param symbols: names of the enum symbols
:type fullName: string or ``None``
:param fullName: optional name of the enum pattern (if not provided, enum types of any name would match)
"""
self._symbols = symbols
self._fullName = fullName
@property
def symbols(self):
return self._symbols
@property
def fullName(self):
return self._fullName
def __repr__(self):
return "P.Enum(" + repr(self.symbols) + ", " + repr(self.fullName) + ")"
class Record(Pattern):
"""Matches titus.datatype.AvroRecord with given fields and an optional name.
Note: not used for any PFA patterns, and probably shouldn't be, even in the future. titus.P.WildRecord is used instead.
"""
def __init__(self, fields, fullName=None):
""":type fields: dict from field names to titus.P
:param fields: patterns for the record fields
:type fullName: string or ``None``
:param fullName: optional name of the record pattern (if not provided, record types of any name would match)
"""
self._fields = fields
self._fullName = fullName
@property
def fields(self):
return self._fields
@property
def fullName(self):
return self._fullName
def __repr__(self):
return "P.Record(" + repr(self.fields) + ", " + repr(self.fullName) + ")"
class Fcn(Pattern):
"""Matches titus.datatype.FcnType with a given sequence of parameter patterns and return pattern."""
def __init__(self, params, ret):
""":type params: list of titus.P
:param params: patterns for each of the slots in the function-argument's signature (must have only one signature with no wildcards)
:type ret: titus.P
:param ret: pattern for the return type of the function-argument
"""
self._params = params
self._ret = ret
@property
def params(self):
return self._params
@property
def ret(self):
return self._ret
def __repr__(self):
return "P.Fcn(" + repr(self.params) + ", " + repr(self.ret) + ")"
class Wildcard(Pattern):
"""Matches any titus.datatype.AvroType or one of a restricted set.
Label letters are shared across a signature (e.g. if two wildcards are both labeled "A", then they both have to resolve to the same type).
"""
def __init__(self, label, oneOf=None):
""":type label: string
:param label: label letter (usually one character long, but in principle an arbitrary string)
:type oneOf: list of titus.datatype.AvroType or ``None``
:param oneOf: allowed types or ``None`` for unrestricted
"""
self._label = label
self._oneOf = oneOf
@property
def label(self):
return self._label
@property
def oneOf(self):
return self._oneOf
def __repr__(self):
return "P.Wildcard(" + repr(self.label) + ", " + repr(self.oneOf) + ")"
class WildRecord(Pattern):
"""Matches a titus.datatype.AvroRecord with *at least* the requested set of fields.
Label letters are shared across a signature (e.g. if two wildcards are both labeled "A", then they both have to resolve to the same type).
"""
def __init__(self, label, minimalFields=None):
""":type label: string
:param label: label letter (usually one character long, but in principle an arbitrary string)
:type minimalFields: dict from field name to titus.P
:param minimalFields: fields that a matching record must have and patterns for their respective types
"""
self._label = label
self._minimalFields = minimalFields
@property
def label(self):
return self._label
@property
def minimalFields(self):
return self._minimalFields
def __repr__(self):
return "P.WildRecord(" + repr(self.label) + ", " + repr(self.minimalFields) + ")"
class WildEnum(Pattern):
"""Matches a titus.datatype.AvroEnum without any constraint on the symbol names.
Label letters are shared across a signature (e.g. if two wildcards are both labeled "A", then they both have to resolve to the same type).
"""
def __init__(self, label):
""":type label: string
:param label: label letter (usually one character long, but in principle an arbitrary string)
"""
self._label = label
@property
def label(self):
return self._label
def __repr__(self):
return "P.WildEnum(" + repr(self.label) + ")"
class WildFixed(Pattern):
"""Matches a titus.datatype.AvroFixed without any constraint on the size.
Label letters are shared across a signature (e.g. if two wildcards are both labeled "A", then they both have to resolve to the same type).
"""
def __init__(self, label):
""":type label: string
:param label: label letter (usually one character long, but in principle an arbitrary string)
"""
self._label = label
@property
def label(self):
return self._label
def __repr__(self):
return "P.WildFixed(" + repr(self.label) + ")"
class EnumFields(Pattern):
"""Matches a titus.datatype.AvroEnum whose symbols match the fields of a given record
Label letters are shared across a signature (e.g. if two wildcards are both labeled "A", then they both have to resolve to the same type).
"""
def __init__(self, label, wildRecord):
""":type label: string
:param label: label letter (usually one character long, but in principle an arbitrary string)
:type wildRecord: string
:param wildRecord: label letter of the record (also a wildcard)
"""
self._label = label
self._wildRecord = wildRecord
@property
def label(self):
return self._label
@property
def wildRecord(self):
return self._wildRecord
def __repr__(self):
return "P.EnumFields(" + repr(self.label) + ", " + repr(self.wildRecord) + ")"
def toType(pat):
"""Convert a pattern to a type, if possible (wildcards can't be converted to types).
:type pat: titus.P
:param pat: pattern to convert
:rtype: titus.datatype.Type
:return: corresponding type (titus.datatype.Type rather than titus.datatype.AvroType to allow for titus.datatype.FcnType)
"""
if isinstance(pat, Null): return AvroNull()
elif isinstance(pat, Boolean): return AvroBoolean()
elif isinstance(pat, Int): return AvroInt()
elif isinstance(pat, Long): return AvroLong()
elif isinstance(pat, Float): return AvroFloat()
elif isinstance(pat, Double): return AvroDouble()
elif isinstance(pat, Bytes): return AvroBytes()
elif isinstance(pat, String): return AvroString()
elif isinstance(pat, Array): return AvroArray(toType(pat.items))
elif isinstance(pat, Map): return AvroMap(toType(pat.values))
elif isinstance(pat, Union): return AvroUnion([toType(x) for x in pat.types])
elif isinstance(pat, Fixed) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroFixed(pat.size, namebits[-1], None)
else:
return AvroFixed(pat.size, namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Fixed):
return AvroFixed(pat.size)
elif isinstance(pat, Enum) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroEnum(pat.symbols, namebits[-1], None)
else:
return AvroEnum(pat.symbols, namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Enum):
return AvroEnum(pat.symbols)
elif isinstance(pat, Record) and pat.fullName is not None:
namebits = pat.fullName.split(".")
if len(namebits) == 1:
return AvroRecord([AvroField(k, toType(v)) for k, v in pat.fields.items()], namebits[-1], None)
else:
return AvroRecord([AvroField(k, toType(v)) for k, v in pat.fields.items()], namebits[-1], ".".join(namebits[:-1]))
elif isinstance(pat, Record):
return AvroRecord([AvroField(k, toType(v)) for k, v in pat.fields.items()])
elif isinstance(pat, Fcn): return FcnType([toType(x) for x in pat.params()], toType(pat.ret()))
else: raise Exception
def fromType(t, memo=None):
"""Convert a type to a pattern.
:type t: titus.datatype.AvroType
:param t: type to convert
:rtype: titus.P
:return: corresponding pattern
"""
if memo is None:
memo = set()
if isinstance(t, AvroNull): return Null()
elif isinstance(t, AvroBoolean): return Boolean()
elif isinstance(t, AvroInt): return Int()
elif isinstance(t, AvroLong): return Long()
elif isinstance(t, AvroFloat): return Float()
elif isinstance(t, AvroDouble): return Double()
elif isinstance(t, AvroBytes): return Bytes()
elif isinstance(t, AvroString): return String()
elif isinstance(t, AvroArray): return Array(fromType(t.items, memo))
elif isinstance(t, AvroMap): return Map(fromType(t.values, memo))
elif isinstance(t, AvroUnion): return Union([fromType(x, memo) for x in t.types])
elif isinstance(t, AvroFixed) and t.namespace is not None and t.namespace != "": return Fixed(t.size, t.namespace + "." + t.name)
elif isinstance(t, AvroFixed): return Fixed(t.size, t.name)
elif isinstance(t, AvroEnum) and t.namespace is not None and t.namespace != "": return Enum(t.symbols, t.namespace + "." + t.name)
elif isinstance(t, AvroEnum): return Enum(t.symbols, t.name)
elif isinstance(t, AvroRecord) and t.namespace is not None and t.namespace != "":
name = t.namespace + "." + t.name
if name in memo:
return Record({}, name)
else:
return Record(dict((f.name, fromType(f.avroType, memo.union(set([name])))) for f in t.fields), name)
elif isinstance(t, AvroRecord):
if t.name in memo:
return Record({}, t.name)
else:
return Record(dict((f.name, fromType(f.avroType, memo.union(set([t.name])))) for f in t.fields), t.name)
elif isinstance(t, FcnType): return Fcn([fromType(x, memo) for x in t.params()], fromType(t.ret(), memo))
elif isinstance(t, ExceptionType): raise IncompatibleTypes("exception type cannot be used in argument patterns")
def mustBeAvro(t):
"""Raise a ``TypeError`` if a given titus.datatype.Type is not a titus.datatype.AvroType; otherwise, pass through.
:type t: titus.datatype.Type
:param t: type to check
:rtype: titus.datatype.AvroType
:return: the input ``t`` or raise an exception
"""
if not isinstance(t, AvroType):
raise TypeError(repr(t) + " is not an Avro type")
else:
return t