-
Notifications
You must be signed in to change notification settings - Fork 574
/
parsers.ex
387 lines (303 loc) · 12.8 KB
/
parsers.ex
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
defmodule Plug.Parsers do
defmodule RequestTooLargeError do
@moduledoc """
Error raised when the request is too large.
"""
defexception message:
"the request is too large. If you are willing to process " <>
"larger requests, please give a :length to Plug.Parsers",
plug_status: 413
end
defmodule UnsupportedMediaTypeError do
@moduledoc """
Error raised when the request body cannot be parsed.
"""
defexception media_type: nil, plug_status: 415
def message(exception) do
"unsupported media type #{exception.media_type}"
end
end
defmodule BadEncodingError do
@moduledoc """
Raised when the request body contains bad encoding.
"""
defexception message: nil, plug_status: 400
end
defmodule ParseError do
@moduledoc """
Error raised when the request body is malformed.
"""
defexception exception: nil, plug_status: 400
def message(%{exception: exception}) do
"malformed request, a #{inspect(exception.__struct__)} exception was raised " <>
"with message #{inspect(Exception.message(exception))}"
end
end
@moduledoc """
A plug for parsing the request body.
It invokes a list of `:parsers`, which are activated based on the
request content-type. Custom parsers are also supported by defining
a module that implements the behaviour defined by this module.
Once a connection goes through this plug, it will have `:body_params`
set to the map of params parsed by one of the parsers listed in
`:parsers` and `:params` set to the result of merging the `:body_params`
and `:query_params`. In case `:query_params` have not yet been parsed,
`Plug.Conn.fetch_query_params/2` is automatically invoked.
This plug will raise `Plug.Parsers.UnsupportedMediaTypeError` by default if
the request cannot be parsed by any of the given types and the MIME type has
not been explicitly accepted with the `:pass` option.
`Plug.Parsers.RequestTooLargeError` will be raised if the request goes over
the given limit. The default length is 8MB and it can be customized by passing
the `:length` option to the Plug. `:read_timeout` and `:read_length`, as
described by `Plug.Conn.read_body/2`, are also supported.
Parsers may raise a `Plug.Parsers.ParseError` if the request has a malformed
body.
This plug only parses the body if the request method is one of the following:
* `POST`
* `PUT`
* `PATCH`
* `DELETE`
For requests with a different request method, this plug will only fetch the
query params.
## Options
* `:parsers` - a list of modules or atoms of built-in parsers to be
invoked for parsing. These modules need to implement the behaviour
outlined in this module.
* `:pass` - an optional list of MIME type strings that are allowed
to pass through. Any mime not handled by a parser and not explicitly
listed in `:pass` will `raise UnsupportedMediaTypeError`. For example:
* `["*/*"]` - never raises
* `["text/html", "application/*"]` - doesn't raise for those values
* `[]` - always raises (default)
* `:query_string_length` - the maximum allowed size for query strings
* `:validate_utf8` - boolean that tells whether or not we want to
validate that parsed binaries are utf8 strings.
* `:body_reader` - an optional replacement (or wrapper) for
`Plug.Conn.read_body/2` to provide a function that gives access to the
raw body before it is parsed and discarded. It is in the standard format
of `{Module, :function, [args]}` (MFA) and defaults to
`{Plug.Conn, :read_body, []}`. Note that this option is not used by
`Plug.Parsers.MULTIPART` which relies instead on other functions defined
in `Plug.Conn`.
All other options given to this Plug are forwarded to the parsers.
## Examples
plug Plug.Parsers,
parsers: [:urlencoded, :multipart],
pass: ["text/*"]
Any other option given to Plug.Parsers is forwarded to the underlying
parsers. Therefore, you can use a JSON parser and pass the `:json_decoder`
option at the root:
plug Plug.Parsers,
parsers: [:urlencoded, :json],
json_decoder: Jason
Or directly to the parser itself:
plug Plug.Parsers,
parsers: [:urlencoded, {:json, json_decoder: Jason}]
It is also possible to pass the `:json_decoder` as a `{module, function, args}` tuple,
useful for passing options to the JSON decoder:
plug Plug.Parsers,
parsers: [:json],
json_decoder: {Jason, :decode!, [[floats: :decimals]]}
A common set of shared options given to Plug.Parsers is `:length`,
`:read_length` and `:read_timeout`, which customizes the maximum
request length you want to accept. For example, to support file
uploads, you can do:
plug Plug.Parsers,
parsers: [:urlencoded, :multipart],
length: 20_000_000
However, the above will increase the maximum length of all request
types. If you want to increase the limit only for multipart requests
(which is typically the ones used for file uploads), you can do:
plug Plug.Parsers,
parsers: [
:urlencoded,
{:multipart, length: 20_000_000} # Increase to 20MB max upload
]
## Built-in parsers
Plug ships with the following parsers:
* `Plug.Parsers.URLENCODED` - parses `application/x-www-form-urlencoded`
requests (can be used as `:urlencoded` as well in the `:parsers` option)
* `Plug.Parsers.MULTIPART` - parses `multipart/form-data` and
`multipart/mixed` requests (can be used as `:multipart` as well in the
`:parsers` option)
* `Plug.Parsers.JSON` - parses `application/json` requests with the given
`:json_decoder` (can be used as `:json` as well in the `:parsers` option)
## File handling
If a file is uploaded via any of the parsers, Plug will
stream the uploaded contents to a file in a temporary directory in order to
avoid loading the whole file into memory. For such, the `:plug` application
needs to be started in order for file uploads to work. More details on how the
uploaded file is handled can be found in the documentation for `Plug.Upload`.
When a file is uploaded, the request parameter that identifies that file will
be a `Plug.Upload` struct with information about the uploaded file (e.g.
filename and content type) and about where the file is stored.
The temporary directory where files are streamed to can be customized by
setting the `PLUG_TMPDIR` environment variable on the host system. If
`PLUG_TMPDIR` isn't set, Plug will look at some environment
variables which usually hold the value of the system's temporary directory
(like `TMPDIR` or `TMP`). If no value is found in any of those variables,
`/tmp` is used as a default.
## Custom body reader
Sometimes you may want to customize how a parser reads the body from the
connection. For example, you may want to cache the body to perform verification
later, such as HTTP Signature Verification. This can be achieved with a custom
body reader that would read the body and store it in the connection, such as:
defmodule CacheBodyReader do
def read_body(conn, opts) do
{:ok, body, conn} = Plug.Conn.read_body(conn, opts)
conn = update_in(conn.assigns[:raw_body], &[body | (&1 || [])])
{:ok, body, conn}
end
end
which could then be set as:
plug Plug.Parsers,
parsers: [:urlencoded, :json],
pass: ["text/*"],
body_reader: {CacheBodyReader, :read_body, []},
json_decoder: Jason
"""
alias Plug.Conn
@callback init(opts :: Keyword.t()) :: Plug.opts()
@doc """
Attempts to parse the connection's request body given the content-type type,
subtype, and its parameters.
The arguments are:
* the `Plug.Conn` connection
* `type`, the content-type type (e.g., `"x-sample"` for the
`"x-sample/json"` content-type)
* `subtype`, the content-type subtype (e.g., `"json"` for the
`"x-sample/json"` content-type)
* `params`, the content-type parameters (e.g., `%{"foo" => "bar"}`
for the `"text/plain; foo=bar"` content-type)
This function should return:
* `{:ok, body_params, conn}` if the parser is able to handle the given
content-type; `body_params` should be a map
* `{:next, conn}` if the next parser should be invoked
* `{:error, :too_large, conn}` if the request goes over the given limit
"""
@callback parse(
conn :: Conn.t(),
type :: binary,
subtype :: binary,
params :: Conn.Utils.params(),
opts :: Plug.opts()
) ::
{:ok, Conn.params(), Conn.t()}
| {:error, :too_large, Conn.t()}
| {:next, Conn.t()}
@behaviour Plug
@methods ~w(POST PUT PATCH DELETE)
@impl true
def init(opts) do
{parsers, opts} = Keyword.pop(opts, :parsers)
{pass, opts} = Keyword.pop(opts, :pass, [])
{query_string_length, opts} = Keyword.pop(opts, :query_string_length, 1_000_000)
validate_utf8 = Keyword.get(opts, :validate_utf8, true)
unless parsers do
raise ArgumentError, "Plug.Parsers expects a set of parsers to be given in :parsers"
end
{convert_parsers(parsers, opts), pass, query_string_length, validate_utf8}
end
defp convert_parsers(parsers, root_opts) do
for parser <- parsers do
{parser, opts} =
case parser do
{parser, opts} when is_atom(parser) and is_list(opts) ->
{parser, Keyword.merge(root_opts, opts)}
parser when is_atom(parser) ->
{parser, root_opts}
end
module =
case Atom.to_string(parser) do
"Elixir." <> _ -> parser
reference -> Module.concat(Plug.Parsers, String.upcase(reference))
end
{module, module.init(opts)}
end
end
@impl true
def call(%{method: method, body_params: %Plug.Conn.Unfetched{}} = conn, options)
when method in @methods do
{parsers, pass, query_string_length, validate_utf8} = options
%{req_headers: req_headers} = conn
conn =
Conn.fetch_query_params(conn,
length: query_string_length,
validate_utf8: validate_utf8
)
case List.keyfind(req_headers, "content-type", 0) do
{"content-type", ct} ->
case Conn.Utils.content_type(ct) do
{:ok, type, subtype, params} ->
reduce(
conn,
parsers,
type,
subtype,
params,
pass,
query_string_length,
validate_utf8
)
:error ->
reduce(conn, parsers, ct, "", %{}, pass, query_string_length, validate_utf8)
end
_ ->
{conn, params} = merge_params(conn, %{}, query_string_length, validate_utf8)
%{conn | params: params, body_params: %{}}
end
end
def call(%{body_params: body_params} = conn, {_, _, query_string_length, validate_utf8}) do
body_params = make_empty_if_unfetched(body_params)
{conn, params} = merge_params(conn, body_params, query_string_length, validate_utf8)
%{conn | params: params, body_params: body_params}
end
defp reduce(
conn,
[{parser, options} | rest],
type,
subtype,
params,
pass,
query_string_length,
validate_utf8
) do
case parser.parse(conn, type, subtype, params, options) do
{:ok, body, conn} ->
{conn, params} = merge_params(conn, body, query_string_length, validate_utf8)
%{conn | params: params, body_params: body}
{:next, conn} ->
reduce(conn, rest, type, subtype, params, pass, query_string_length, validate_utf8)
{:error, :too_large, _conn} ->
raise RequestTooLargeError
end
end
defp reduce(conn, [], type, subtype, _params, pass, query_string_length, validate_utf8) do
if accepted_mime?(type, subtype, pass) do
{conn, params} = merge_params(conn, %{}, query_string_length, validate_utf8)
%{conn | params: params}
else
raise UnsupportedMediaTypeError, media_type: "#{type}/#{subtype}"
end
end
defp accepted_mime?(_type, _subtype, ["*/*"]),
do: true
defp accepted_mime?(type, subtype, pass),
do: "#{type}/#{subtype}" in pass || "#{type}/*" in pass
defp merge_params(conn, body_params, query_string_length, validate_utf8) do
%{params: params, path_params: path_params} = conn
params = make_empty_if_unfetched(params)
conn =
Plug.Conn.fetch_query_params(conn,
length: query_string_length,
validate_utf8: validate_utf8
)
{conn,
conn.query_params
|> Map.merge(params)
|> Map.merge(body_params)
|> Map.merge(path_params)}
end
defp make_empty_if_unfetched(%Plug.Conn.Unfetched{}), do: %{}
defp make_empty_if_unfetched(params), do: params
end