Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with
or
.
Download ZIP

Loading…

10x speedup for cowboy_rest and dependents (eg cowboy_static) #305

Closed
wants to merge 2 commits into from

4 participants

@dvv
dvv commented

httpd_util:rfc1123/1 is way slow. by introducing a faster ad hoc alternative a huge speedup is achieved.
please, consider applying.
tia, --vladimir

@essen
Owner

Yes, httpd_util dependency is to be removed, there's a ticket for that.

However, why do you replace rfc1123 by rfc2109?

Also io_lib:format is slow too.

@narma

httpd_util:rfc1123_date converts local time to universal, it's for compliance with HTTP/1.1 RFC2616 wich says:
"All HTTP date/time stamps MUST be represented in Greenwich Mean Time
(GMT), without exception. For the purposes of HTTP, GMT is exactly
equal to UTC (Coordinated Universal Time)."

3> cowboy_clock:rfc2109_fast({{2012,11,01},{17,0,0}}).
<<"Thu, 01 Nov 2012 17:00:00 GMT">>
4> httpd_util:rfc1123_date({{2012,11,01},{17,0,0}}).
"Thu, 01 Nov 2012 10:00:00 GMT"`

hence, after the patch time returning in rest handler's, last_modified and expires, is not turning from local to UTC.

So, additional changes required in cowboy_static.erl for this patch:
from
Fileinfo = file:read_file_info(Filepath1),
to
Fileinfo = file:read_file_info(Filepath1, [{time, universal}]),

I don't presume to judge which one is preferable and more reasonable.

@dvv
dvv commented

right, in my barebone static handler i use {time, universal}, so should imo do cowboy_static.
update: here is the crap\c\c\c\ccode

@dvv
dvv commented

@essen i just added _fast suffix to what cowboy_clock exposed. haven't studied diff between both RFCs.

@essen
Owner

Ultimately we only want to deal with universal time, so this is definitely in the right direction, however the current rfc2109 function should be converted to this, it's only doing local time at the moment because cookies needed it, but cookies are a temporary implementation so the local code can be moved in the cookie module without any issue for now.

@si14

Can I suggest our lib for time formatting/parsing? It's quite battle tested and fast as hell: https://github.com/selectel/tempo/

@essen
Owner

No NIFs, sorry.

@essen
Owner

By the way this depends on #282. First fix the localtime that should always be UTC, then we can fix the parsing here properly.

@si14

Why don't use such small&pure NIFs like that one?

@essen
Owner

Because they mess up with the reduction counts, may crash, are harder to upgrade properly, etc. It's also not going to be faster than code written specifically for a datetime format, which only uses a single binary construct.

@essen
Owner

Done in 8bc6bde. Using the existing rfc1123 code from Cowboy. Dependency on httpd_util is gone!

Thanks for the initial patch and report!

@essen essen closed this
@essen
Owner

Oh and this didn't need to depend on #282. I was just misled. :)

Sign up for free to join this conversation on GitHub. Already have an account? Sign in to comment
This page is out of date. Refresh to see the latest.
View
17 src/cowboy_clock.erl
@@ -25,7 +25,7 @@
-export([start_link/0]).
-export([stop/0]).
-export([rfc1123/0]).
--export([rfc2109/1]).
+-export([rfc2109/1, rfc2109_fast/1]).
%% gen_server.
-export([init/1]).
@@ -106,6 +106,21 @@ rfc2109(LocalTime) ->
MinBin/binary, ":",
SecBin/binary, " GMT">>.
+%% @doc Return the current date and time formatted according to RFC-2109.
+%%
+%% This format is used in the <em>set-cookie</em> header sent with
+%% HTTP responses.
+-spec rfc2109_fast(calendar:datetime()) -> binary().
+rfc2109_fast(DateTime) ->
+ {{Year, Month, Day}, {Hour, Minute, Second}} = DateTime,
+ DoW = calendar:day_of_the_week(Year, Month, Day),
+ list_to_binary(io_lib:format(
+ "~s, ~2..0B ~s ~4..0B ~2..0B:~2..0B:~2..0B GMT", [
+ weekday(DoW),
+ Day, month(Month), Year,
+ Hour, Minute, Second
+ ])).
+
%% gen_server.
%% @private
View
20 src/cowboy_req.erl
@@ -91,6 +91,7 @@
-export([set_resp_header/3]).
-export([set_resp_body/2]).
-export([set_resp_body_fun/3]).
+-export([set_resp_body_fun/5]).
-export([has_resp_header/2]).
-export([has_resp_body/1]).
-export([delete_resp_header/2]).
@@ -836,6 +837,12 @@ set_resp_body(Body, Req) ->
set_resp_body_fun(StreamLen, StreamFun, Req) ->
Req#http_req{resp_body={StreamLen, StreamFun}}.
+%% @see cowboy_req:set_resp_body_fun/3.
+-spec set_resp_body_fun(non_neg_integer(), non_neg_integer(), non_neg_integer(), resp_body_fun(), Req)
+ -> Req when Req::req().
+set_resp_body_fun(StreamStart, StreamEnd, StreamLen, StreamFun, Req) ->
+ Req#http_req{resp_body={StreamStart, StreamEnd, StreamLen, StreamFun}}.
+
%% @doc Return whether the given header has been set for the response.
-spec has_resp_header(binary(), req()) -> boolean().
has_resp_header(Name, #http_req{resp_headers=RespHeaders}) ->
@@ -888,6 +895,19 @@ reply(Status, Headers, Body, Req=#http_req{
if RespType =/= hook, Method =/= <<"HEAD">> -> BodyFun();
true -> ok
end;
+ {Start, End, Length, BodyFun} ->
+ % partial response has 206 status
+ Status1 = case Status of 200 -> 206; _ -> Status end,
+ {RespType, Req2} = response(Status1, Headers, RespHeaders, [
+ {<<"content-length">>, integer_to_list(End - Start + 1)},
+ {<<"content-range">>,
+ io_lib:format("~B-~B/~B", [Start, End, Length])},
+ {<<"date">>, cowboy_clock:rfc1123()},
+ {<<"server">>, <<"Cowboy">>}
+ |HTTP11Headers], <<>>, Req),
+ if RespType =/= hook, Method =/= <<"HEAD">> -> BodyFun(Start, End - Start + 1);
+ true -> ok
+ end;
_ ->
{_, Req2} = response(Status, Headers, RespHeaders, [
{<<"content-length">>, integer_to_list(iolist_size(Body))},
View
6 src/cowboy_rest.erl
@@ -765,7 +765,7 @@ set_resp_body(Req, State=#state{content_type_a={_Type, Fun}}) ->
LastModified when is_atom(LastModified) ->
Req4 = Req3;
LastModified ->
- LastModifiedStr = httpd_util:rfc1123_date(LastModified),
+ LastModifiedStr = cowboy_clock:rfc2109_fast(LastModified),
Req4 = cowboy_req:set_resp_header(
<<"Last-Modified">>, LastModifiedStr, Req3)
end,
@@ -778,6 +778,8 @@ set_resp_body(Req, State=#state{content_type_a={_Type, Fun}}) ->
Req7 = case Body of
{stream, Len, Fun1} ->
cowboy_req:set_resp_body_fun(Len, Fun1, Req6);
+ {stream, Start, End, Length, Fun1} ->
+ cowboy_req:set_resp_body_fun(Start, End, Length, Fun1, Req6);
_Contents ->
cowboy_req:set_resp_body(Body, Req6)
end,
@@ -810,7 +812,7 @@ set_resp_expires(Req, State) ->
Expires when is_atom(Expires) ->
{Req2, State2};
Expires ->
- ExpiresStr = httpd_util:rfc1123_date(Expires),
+ ExpiresStr = cowboy_clock:rfc2109_fast(Expires),
Req3 = cowboy_req:set_resp_header(
<<"Expires">>, ExpiresStr, Req2),
{Req3, State2}
View
18 src/cowboy_static.erl
@@ -238,7 +238,7 @@ rest_init(Req, Opts) ->
etag_fun=ETagFunction};
ok ->
Filepath1 = join_paths(Directory1, Filepath),
- Fileinfo = file:read_file_info(Filepath1),
+ Fileinfo = file:read_file_info(Filepath1, [{time, universal}]),
#state{filepath=Filepath1, fileinfo=Fileinfo, mimetypes=Mimetypes1,
etag_fun=ETagFunction}
end,
@@ -322,7 +322,21 @@ file_contents(Req, #state{filepath=Filepath,
fileinfo={ok, #file_info{size=Filesize}}}=State) ->
{ok, Transport, Socket} = cowboy_req:transport(Req),
Writefile = content_function(Transport, Socket, Filepath),
- {{stream, Filesize, Writefile}, Req, State}.
+ % if a data range requested, specify limits
+ % NB: so far we handle only one range
+ {Headers, _Req1} = cowboy_req:headers(Req),
+ Range = cowboy_util:get_ranges(
+ proplists:get_value(<<"range">>, Headers, <<>>),
+ <<"bytes">>, Filesize),
+ case Range of
+ % TODO: Start < 0 or End < Start should result in 416 Requested Range Not Satisfiable
+ [{Start, End}] when Start >= 0, End >= Start ->
+ {{stream, Start, End, Filesize, Writefile}, Req, State};
+ [{Start, End} | _T] when Start >= 0, End >= Start ->
+ {{stream, Start, End, Filesize, Writefile}, Req, State};
+ _ ->
+ {{stream, Filesize, Writefile}, Req, State}
+ end.
%% @private Return a function writing the contents of a file to a socket.
View
66 src/cowboy_util.erl
@@ -0,0 +1,66 @@
+-module(cowboy_util).
+
+-export([get_ranges/2, get_ranges/3]).
+-export([parse_range_header/1, parse_range_header/2]).
+
+get_ranges(Header, Name) ->
+ get_ranges(Header, Name, 0).
+
+get_ranges(Header, Name, Length) ->
+ NameStr = binary_to_list(Name),
+ lists:foldr(fun(E, A) ->
+ case E of
+ {NameStr, bad} ->
+ [bad | A];
+ {NameStr, Start, End} ->
+ [{Start, End} | A];
+ _ ->
+ A
+ end
+ end, [], parse_range_header(Header, Length)).
+
+parse_range_header(Header) ->
+ parse_range_header(Header, 0).
+
+parse_range_header(Header, Length) when is_binary(Header), is_integer(Length) ->
+ MayBeRanges = string:tokens(binary_to_list(Header), "\s;"),
+ Ranges = lists:map(fun(E) ->
+ case re:run(E, "^([a-z]+)=([0-9-]+)$", [{capture, all, list}]) of
+ {match, [_, Name, Value]} ->
+ case parse_range(Value) of
+ bad ->
+ {Name, bad};
+ {Start, End} ->
+ Start1 = wrap_negative(Start, Length),
+ End1 = wrap_negative(End, Length),
+ {Name, Start1, End1}
+ end;
+ _ -> bad
+ end
+ end, MayBeRanges),
+ Ranges;
+
+parse_range_header(undefined, _) ->
+ undefined.
+
+% X-Y
+parse_range(Data) ->
+ case string:to_integer(Data) of
+ {error, _} ->
+ bad;
+ {Start, ""} ->
+ {Start, Start};
+ {Start, "-"} ->
+ {Start, -1};
+ {Start, "-" ++ Data1} ->
+ case string:to_integer(Data1) of
+ {End, ""} ->
+ {Start, End};
+ _ ->
+ bad
+ end
+ end.
+
+% X < 0 ? X + Y : X
+wrap_negative(X, L) when X < 0 -> X + L;
+wrap_negative(X, _L) -> X.
Something went wrong with that request. Please try again.