Browse files

Add options to configure deflate compression behaviour

By adding "<deflate> ... </deflate>" structure in vhosts configuration,
it is possible to configure how deflate compression will be applied
and when it will come into effect. Now we can:

 * define the smallest response size that will be compressed
 * define the compression level to be used
 * specify the zlib compression window size
 * specify how much memory should be allocated for the internal
   compression state
 * choose the strategy used to tune the compression algorithm

All these parameters are used when a zlib stream is initialized for
compression.

It is also possible to define all compressible mime types.
Here is an example:

<server localhost>
  ...
  deflate = true
  <deflate>
    min_compress_size = 4096
    compression_level = best_compression
    mime_types        = defaults, image/*
    mime_types        = application/xml, application/xhtml+xml, application/rss+xml
    mem_level         = 9
    strategy          = default
    window_size       = 15
  </deflate>
  ...
</server>
  • Loading branch information...
1 parent 172b8b4 commit c74a92e8c893012b7edd1d35ec8bb20d5f1afdaf @capflam capflam committed Apr 25, 2012
View
4 .gitignore
@@ -26,8 +26,8 @@ test/ibrowse.tar.gz
test/support/include.mk
test/support/include.sh
test/t1/localhost:8000/
-test/t[1234]/logs/
-test/t[1234]/yaws.conf
+test/t[12345]/logs/
+test/t[12345]/yaws.conf
www/yaws.pdf
www/yaws.ps
www/*.txt
View
52 doc/yaws.tex
@@ -2480,6 +2480,58 @@ \section{Server Part}
Turns on or off deflate compression for a server. Default is
false.
+\item \verb+<deflate> ... </deflate>+ ---
+ This begins and ends the deflate compression configuration for this
+ server. The following items are allowed within a matching pair of
+ <deflate> and </deflate> delimiters.
+
+ \begin{itemize}
+ \item \verb+min_compress_size = nolimit | Integer+ --- Defines the
+ smallest response size that will be compressed. If nolimit is not
+ used, the specified value must be strictly positive. The default
+ value is nolimit.
+
+ \item \verb+compress_level = none | default | best_compression | best_speed | 0..9+ ---
+ Defines the compression level to be used. 0 (none), gives no
+ compression at all, 1 (best\_speed) gives best speed and 9
+ (best\_compression) gives best compression. The default value is
+ default.
+
+ \item \verb+window_size = 9..15+ --- Specifies the zlib compression
+ window size. It should be in the range 9 through 15. Larger
+ values of this parameter result in better compression at the
+ expense of memory usage. The default value is 15.
+
+ \item \verb+mem_level = 1..9+ --- Specifies how much memory should
+ be allocated for the internal compression
+ state. \verb+mem_level=1+ uses minimum memory but is slow and
+ reduces compression ratio; \verb+mem_level=9+ uses maximum memory
+ for optimal speed. The default value is 8.
+
+ \item \verb+strategy = default | filtered | huffman_only+ --- This
+ parameter is used to tune the compression algorithm. See
+ \verb+zlib(3erl)+ for more details on the \verb+strategy+
+ parameter. The default value is default.
+
+ \item \verb+mime_types = ListOfTypes | defaults | all+ ---
+ Restricts the deflate compression to particular mime types. The
+ special value all enable it for all types (It is a synonym of
+ `*/*'). Mime types into \verb+ListOfTypes+ must have the form
+ `type/subtype' or `type/*' (indicating all subtypes of that
+ type). Here is an example:
+\begin{verbatim}
+ mime_types = default image/*
+ mime_types = application/xml application/xhtml+xml application/rss+xml
+\end{verbatim}
+ By default, following mime types are compressed (if
+ \verb+deflate+ is set to true): \verb+text/*+,
+ \verb+application/rtf+, \verb+application/msword+,
+ \verb+application/pdf+, \verb+application/x-dvi+,
+ \verb+application/javascript+,
+ \verb+application/x-javascript+. Multiple \verb+mime_types+
+ directive can be used.
+ \end{itemize}
+
\item \verb+docroot = Directory ...+ ---
This makes the server serve all its content from
\verb+Directory+.
View
30 include/yaws.hrl
@@ -243,11 +243,11 @@
stats, % raw traffic statistics
fcgi_app_server, % FastCGI application server {host,port}
php_handler = {cgi, "/usr/bin/php-cgi"},
- shaper
+ shaper,
+ deflate_options
}).
-
%% Auth conf - from server conf and .yaws_auth
-record(auth, {
dir = [],
@@ -264,6 +264,32 @@
}).
+%% Macro used to list default compressible mime-types
+-define(DEFAULT_COMPRESSIBLE_MIME_TYPES, [
+ {"text", all},
+ {"application", "rtf"},
+ {"application", "msword"},
+ {"application", "postscript"},
+ {"application", "pdf"},
+ {"application", "x-dvi"},
+ {"application", "javascript"},
+ {"application", "x-javascript"}
+ ]).
+
+%% Internal record used to initialize a zlib stream for compression
+-record(deflate, {
+ min_compress_size = nolimit, % nolimit or non negative integer
+ % (in bytes)
+ compression_level = default, % none | default | best_compression |
+ % best_speed | 0..9
+ window_size = -15, % -15..-9
+ mem_level = 8, % 1..9
+ strategy = default, % default | filtered | huffman_only
+
+ %% [{Type, undefined|SubType}] | all
+ mime_types = ?DEFAULT_COMPRESSIBLE_MIME_TYPES
+ }).
+
%% this internal record is used and returned by the URL path parser
-record(urltype, {
View
73 man/yaws.conf.5
@@ -425,6 +425,77 @@ non-zero exit value). Disabled by default.
\fBdeflate = true | false\fR
Turns on or off deflate compression for a server. Default is \fIfalse\fR.
+
+.TP
+\fB<deflate> ... </deflate>\fR
+This begins and ends the deflate compression configuration for this server. The
+following items are allowed within a matching pair of <deflate> and </deflate>
+delimiters.
+
+\fBmin_compress_size = nolimit | Integer\fR
+.RS 12
+Defines the smallest response size that will be compressed. If nolimit is not
+used, the specified value must be strictly positive. The default value is
+\fInolimit\fR.
+.RE
+.HP
+
+\fBcompress_level = none | default | best_compression | best_speed | 0..9\fR
+.RS 12
+Defines the compression level to be used. 0 (none), gives no compression at all,
+1 (best_speed) gives best speed and 9 (best_compression) gives best
+compression. The default value is \fIdefault\fR.
+.RE
+.HP
+
+
+\fBwindow_size = 9..15\fR
+.RS 12
+Specifies the zlib compression window size. It should be in the range 9 through
+15. Larger values of this parameter result in better compression at the expense
+of memory usage. The default value is \fI15\fR.
+.RE
+.HP
+
+
+\fBmem_level = 1..9\fR
+.RS 12
+Specifies how much memory should be allocated for the internal compression
+state. \fImem_level=1\fR uses minimum memory but is slow and reduces compression
+ratio; \fImem_level=9\fR uses maximum memory for optimal speed. The default
+value is \fI8\fR.
+.RE
+.HP
+
+
+\fBstrategy = default | filtered | huffman_only\fR
+.RS 12
+This parameter is used to tune the compression algorithm. See \fBzlib(3erl)\fR
+for more details on the \fIstrategy\fR parameter. The default value is
+\fIdefault\fR.
+.RE
+.HP
+
+
+\fBmime_types = ListOfTypes | defaults | all\fR
+.RS 12
+Restricts the deflate compression to particular mime types. The special value
+\fIall\fR enable it for all types (It is a synonym of `*/*'). Mime types into
+\fIListOfTypes\fR must have the form `type/subtype' or `type/*' (indicating all
+subtypes of that type). Here is an example:
+
+.nf
+ mime_types = default image/*
+ mime_types = application/xml application/xhtml+xml application/rss+xml
+.fi
+
+By default, following mime types are compressed (if
+\fBdeflate\fR is set to true): \fItext/*, application/rtf, application/msword,
+application/pdf, application/x-dvi, application/javascript,
+application/x-javascript\fR. Multiple mime_types directive can be used.
+.RE
+.HP
+
.TP
\fBdocroot = Directory ...\fR
This makes the server serve all its content from Directory.
@@ -948,7 +1019,7 @@ Ordering is one of (Default value is \fIdeny,allow\fR):
.TP
\fBallow,deny\fR
-First, all\fIallow\fR directives are evaluated; at least one must match, or the
+First, all \fIallow\fR directives are evaluated; at least one must match, or the
request is rejected. Next, \fIdeny\fR directives are evaluated. If any matches,
the request is rejected. Last, any requests which do not match an \fIallow\fR or
a \fIdeny\fR directive are denied by default.
View
126 src/yaws_config.erl
@@ -938,7 +938,8 @@ fload(FD, server, GC, C, Cs, Lno, Chars) ->
["deflate", '=', Bool] ->
case is_bool(Bool) of
{true, Val} ->
- C2 = ?sc_set_deflate(C, Val),
+ C2 = ?sc_set_deflate(C#sconf{deflate_options=#deflate{}},
+ Val),
fload(FD, server, GC, C2, Cs, Lno+1, Next);
false ->
{error, ?F("Expect true|false at line ~w", [Lno])}
@@ -1073,6 +1074,10 @@ fload(FD, server, GC, C, Cs, Lno, Chars) ->
['<', "redirect", '>'] ->
fload(FD, server_redirect, GC, C, Cs, Lno+1, Next, []);
+ ['<', "deflate", '>'] ->
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next,
+ #deflate{mime_types=[]});
+
%% noop
["default_server_on_this_ip", '=', _Bool] ->
fload(FD, server, GC, C, Cs, Lno+1, Next);
@@ -1597,6 +1602,98 @@ fload(FD, server_redirect, GC, C, Cs, Lno, Chars, RedirMap) ->
Err
end;
+fload(FD, server_deflate, _GC, _C, _Cs, Lno, eof, _Deflate) ->
+ file:close(FD),
+ {error, ?F("Unexpected end of file at line ~w", [Lno])};
+
+fload(FD, server_deflate, GC, C, Cs, Lno, Chars, Deflate) ->
+ Next = io:get_line(FD, ''),
+ case toks(Lno, Chars) of
+ [] ->
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, Deflate);
+ ["min_compress_size", '=', CSize] ->
+ case (catch list_to_integer(CSize)) of
+ I when is_integer(I), I > 0 ->
+ D2 = Deflate#deflate{min_compress_size=I},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ _ when CSize == "nolimit" ->
+ D2 = Deflate#deflate{min_compress_size=nolimit},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ _ ->
+ {error, ?F("Expect integer > 0 at line ~w", [Lno])}
+ end;
+ ["mime_types", '=' | MimeTypes] ->
+ case parse_compressible_mime_types(MimeTypes,
+ Deflate#deflate.mime_types) of
+ {ok, L} ->
+ D2 = Deflate#deflate{mime_types=L},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ {error, Str} ->
+ {error, ?F("~s at line ~w", [Str, Lno])}
+ end;
+ ["compression_level", '=', CLevel] ->
+ L = try
+ list_to_integer(CLevel)
+ catch error:badarg ->
+ list_to_atom(CLevel)
+ end,
+ if
+ L =:= none; L =:= default;
+ L =:= best_compression; L =:= best_speed ->
+ D2 = Deflate#deflate{compression_level=L},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ is_integer(L), L >= 0, L =< 9 ->
+ D2 = Deflate#deflate{compression_level=L},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ true ->
+ {error, ?F("Bad compression level at line ~w", [Lno])}
+ end;
+ ["window_size", '=', WSize] ->
+ case (catch list_to_integer(WSize)) of
+ I when is_integer(I), I > 8, I < 16 ->
+ D2 = Deflate#deflate{window_size=I * -1},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ _ ->
+ {error,
+ ?F("Expect integer between 9..15 at line ~w",
+ [Lno])}
+ end;
+ ["mem_level", '=', MLevel] ->
+ case (catch list_to_integer(MLevel)) of
+ I when is_integer(I), I >= 1, I =< 9 ->
+ D2 = Deflate#deflate{mem_level=I},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ _ ->
+ {error, ?F("Expect integer between 1..9 at line ~w", [Lno])}
+ end;
+ ["strategy", '=', Strategy] ->
+ if
+ Strategy =:= "default";
+ Strategy =:= "filtered";
+ Strategy =:= "huffman_only" ->
+ D2 = Deflate#deflate{strategy=list_to_atom(Strategy)},
+ fload(FD, server_deflate, GC, C, Cs, Lno+1, Next, D2);
+ true ->
+ {error,
+ ?F("Unknown strategy ~p at line ~w", [Strategy, Lno])}
+ end;
+ ['<', "/deflate", '>'] ->
+ D2 = case Deflate#deflate.mime_types of
+ [] ->
+ Deflate#deflate{
+ mime_types = ?DEFAULT_COMPRESSIBLE_MIME_TYPES
+ };
+ _ ->
+ Deflate
+ end,
+ C2 = C#sconf{deflate_options = D2},
+ fload(FD, server, GC, C2, Cs, Lno+1, Next);
+ [H|T] ->
+ {error, ?F("Unexpected input ~p at line ~w", [[H|T], Lno])};
+ Err ->
+ Err
+ end;
+
fload(FD, extra_cgi_vars, _GC, _C, _Cs, Lno, eof, _EVars) ->
file:close(FD),
{error, ?F("Unexpected end of file at line ~w", [Lno])};
@@ -1733,7 +1830,7 @@ is_string_char([C|T]) ->
%% FIXME check that [C, hd(T)] really is a char ?? how
utf8;
true ->
- lists:member(C, [$., $/, $:, $_, $-, $+, $~, $@])
+ lists:member(C, [$., $/, $:, $_, $-, $+, $~, $@, $*])
end.
is_special(C) ->
@@ -1893,6 +1990,31 @@ parse_phpmod(['<', "extern", ',', NodeModFunSpec, '>'], _) ->
end.
+parse_compressible_mime_types(_, all) ->
+ {ok, all};
+parse_compressible_mime_types(["all"|_], _Acc) ->
+ {ok, all};
+parse_compressible_mime_types(["*/*"|_], _Acc) ->
+ {ok, all};
+parse_compressible_mime_types(["defaults"|Rest], Acc) ->
+ parse_compressible_mime_types(Rest, ?DEFAULT_COMPRESSIBLE_MIME_TYPES++Acc);
+parse_compressible_mime_types([',' | Rest], Acc) ->
+ parse_compressible_mime_types(Rest, Acc);
+parse_compressible_mime_types([MimeType | Rest], Acc) ->
+ Res = re:run(MimeType, "^([-\\w\+]+)/([-\\w\+\.]+|\\*)$",
+ [{capture, all_but_first, list}]),
+ case Res of
+ {match, [Type,"*"]} ->
+ parse_compressible_mime_types(Rest, [{Type, all}|Acc]);
+ {match, [Type,SubType]} ->
+ parse_compressible_mime_types(Rest, [{Type, SubType}|Acc]);
+ nomatch ->
+ {error, "Invalid MimeType"}
+ end;
+parse_compressible_mime_types([], Acc) ->
+ {ok, Acc}.
+
+
ssl_start() ->
case catch ssl:start() of
ok ->
View
144 src/yaws_server.erl
@@ -3475,36 +3475,48 @@ handle_crash(ARG, L) ->
end.
%% Ret: true | false | {data, Data}
-decide_deflate(false, _, _, _, _) ->
+decide_deflate(false, _, _, _, _, _) ->
false;
-decide_deflate(_, _, _, no, _) ->
+decide_deflate(_, _, _, _, no, _) ->
false;
-decide_deflate(true, _, _, deflate, stream) ->
+decide_deflate(true, _, _, _, deflate, stream) ->
true;
-decide_deflate(true, Arg, Data, decide, Mode) ->
- case yaws:outh_get_content_encoding_header() of
- undefined ->
+decide_deflate(true, SC, Arg, Data, decide, Mode) ->
+ Bin = to_binary(Data),
+ DOpts = SC#sconf.deflate_options,
+ CE = yaws:outh_get_content_encoding_header(),
+ if
+ size(Bin) == 0 ->
+ ?Debug("No data to be compressed~n",[]),
+ false;
+
+ DOpts#deflate.min_compress_size /= nolimit,
+ size(Bin) < DOpts#deflate.min_compress_size ->
+ ?Debug("Data too small to be compressed~n",[]),
+ false;
+
+ CE /= undefined ->
+ ?Debug("Content-Encoding already defined: ~p~n", [CE]),
+ false;
+
+ true ->
?Debug("No Content-Encoding defined~n",[]),
Mime = yaws:outh_get_content_type(),
?Debug("Mime-Type: ~p~n", [Mime]),
- case compressible_mime_type(Mime) of
+ case compressible_mime_type(Mime, DOpts) of
true ->
case yaws:accepts_gzip(Arg#arg.headers, Mime) of
- true ->
- case Mode of
- final ->
- {ok, DB} = yaws_zlib:gzip(to_binary(Data)),
- {data, DB};
- stream ->
- true
- end;
- false -> false
+ true when Mode =:= final ->
+ {ok, DB} = yaws_zlib:gzip(to_binary(Data), DOpts),
+ {data, DB};
+ true when Mode =:= stream ->
+ true;
+ false ->
+ false
end;
false ->
false
- end;
- _CE -> ?Debug("Content-Encoding: ~p~n",[_CE]),
- false
+ end
end.
@@ -3548,7 +3560,7 @@ deliver_accumulated(Arg, Sock, Encoding, ContentLength, Mode) ->
_ ->
SC = get(sc),
case decide_deflate(?sc_has_deflate(SC),
- Arg, Cont, Encoding, Mode) of
+ SC, Arg, Cont, Encoding, Mode) of
{data, Data} -> % implies Mode==final
yaws:outh_set_content_encoding(deflate),
Size = binary_size(Data),
@@ -3557,8 +3569,10 @@ deliver_accumulated(Arg, Sock, Encoding, ContentLength, Mode) ->
yaws:outh_set_content_encoding(deflate),
Z = zlib:open(),
{ok, Priv, Data} =
- yaws_zlib:gzipDeflate(Z, yaws_zlib:gzipInit(Z),
- to_binary(Cont), none),
+ yaws_zlib:gzipDeflate(
+ Z, yaws_zlib:gzipInit(Z, SC#sconf.deflate_options),
+ to_binary(Cont), none
+ ),
Ret = {Z, Priv},
Size = undefined;
false ->
@@ -3637,6 +3651,7 @@ ut_read(UT) ->
B
end;
deflate ->
+ %% FIXME: be sure that UT#urltype.deflate cannot be undefined
case UT#urltype.deflate of
B when is_binary(B) ->
?Debug("ut_read using deflated binary of size ~p~n",
@@ -3742,7 +3757,8 @@ deliver_large_file(CliSock, _Req, UT, Range) ->
end;
_ -> no
end,
- case deliver_accumulated(undefined, CliSock, Enc, undefined, stream) of
+ Sz = (UT#urltype.finfo)#file_info.size,
+ case deliver_accumulated(undefined, CliSock, Enc, Sz, stream) of
discard ->
ok;
Priv ->
@@ -3859,7 +3875,7 @@ file_changed(UT1, UT2) ->
cache_size(UT) when is_binary(UT#urltype.deflate),
-is_binary(UT#urltype.data) ->
+ is_binary(UT#urltype.data) ->
size(UT#urltype.deflate) + size(UT#urltype.data);
cache_size(UT) when is_binary(UT#urltype.data) ->
size(UT#urltype.data);
@@ -3884,15 +3900,15 @@ cache_file(SC, GC, Path, UT)
?Debug("FI=~s\n", [?format_record(FI, file_info)]),
if
N + 1 > GC#gconf.max_num_cached_files ->
- error_logger:info_msg("Max NUM cached files reached for server "
- "~p", [SC#sconf.servername]),
+ error_logger:info_msg("Max NUM cached files reached for server ~p",
+ [SC#sconf.servername]),
cleanup_cache(E, num),
cache_file(SC, GC, Path, UT);
FI#file_info.size < GC#gconf.max_size_cached_file,
FI#file_info.size < GC#gconf.max_num_cached_bytes,
B + FI#file_info.size > GC#gconf.max_num_cached_bytes ->
- error_logger:info_msg("Max size cached bytes reached for server "
- "~p", [SC#sconf.servername]),
+ error_logger:info_msg("Max size cached bytes reached for server ~p",
+ [SC#sconf.servername]),
cleanup_cache(E, size),
cache_file(SC, GC, Path, UT);
true ->
@@ -3904,31 +3920,34 @@ cache_file(SC, GC, Path, UT)
UT;
true ->
?Debug("File fits\n",[]),
- {ok, Bin} = prim_file:read_file(
- UT#urltype.fullpath),
+ {ok, Bin} = prim_file:read_file(UT#urltype.fullpath),
+ DOpts = SC#sconf.deflate_options,
Deflated =
- case ?sc_has_deflate(SC)
- and (UT#urltype.type==regular) of
+ if
+ size(Bin) == 0 ->
+ undefined;
+ DOpts#deflate.min_compress_size /= nolimit,
+ size(Bin) < DOpts#deflate.min_compress_size ->
+ undefined;
+ UT#urltype.deflate /= dynamic ->
+ undefined;
true ->
- {ok, DBL} = yaws_zlib:gzip(Bin),
+ {ok, DBL} = yaws_zlib:gzip(Bin, DOpts),
DB = list_to_binary(DBL),
if
- size(DB)*10<size(Bin)*9 ->
+ (10 * size(DB)) < (9 * size(Bin)) ->
?Debug("storing deflated version "
- "of ~p~n",
- [UT#urltype.fullpath]),
+ "of ~p~n",[UT#urltype.fullpath]),
DB;
- true -> undefined
- end;
- false -> undefined
+ true ->
+ undefined
+ end
end,
- UT2 = UT#urltype{data = Bin,
- deflate = Deflated},
+ UT2 = UT#urltype{data = Bin, deflate = Deflated},
ets:insert(E, {{url, Path}, now_secs(), UT2}),
ets:insert(E, {{urlc, Path}, 1}),
ets:update_counter(E, num_files, 1),
- ets:update_counter(E, num_bytes,
- cache_size(UT2)),
+ ets:update_counter(E, num_bytes, cache_size(UT2)),
UT2
end
end;
@@ -4018,7 +4037,7 @@ do_url_type(SC, GetPath, ArgDocroot, VirtualDir) ->
#urltype{type=Type,
finfo=FI,
deflate=deflate_q(?sc_has_deflate(SC),
- Type, Mime),
+ SC, Type, Mime),
dir = conc_path(Comps),
path = GetPath,
getpath = GetPath,
@@ -4362,7 +4381,7 @@ maybe_return_path_info(SC, Comps, RevFile, DR, VirtualDir) ->
#urltype{type = Type2,
finfo=FI,
deflate=deflate_q(?sc_has_deflate(SC),
- Type, Mime),
+ SC, Type, Mime),
dir = conc_path(HeadComps),
path = conc_path(HeadComps ++ [File]),
fullpath = FullPath,
@@ -4541,12 +4560,12 @@ parse_user_path(DR, [H|T], User) ->
parse_user_path(DR, T, [H|User]).
-deflate_q(true, regular, Mime) ->
- case compressible_mime_type(Mime) of
+deflate_q(true, SC, regular, Mime) ->
+ case compressible_mime_type(Mime, SC#sconf.deflate_options) of
true -> dynamic;
false -> undefined
end;
-deflate_q(_, _, _) ->
+deflate_q(_, _, _, _) ->
undefined.
@@ -4568,26 +4587,23 @@ suffix_type(L) ->
-%% Some silly heuristics.
+compressible_mime_type(Mime, #deflate{mime_types=MimeTypes}) ->
+ case yaws:split_sep(Mime, $/) of
+ [Type, SubType] -> compressible_mime_type(Type, SubType, MimeTypes);
+ _ -> false
+ end.
-compressible_mime_type("text/"++_) ->
- true;
-compressible_mime_type("application/rtf") ->
- true;
-compressible_mime_type("application/msword") ->
+compressible_mime_type(_, _, all) ->
true;
-compressible_mime_type("application/postscript") ->
- true;
-compressible_mime_type("application/pdf") ->
- true;
-compressible_mime_type("application/x-dvi") ->
- true;
-compressible_mime_type("application/javascript") ->
+compressible_mime_type(_, _, []) ->
+ false;
+compressible_mime_type(Type, _, [{Type, all}|_]) ->
true;
-compressible_mime_type("application/x-javascript") ->
+compressible_mime_type(Type, SubType, [{Type, SubType}|_]) ->
true;
-compressible_mime_type(_) ->
- false.
+compressible_mime_type(Type, SubType, [_|Rest]) ->
+ compressible_mime_type(Type, SubType, Rest).
+
flush(Sock, Sz, TransferEncoding) ->
View
23 src/yaws_zlib.erl
@@ -2,11 +2,20 @@
-module(yaws_zlib).
-author('carsten@codimi.de').
--export([gzipInit/1, gzipEnd/1, gzipDeflate/4, gzip/1]).
+
+-include("../include/yaws.hrl").
+
+
+-export([gzipInit/1, gzipInit/2, gzipEnd/1, gzipDeflate/4, gzip/1, gzip/2]).
gzipInit(Z) ->
- ok = zlib:deflateInit(Z, default, deflated, -15, 8, default),
+ gzipInit(Z, #deflate{}).
+
+gzipInit(Z, DOpts) ->
+ ok = zlib:deflateInit(Z, DOpts#deflate.compression_level, deflated,
+ DOpts#deflate.window_size, DOpts#deflate.mem_level,
+ DOpts#deflate.strategy),
undefined.
@@ -52,17 +61,19 @@ gzipDeflate(Z, {Crc32,Size}, Bin, Flush) ->
%% like zlib:gzip/1, but returns an io list
+gzip(Data) ->
+ gzip(Data, #deflate{}).
-gzip(Data) when is_binary(Data) ->
+gzip(Data, DOpts) when is_binary(Data) ->
Z = zlib:open(),
- {ok, _, D} = gzipDeflate(Z, gzipInit(Z), Data, finish),
+ {ok, _, D} = gzipDeflate(Z, gzipInit(Z, DOpts), Data, finish),
gzipEnd(Z),
zlib:close(Z),
{ok, D};
-gzip(Data) ->
+gzip(Data, DOpts) ->
Z = zlib:open(),
- gzip_loop(Z, gzipInit(Z), Data, [], []).
+ gzip_loop(Z, gzipInit(Z, DOpts), Data, [], []).
gzip_loop(Z, P, [], [], A) ->
{ok, _, D} = gzipDeflate(Z, P, <<>>, finish),
View
2 test/Makefile
@@ -1,6 +1,6 @@
include support/include.mk
-SUBDIRS = t1 t2 t3 t4 eunit
+SUBDIRS = t1 t2 t3 t4 t5 eunit
all: conf ibrowse
@cd src; $(MAKE) all
View
182 test/conf/deflate.conf
@@ -0,0 +1,182 @@
+
+
+logdir = ./logs
+
+# This the path to a directory where additional
+# beam code can be placed. The daemon will add this
+# directory to its search path
+
+ebin_dir = %YTOP%/test/ibrowse/ebin
+include_dir = %YTOP%/test/include
+
+
+
+# This is a debug variable, possible values are http | traffic | false
+# It is also possible to set the trace (possibly to the tty) while
+# invoking yaws from the shell as in
+# yaws -i -T -x (see man yaws)
+
+trace = false
+
+
+
+# it is possible to have yaws start additional
+# application specific code at startup
+#
+# runmod = mymodule
+
+
+# By default yaws will copy the erlang error_log and
+# end write it to a wrap log called report.log (in the logdir)
+# this feature can be turned off. This would typically
+# be the case when yaws runs within another larger app
+
+copy_error_log = true
+
+
+# Logs are wrap logs
+
+log_wrap_size = 1000000
+
+
+# Possibly resolve all hostnames in logfiles so webalizer
+# can produce the nice geography piechart
+
+log_resolve_hostname = false
+
+
+
+# fail completely or not if yaws fails
+# to bind a listen socket
+fail_on_bind_err = true
+
+
+
+# If yaws is started as root, it can, once it has opened
+# all relevant sockets for listening, change the uid to a
+# user with lower accessrights than root
+
+# username = nobody
+
+
+# If HTTP auth is used, it is possible to have a specific
+# auth log.
+# Deprecated and ignored. Now, this target must be set in server part
+#auth_log = true
+
+
+# When we're running multiple yaws systems on the same
+# host, we need to give each yaws system an individual
+# name. Yaws will write a number of runtime files under
+# /tmp/yaws/${id}
+# The default value is "default"
+
+
+# id = myname
+
+
+# earlier versions of Yaws picked the first virtual host
+# in a list of hosts with the same IP/PORT when the Host:
+# header doesn't match any name on any Host
+# This is often nice in testing environments but not
+# acceptable in real live hosting scenarios
+
+pick_first_virthost_on_nomatch = true
+
+
+# All unices are broken since it's not possible to bind to
+# a privileged port (< 1024) unless uid==0
+# There is a contrib in jungerl which makes it possible by means
+# of an external setuid root programm called fdsrv to listen to
+# to privileged port.
+# If we use this feature, it requires fdsrv to be properly installed.
+# Doesn't yet work with SSL.
+
+use_fdsrv = false
+
+max_num_cached_bytes = 10485760
+max_size_cached_file = 5120000
+
+# end then a set of virtual servers
+# First two virthosted servers on the same IP (0.0.0.0)
+# in this case, but an explicit IP can be given as well
+
+<server localhost>
+ port = 8000
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = false
+ appmods = emptytest streamtest
+ docroot = %YTOP%/www
+</server>
+
+<server localhost>
+ port = 8001
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ appmods = emptytest streamtest
+ docroot = %YTOP%/www
+</server>
+
+<server localhost>
+ port = 8002
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ <deflate>
+ mime_types = application/*
+ </deflate>
+</server>
+
+<server localhost>
+ port = 8003
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ <deflate>
+ mime_types = defaults
+ mime_types = application/*
+ </deflate>
+</server>
+
+<server localhost>
+ port = 8004
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ <deflate>
+ mime_types = all
+ </deflate>
+</server>
+
+<server localhost>
+ port = 8005
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ appmods = smalltest bigtest
+ <deflate>
+ min_compress_size = 2048000
+ </deflate>
+</server>
+
+
+<server localhost>
+ port = 8006
+ listen = 0.0.0.0
+ listen_backlog = 512
+ deflate = true
+ docroot = %YTOP%/www
+ appmods = smalltest bigtest streamtest
+ <deflate>
+ compression_level = best_speed
+ mem_level = 9
+ strategy = huffman_only
+ window_size = 9
+ </deflate>
+</server>
View
4 test/support/include.mk.in
@@ -60,3 +60,7 @@ authconf:
revproxyconf:
cat ../conf/revproxy.conf | \
../../scripts/Subst %YTOP% $(YTOP) > yaws.conf
+
+deflateconf:
+ cat ../conf/deflate.conf | \
+ ../../scripts/Subst %YTOP% $(YTOP) > yaws.conf
View
2 test/t1/Makefile
@@ -8,7 +8,7 @@ all: conf setup
## to run test, do
# make all test
-test:
+test: all
set -e ; \
for t in t1 t2 t3 t4 t5 t6 t7 json; do \
$(MAKE) $$t ; \
View
2 test/t2/Makefile
@@ -15,7 +15,7 @@ all: conf setup app_test.beam streamtest.beam jsontest.beam posttest.beam \
ULIMIT = 768
-test: conf start
+test: all start
dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1
dd if=/dev/zero of=../../www/2000.txt bs=1024 count=2000 >/dev/null 2>&1
dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1
View
2 test/t2/app_test.erl
@@ -11,7 +11,7 @@ start([F]) ->
ibrowse:stop().
start() ->
- ?line ok,
+ io:format("\n ==== MAIN TESTS ==== \n\n", []),
?line {ok, _} = ibrowse:start_link(),
server_options_test(),
test1(),
View
2 test/t3/Makefile
@@ -13,7 +13,7 @@ all: conf setup app_test.beam authmodtest.beam outmodtest.beam
ULIMIT = 768
-test: conf start
+test: all start
ul=`ulimit -n` ; \
val=`expr $$ul '<' $(ULIMIT)` ; \
if [ $$val = 1 ] ; then \
View
2 test/t3/app_test.erl
@@ -11,7 +11,7 @@ start([F]) ->
ibrowse:stop().
start() ->
- ?line ok,
+ io:format("\n ==== AUTH TESTS ==== \n\n", []),
?line {ok, _} = ibrowse:start_link(),
test1(),
test2(),
View
2 test/t4/Makefile
@@ -13,7 +13,7 @@ all: conf setup app_test.beam rewritetest.beam posttest.beam streamtest.beam
ULIMIT = 768
-test: conf start
+test: all start
dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1
dd if=/dev/zero of=../../www/2000.txt bs=1024 count=2000 >/dev/null 2>&1
dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1
View
2 test/t4/app_test.erl
@@ -18,7 +18,7 @@ start([F]) ->
end.
start() ->
- ?line ok,
+ io:format("\n ==== REVERSE PROXY TESTS ==== \n\n", []),
?line {ok, _} = ibrowse:start_link(),
try
deflate_revproxy_test1(),
View
38 test/t5/Makefile
@@ -0,0 +1,38 @@
+include ../support/include.mk
+
+.PHONY: all test debug clean
+
+#
+all: conf setup app_test.beam emptytest.beam streamtest.beam smalltest.beam bigtest.beam
+ @echo "all ok"
+
+
+# invoke as
+# TEST=test3 make test
+# or just make test to run all
+
+ULIMIT = 768
+
+test: all start
+ touch ../../www/0.txt
+ dd if=/dev/zero of=../../www/1000.txt bs=1024 count=1000 >/dev/null 2>&1
+ dd if=/dev/zero of=../../www/3000.txt bs=1024 count=3000 >/dev/null 2>&1
+ dd if=/dev/zero of=../../www/10000.txt bs=1024 count=10000 >/dev/null 2>&1
+ ul=`ulimit -n` ; \
+ val=`expr $$ul '<' $(ULIMIT)` ; \
+ if [ $$val = 1 ] ; then \
+ echo trying to raise "ulimit -n" for the test... ; \
+ set -e ; \
+ ulimit -n $(ULIMIT) ; \
+ fi ; \
+ $(ERL) -noinput $(PA) -s tftest
+ $(MAKE) stop
+
+conf: deflateconf
+
+debug:
+ $(ERL) $(PA)
+
+clean: tclean
+ -rm -f ../../www/0.txt ../../www/1000.txt ../../www/10000.txt
+ -rm -rf localhost:8000 logs yaws.conf
View
227 test/t5/app_test.erl
@@ -0,0 +1,227 @@
+-module(app_test).
+-include("../include/tftest.hrl").
+-include("../ibrowse/include/ibrowse.hrl").
+-compile(export_all).
+
+
+%% Way to invoke just one test
+start([F]) ->
+ ?line {ok, _} = ibrowse:start_link(),
+ apply(app_test, F, []),
+ ibrowse:stop().
+
+start() ->
+ io:format("\n ==== DEFLATE TESTS ==== \n\n", []),
+ ?line {ok, _} = ibrowse:start_link(),
+ deflate_disabled(),
+ deflate_enabled(),
+ deflate_empty_response(),
+ deflate_streamcontent(),
+ deflate_options(),
+ ibrowse:stop().
+
+
+deflate_disabled() ->
+ io:format("deflate_disabled\n", []),
+
+ %% Static content (and cached)
+ Uri1 = "http://localhost:8000/1000.txt",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+
+ %% Dynamic content
+ Uri2 = "http://localhost:8000/index.yaws",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs2),
+ ok.
+
+deflate_enabled() ->
+ io:format("deflate_enabled\n", []),
+
+ %% Static content (and catched)
+ Uri1 = "http://localhost:8001/1000.txt",
+ ?line {ok, "200", Hdrs1, Body1} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs1),
+ ?line true = is_binary(zlib:gunzip(Body1)),
+
+ %% Dynamic content
+ Uri2 = "http://localhost:8001/index.yaws",
+ ?line {ok, "200", Hdrs2, Body2} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line true = is_binary(zlib:gunzip(Body2)),
+ ok.
+
+
+deflate_empty_response() ->
+ io:format("deflate_empty_response\n", []),
+
+ %% Static content
+ Uri1 = "http://localhost:8000/0.txt",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+ ?line "0" = proplists:get_value("Content-Length", Hdrs1),
+
+ Uri2 = "http://localhost:8001/0.txt",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line "0" = proplists:get_value("Content-Length", Hdrs2),
+
+ %% Dynamic content
+ Uri3 = "http://localhost:8000/emptytest",
+ ?line {ok, "200", Hdrs3, _} =
+ ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs3),
+ ?line "0" = proplists:get_value("Content-Length", Hdrs3),
+
+ Uri4 = "http://localhost:8001/emptytest",
+ ?line {ok, "200", Hdrs4, _} =
+ ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs4),
+ ?line "0" = proplists:get_value("Content-Length", Hdrs4),
+ ok.
+
+
+deflate_streamcontent() ->
+ io:format("deflate_streamcontent\n", []),
+
+ %% Static content (cannot be cached so the file is chunked)
+ Uri1 = "http://localhost:8001/10000.txt",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs1),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs1),
+ ?line undefined = proplists:get_value("Content-Length", Hdrs1),
+
+ %% Dynamic content (chunked)
+ Uri2 = "http://localhost:8001/streamtest",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs2),
+ ?line undefined = proplists:get_value("Content-Length", Hdrs2),
+ ok.
+
+
+deflate_options() ->
+ io:format("deflate_options\n", []),
+ deflate_mime_types(),
+ deflate_compress_size(),
+ deflate_otheroptions(),
+ ok.
+
+deflate_mime_types() ->
+ io:format(" deflate_mime_types\n", []),
+
+ %% image/gif not compressed on localhost:8001
+ Uri1 = "http://localhost:8001/icons/yaws.gif",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+
+ %% application/* compressed on localhost:8002 but text/plain not
+ Uri2 = "http://localhost:8002/1000.txt",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs2),
+
+ Uri3 = "http://localhost:8002/yaws.eps",
+ ?line {ok, "200", Hdrs3, Body3} =
+ ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs3),
+ ?line true = is_binary(zlib:gunzip(Body3)),
+
+ %% application/* and text/* compressed on localhost:8003
+ Uri4 = "http://localhost:8003/1000.txt",
+ ?line {ok, "200", Hdrs4, Body4} =
+ ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs4),
+ ?line true = is_binary(zlib:gunzip(Body4)),
+
+ Uri5 = "http://localhost:8003/yaws.eps",
+ ?line {ok, "200", Hdrs5, Body5} =
+ ibrowse:send_req(Uri5, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs5),
+ ?line true = is_binary(zlib:gunzip(Body5)),
+
+ %% All mime types are compressed on localhost:8004
+ Uri6 = "http://localhost:8004/1000.txt",
+ ?line {ok, "200", Hdrs6, Body6} =
+ ibrowse:send_req(Uri6, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs6),
+ ?line true = is_binary(zlib:gunzip(Body6)),
+
+ Uri7 = "http://localhost:8004/yaws.eps",
+ ?line {ok, "200", Hdrs7, Body7} =
+ ibrowse:send_req(Uri7, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs7),
+ ?line true = is_binary(zlib:gunzip(Body7)),
+ ok.
+
+
+deflate_compress_size() ->
+ io:format(" deflate_compress_size\n", []),
+
+ %% Static content (cached)
+ Uri1 = "http://localhost:8005/1000.txt",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs1),
+
+ Uri2 = "http://localhost:8005/3000.txt",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+
+ %% Dynamic content
+ Uri3 = "http://localhost:8005/smalltest",
+ ?line {ok, "200", Hdrs3, _} =
+ ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line undefined = proplists:get_value("Content-Encoding", Hdrs3),
+
+ Uri4 = "http://localhost:8005/bigtest",
+ ?line {ok, "200", Hdrs4, _} =
+ ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs4),
+ ok.
+
+
+deflate_otheroptions() ->
+ io:format(" deflate_otheroptions\n", []),
+
+ %% Static content
+ Uri1 = "http://localhost:8006/1000.txt",
+ ?line {ok, "200", Hdrs1, _} =
+ ibrowse:send_req(Uri1, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs1),
+
+ Uri2 = "http://localhost:8006/10000.txt",
+ ?line {ok, "200", Hdrs2, _} =
+ ibrowse:send_req(Uri2, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs2),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs2),
+ ?line undefined = proplists:get_value("Content-Length", Hdrs2),
+
+ %% Dynamic content
+ Uri3 = "http://localhost:8006/smalltest",
+ ?line {ok, "200", Hdrs3, _} =
+ ibrowse:send_req(Uri3, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs3),
+
+ Uri4 = "http://localhost:8006/bigtest",
+ ?line {ok, "200", Hdrs4, _} =
+ ibrowse:send_req(Uri4, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs4),
+
+ Uri5 = "http://localhost:8006/streamtest",
+ ?line {ok, "200", Hdrs5, _} =
+ ibrowse:send_req(Uri5, [{"Accept-Encoding", "gzip, deflate"}], get),
+ ?line "gzip" = proplists:get_value("Content-Encoding", Hdrs5),
+ ?line "chunked" = proplists:get_value("Transfer-Encoding", Hdrs5),
+ ?line undefined = proplists:get_value("Content-Length", Hdrs5),
+ ok.
View
9 test/t5/bigtest.erl
@@ -0,0 +1,9 @@
+-module(bigtest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(_Arg) ->
+ [{status, 200}, {page, "/3000.txt"}].
+
View
9 test/t5/emptytest.erl
@@ -0,0 +1,9 @@
+-module(emptytest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(_Arg) ->
+ {status, 200}.
+
View
9 test/t5/smalltest.erl
@@ -0,0 +1,9 @@
+-module(smalltest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(_Arg) ->
+ [{status, 200}, {page, "/1000.txt"}].
+
View
12 test/t5/streamtest.erl
@@ -0,0 +1,12 @@
+-module(streamtest).
+-export([out/1]).
+
+-include("../../include/yaws.hrl").
+-include("../../include/yaws_api.hrl").
+
+out(_Arg) ->
+ yaws_api:stream_chunk_deliver(self(), "and this is the second one\n"),
+ yaws_api:stream_chunk_deliver(self(), "con"),
+ yaws_api:stream_chunk_deliver(self(), "sequence"),
+ yaws_api:stream_chunk_end(self()),
+ {streamcontent, "text/plain", "This is the data in the first chunk\n"}.

0 comments on commit c74a92e

Please sign in to comment.