Skip to content

Commit 854581c

Browse files
RaoHRaoH
committed
Moved everything that handles minifying to a specific module.
Added call to all nodes to build minified files on update event for component. Added some caching, made things a tad quicker.
1 parent d4e0562 commit 854581c

File tree

3 files changed

+156
-111
lines changed

3 files changed

+156
-111
lines changed

‎src/component_action.erl

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -52,8 +52,10 @@ update_component(Req, ComponentName) ->
5252
ComponentName/binary>>);
5353
_ ->
5454
%% Successful update, send to all other nodes as well.
55+
send_reply(Req, success, <<"Component has been updated">>),
5556
rpc:eval_everywhere(git_utils, refresh_repo, [ComponentName]),
56-
send_reply(Req, success, <<"Component has been updated">>)
57+
rpc:eval_everywhere(divapi_js_minifier, minify, [ComponentName]),
58+
ok
5759
end;
5860
false ->
5961
send_reply(Req, error, <<"Component does not exists.">>)

‎src/components_js_minifier.erl

Lines changed: 7 additions & 110 deletions
Original file line numberDiff line numberDiff line change
@@ -12,126 +12,23 @@ init(_, Req, _Opts) ->
1212
{ok, Req, []}.
1313

1414
handle(Req, State) ->
15-
{ok, Req1} = cowboy_req:chunked_reply(
16-
200, [{<<"content-type">>, <<"application/javascript">>}], Req
15+
{Components, Req1} = cowboy_req:qs_vals(Req),
16+
{ok, Req2} = cowboy_req:chunked_reply(
17+
200, [{<<"content-type">>, <<"application/javascript">>}], Req1
1718
),
18-
%% Proplist with one or more components to minfify
19-
{Components, Req2} = cowboy_req:qs_vals(Req1),
20-
21-
%% Build a map with all components like
22-
ComponentScriptFilePaths = pmap(
23-
fun ({Component, Tag}) ->
24-
{DiversityProps} = jiffy:decode(git_utils:get_diversity_json(Component, Tag)),
25-
ScriptFiles = proplists:get_value(<<"script">>, DiversityProps),
26-
TaggedScriptFiles = lists:map(fun get_file_type/1, ScriptFiles),
27-
{{Component, Tag}, TaggedScriptFiles}
28-
end,
29-
Components
30-
),
31-
32-
Z = zlib:open(),
33-
ok = zlib:deflateInit(Z, default),
3419
pmap(
35-
fun ({{Component, Tag}, FilePaths}) ->
36-
% zlib:deflateReset(Z),
37-
Result = get_minified_file(Component, Tag, FilePaths, fun get_file_contents/4),
38-
Result1 = zlib:deflate(Z, Result, finish),
39-
io:format("Result ~p" , [Result1]),
40-
cowboy_req:chunk(Result1, Req2),
20+
fun ({Component, Tag}) ->
21+
Result = divapi_js_minifier:minify(Component, Tag),
22+
cowboy_req:chunk(Result, Req2),
4123
Result
4224
end,
43-
ComponentScriptFilePaths
25+
Components
4426
),
45-
cowboy_req:chunk(zlib:deflate(Z, <<>>, finish), Req2),
46-
ok = zlib:deflateEnd(Z),
47-
zlib:close(Z),
4827
{ok, Req2, State}.
4928

5029
terminate(_Reason, _Req, _State) ->
5130
ok.
5231

53-
get_file_type(<<"//", _Rest/binary>> = ScriptFile) ->
54-
%% This is an exteral js file that needs to be downloaded.
55-
{external, <<"http:", ScriptFile/binary>>};
56-
get_file_type(<<"http://", _Rest/binary>> = ScriptFile) ->
57-
%% This is an exteral js file that needs to be downloaded.
58-
{external, ScriptFile};
59-
get_file_type(<<"https://", _Rest/binary>> = ScriptFile) ->
60-
%% This is an exteral js file that needs to be downloaded.
61-
{external, ScriptFile};
62-
get_file_type(ScriptFile) ->
63-
case re:run(ScriptFile, <<".min.js">>) of
64-
{match, _} ->
65-
%% It's already minified. And exists in our component already
66-
{already_minified, ScriptFile};
67-
_ ->
68-
%% It's not a minified file. Assume internal
69-
{jsfile, ScriptFile}
70-
end.
71-
72-
get_minified_file(Component, Tag, FilePaths, MinifierFun) ->
73-
MinifiedPath = case code:priv_dir(divapi) of
74-
{error, bad_name} -> <<"priv/js_minfied/">>;
75-
PrivDir -> <<(list_to_binary(PrivDir))/binary, "/js_minified">>
76-
end,
77-
ComponentMinifiedPath = filename:join(MinifiedPath, <<Component/binary, "/", Tag/binary, "/">>),
78-
ok = filelib:ensure_dir(<<ComponentMinifiedPath/binary, "/">>),
79-
FilePath = filename:join(ComponentMinifiedPath, <<"scripts.min.js">>),
80-
case filelib:is_file(FilePath) of
81-
true ->
82-
{ok, FileBody} = file:read_file(FilePath),
83-
FileBody;
84-
false ->
85-
FileBody = MinifierFun(Component, Tag, FilePaths, <<>>),
86-
ok = file:write_file(FilePath, FileBody),
87-
FileBody
88-
end.
89-
90-
%% Fetch filec content from outside. And add it to the filecontent
91-
get_file_contents(Component, Tag, [{external, ScriptFilePath} | Rest], FileContent) ->
92-
Opts = [{body_format, binary}],
93-
Request = {binary_to_list(ScriptFilePath), []},
94-
ScriptFile = case httpc:request(get, Request, [], Opts) of
95-
{ok, {{_Version, Status, _ReasonPhrase}, _Headers, Body}} ->
96-
97-
case Status of
98-
404 -> throw({resource_not_found, ScriptFilePath});
99-
500 -> throw({server_error, ScriptFilePath});
100-
200 -> Body
101-
end
102-
end,
103-
get_file_contents(Component, Tag, Rest, <<FileContent/binary, ScriptFile/binary>>);
104-
%% No action needed, add it to the file acc as is
105-
get_file_contents(ComponentName, Tag, [{already_minified, ScriptFilePath} | Rest], FileContent) ->
106-
FileBody = git_utils:get_file(ComponentName, Tag, ScriptFilePath),
107-
get_file_contents(ComponentName, Tag, Rest, <<FileContent/binary, FileBody/binary>>);
108-
%% Fetches filecontent for component and minify it through uglify
109-
get_file_contents(ComponentName, Tag, [{jsfile, ScriptFilePath} | Rest], FileContent) ->
110-
TmpName0 = {ComponentName, Tag, ScriptFilePath},
111-
TmpName1 = integer_to_binary(erlang:phash2(TmpName0)),
112-
TmpPath = filename:join(<<"/tmp/divapi/">>, <<TmpName1/binary, ".js">>),
113-
File = git_utils:get_file(ComponentName, Tag, ScriptFilePath),
114-
ok = filelib:ensure_dir(TmpPath),
115-
ok = file:write_file(TmpPath, File),
116-
Command = <<"uglifyjs ", TmpPath/binary>>,
117-
Port = erlang:open_port({spawn, Command}, [exit_status, binary, stderr_to_stdout]),
118-
Result = wait_for_file(Port, <<>>),
119-
%% Cleanup
120-
file:delete(TmpPath),
121-
get_file_contents(ComponentName, Tag, Rest, <<FileContent/binary, Result/binary>>);
122-
get_file_contents(_Component, _Tag, [], FileContent) ->
123-
FileContent.
124-
125-
wait_for_file(Port, File) ->
126-
receive
127-
{Port, {data, Chunk}} ->
128-
wait_for_file(Port, <<File/binary, Chunk/binary>>);
129-
{_ , {exit_status, 0}} ->
130-
File;
131-
{_, {exit_status, _}} ->
132-
throw({wait_for_response, failure})
133-
end.
134-
13532
pmap(Fun, List) ->
13633
pmap(Fun, List, 5000).
13734

‎src/divapi_js_minifier.erl

Lines changed: 146 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,146 @@
1+
-module(divapi_js_minifier).
2+
3+
-export([minify/1, minify/2, get_minified_tags/1]).
4+
5+
%% @doc Minifies all scripts and concatenates it to one file for all tagged versions not
6+
%% already minfied.
7+
minify(ComponentName) ->
8+
ComponentTags = git_utils:tags(ComponentName),
9+
case get_minified_tags(ComponentName) of
10+
[] -> %% phew do allll tags now
11+
lists:foreach(
12+
fun (Tag) ->
13+
minify(ComponentName, Tag)
14+
end,
15+
ComponentTags
16+
);
17+
MinifiedTags ->
18+
lists:foreach(
19+
fun (Tag) ->
20+
case lists:member(Tag, MinifiedTags) of
21+
false -> minify(ComponentName, Tag);
22+
true -> ok
23+
end
24+
end,
25+
ComponentTags
26+
)
27+
end,
28+
ok.
29+
30+
%% @doc Minifies all scripts for a specific component and tag. Returns minified file.
31+
minify(Component, Tag) ->
32+
ScriptFilePaths = get_component_script_files(Component, Tag),
33+
minify(Component, Tag, ScriptFilePaths).
34+
35+
%% @doc Minifies all scripts for a specific component and tag. Returns minified file.
36+
%% If component/tag already minfied, that result will be returned.
37+
minify(ComponentName, Tag, ScriptFilePaths) ->
38+
MinifiedPath = get_minify_path(ComponentName),
39+
ComponentMinifiedPath = filename:join(MinifiedPath, <<Tag/binary, "/">>),
40+
ok = filelib:ensure_dir(<<ComponentMinifiedPath/binary, "/">>),
41+
FilePath = filename:join(ComponentMinifiedPath, <<"scripts.min.js">>),
42+
case filelib:is_file(FilePath) of
43+
true ->
44+
{ok, FileBody} = file:read_file(FilePath),
45+
FileBody;
46+
false ->
47+
FileBody = get_file_contents(ComponentName, Tag, ScriptFilePaths, <<>>),
48+
ok = file:write_file(FilePath, FileBody),
49+
FileBody
50+
end.
51+
52+
%% @doc Returns all scriptfiles listed in the diversity.json setting. It will tag
53+
%% each file if it's external, jsfile or already_minified.
54+
get_component_script_files(ComponentName, Tag) ->
55+
{DiversityProps} = get_diversity_json(ComponentName, Tag),
56+
ScriptFiles = proplists:get_value(<<"script">>, DiversityProps),
57+
TaggedScriptFiles = lists:map(fun get_file_type/1, ScriptFiles),
58+
TaggedScriptFiles.
59+
60+
%% @doc Fetches the diversity json for a spcific component/tag. Will cache the result.
61+
get_diversity_json(ComponentName, Tag) ->
62+
divapi_cache:get(
63+
{diversity_json, ComponentName, Tag},
64+
fun () -> jiffy:decode(git_utils:get_diversity_json(ComponentName, Tag)) end,
65+
1000 * 60 * 60 * 5 % 5 hours
66+
).
67+
68+
%% @doc Fetches a list of already minified tags for a specific component.
69+
get_minified_tags(ComponentName) ->
70+
MinifiedPath = get_minify_path(ComponentName),
71+
case file:list_dir(MinifiedPath) of
72+
{ok, TagList} -> lists:map(fun(Tag) -> unicode:characters_to_binary(Tag) end, TagList);
73+
{error, _} -> []
74+
end.
75+
76+
%% @doc Returns path to where each minified file is stored for component.
77+
get_minify_path(ComponentName) ->
78+
MinifiedPath = case code:priv_dir(divapi) of
79+
{error, bad_name} -> <<"priv/js_minfied/">>;
80+
PrivDir -> <<(list_to_binary(PrivDir))/binary, "/js_minified">>
81+
end,
82+
filename:join(MinifiedPath, <<ComponentName/binary, "/">>).
83+
84+
get_file_type(<<"//", _Rest/binary>> = ScriptFile) ->
85+
%% This is an exteral js file that needs to be downloaded.
86+
{external, <<"http:", ScriptFile/binary>>};
87+
get_file_type(<<"http://", _Rest/binary>> = ScriptFile) ->
88+
%% This is an exteral js file that needs to be downloaded.
89+
{external, ScriptFile};
90+
get_file_type(<<"https://", _Rest/binary>> = ScriptFile) ->
91+
%% This is an exteral js file that needs to be downloaded.
92+
{external, ScriptFile};
93+
get_file_type(ScriptFile) ->
94+
case re:run(ScriptFile, <<".min.js">>) of
95+
{match, _} ->
96+
%% It's already minified. And exists in our component already
97+
{already_minified, ScriptFile};
98+
_ ->
99+
%% It's not a minified file. Assume internal
100+
{jsfile, ScriptFile}
101+
end.
102+
103+
%% Fetch filec content from outside. And add it to the filecontent
104+
get_file_contents(Component, Tag, [{external, ScriptFilePath} | Rest], FileContent) ->
105+
Opts = [{body_format, binary}],
106+
Request = {binary_to_list(ScriptFilePath), []},
107+
ScriptFile = case httpc:request(get, Request, [], Opts) of
108+
{ok, {{_Version, Status, _ReasonPhrase}, _Headers, Body}} ->
109+
110+
case Status of
111+
404 -> throw({resource_not_found, ScriptFilePath});
112+
500 -> throw({server_error, ScriptFilePath});
113+
200 -> Body
114+
end
115+
end,
116+
get_file_contents(Component, Tag, Rest, <<FileContent/binary, ScriptFile/binary>>);
117+
%% No action needed, add it to the file acc as is
118+
get_file_contents(ComponentName, Tag, [{already_minified, ScriptFilePath} | Rest], FileContent) ->
119+
FileBody = git_utils:get_file(ComponentName, Tag, ScriptFilePath),
120+
get_file_contents(ComponentName, Tag, Rest, <<FileContent/binary, FileBody/binary>>);
121+
%% Fetches filecontent for component and minify it through uglify
122+
get_file_contents(ComponentName, Tag, [{jsfile, ScriptFilePath} | Rest], FileContent) ->
123+
TmpName0 = {ComponentName, Tag, ScriptFilePath},
124+
TmpName1 = integer_to_binary(erlang:phash2(TmpName0)),
125+
TmpPath = filename:join(<<"/tmp/divapi/">>, <<TmpName1/binary, ".js">>),
126+
File = git_utils:get_file(ComponentName, Tag, ScriptFilePath),
127+
ok = filelib:ensure_dir(TmpPath),
128+
ok = file:write_file(TmpPath, File),
129+
Command = <<"uglifyjs ", TmpPath/binary>>,
130+
Port = erlang:open_port({spawn, Command}, [exit_status, binary, stderr_to_stdout]),
131+
Result = wait_for_file(Port, <<>>),
132+
%% Cleanup
133+
file:delete(TmpPath),
134+
get_file_contents(ComponentName, Tag, Rest, <<FileContent/binary, Result/binary>>);
135+
get_file_contents(_Component, _Tag, [], FileContent) ->
136+
FileContent.
137+
138+
wait_for_file(Port, File) ->
139+
receive
140+
{Port, {data, Chunk}} ->
141+
wait_for_file(Port, <<File/binary, Chunk/binary>>);
142+
{_ , {exit_status, 0}} ->
143+
File;
144+
{_, {exit_status, _}} ->
145+
throw({wait_for_response, failure})
146+
end.

0 commit comments

Comments
 (0)