Skip to content

HTTPS clone URL

Subversion checkout URL

You can clone with HTTPS or Subversion.

Download ZIP
Browse files

mod_export: added generic export controller and notifications. (docs …

…will be added)
  • Loading branch information...
commit 79841c16fe520312437e98463c93dca53d1ce470 1 parent 3433892
@mworrell mworrell authored
View
52 include/zotonic_notifications.hrl
@@ -392,6 +392,58 @@
-record(debug, {what, arg=[]}).
+%% @doc mod_export - return the content type (like {ok, "text/csv"}) for the dispatch rule/id export.
+-record(export_resource_content_type, {
+ dispatch :: atom(),
+ id :: integer()
+ }).
+
+%% @doc mod_export - return the {ok, Filename} for the content disposition.
+-record(export_resource_filename, {
+ dispatch :: atom(),
+ id :: integer(),
+ content_type :: string()
+ }).
+
+%% @doc mod_export - Fetch the header for the export.
+%% The 'first' notification should return: {ok, binary()} | {ok, binary(), ContinuationState} | {error, Reason}.
+-record(export_resource_header, {
+ dispatch :: atom(),
+ id :: integer(),
+ content_type :: string()
+ }).
+
+%% @doc mod_export - fetch a row for the export, can return a list of rows, a binary, and optionally a continuation state.
+%% The 'first' notification should return: {ok, Values|binary()} | {ok, Values|binary(), ContinuationState} | {error, Reason}.
+%% Where Values is [ term() ], i.e. a list of opaque values, to be formatted with #export_resource_format.
+%% Return the empty list of values to signify the end of the data stream.
+-record(export_resource_data, {
+ dispatch :: atom(),
+ id :: integer(),
+ content_type :: string(),
+ state :: term()
+ }).
+
+%% @doc mod_export - Encode a single data element.
+%% The 'first' notification should return: {ok, binary()} | {ok, binary(), ContinuationState} | {error, Reason}.
+-record(export_resource_encode, {
+ dispatch :: atom(),
+ id :: integer(),
+ content_type :: string(),
+ data :: term(),
+ state :: term()
+ }).
+
+%% @doc mod_export - Fetch the footer for the export. Should cleanup the continuation state, if needed.
+%% The 'first' notification should return: {ok, binary()} | {error, Reason}.
+-record(export_resource_footer, {
+ dispatch :: atom(),
+ id :: integer(),
+ content_type :: string(),
+ state :: term()
+ }).
+
+
% Simple mod_development notifications:
% development_reload - Reload all template, modules etc
% development_make - Perform a 'make' on Zotonic, reload all new beam files
View
220 modules/mod_export/controllers/controller_export_resource.erl
@@ -0,0 +1,220 @@
+%% @author Marc Worrell <marc@worrell.nl>
+%% @copyright 2013 Marc Worrell <marc@worrell.nl>
+%% @doc Export a resource in the given format, uses notifiers for fetching and encoding data.
+
+%% Copyright 2013 Marc Worrell
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(controller_export_resource).
+-author("Marc Worrell <marc@worrell.nl>").
+
+-export([
+ init/1,
+ service_available/2,
+ resource_exists/2,
+ previously_existed/2,
+ forbidden/2,
+ content_types_provided/2,
+ charsets_provided/2,
+
+ do_export/2
+]).
+
+-include_lib("controller_webmachine_helper.hrl").
+-include_lib("zotonic.hrl").
+
+init(DispatchArgs) -> {ok, DispatchArgs}.
+
+service_available(ReqData, DispatchArgs) when is_list(DispatchArgs) ->
+ Context = z_context:new(ReqData, ?MODULE),
+ Context1 = z_context:set(DispatchArgs, Context),
+ ?WM_REPLY(true, Context1).
+
+resource_exists(ReqData, Context) ->
+ Context1 = ?WM_REQ(ReqData, Context),
+ {Id, ContextQs} = get_id(z_context:ensure_qs(z_context:continue_session(Context1))),
+ ?WM_REPLY(m_rsc:exists(Id, ContextQs), ContextQs).
+
+previously_existed(ReqData, Context) ->
+ Context1 = ?WM_REQ(ReqData, Context),
+ {Id, Context2} = get_id(Context1),
+ IsGone = m_rsc_gone:is_gone(Id, Context2),
+ ?WM_REPLY(IsGone, Context2).
+
+forbidden(ReqData, Context) ->
+ Context1 = ?WM_REQ(ReqData, Context),
+ {Id, Context2} = get_id(z_context:ensure_qs(z_context:continue_session(Context1))),
+ ?WM_REPLY(not z_acl:rsc_visible(Id, Context2), Context2).
+
+content_types_provided(ReqData, Context0) ->
+ Context = ?WM_REQ(ReqData, Context0),
+ {Id, Context1} = get_id(z_context:ensure_qs(z_context:continue_session(Context))),
+ Dispatch = z_context:get(zotonic_dispatch, Context1),
+ case get_content_type(Id, Dispatch, Context1) of
+ {ok, ContentType} ->
+ Context2 = z_context:set(content_type_mime, ContentType, Context1),
+ ?WM_REPLY([{ContentType, do_export}], Context2);
+ {error, Reason} = Error ->
+ lager:error("~p: mod_export error when fetching content type for ~p:~p: ~p",
+ [z_context:site(Context1), Dispatch, Id, Reason]),
+ throw(Error)
+ end.
+
+charsets_provided(ReqData, Context) ->
+ {[{"utf-8", fun(X) -> X end}], ReqData, Context}.
+
+do_export(ReqData, Context0) ->
+ Context = ?WM_REQ(ReqData, Context0),
+ Stream = {stream, {<<>>, fun() -> do_header(Context) end}},
+ Context1 = set_filename(Context),
+ ?WM_REPLY(Stream, Context1).
+
+do_header(Context) ->
+ ContentType = z_context:get(content_type_mime, Context),
+ {Id, _} = get_id(Context),
+ Dispatch = z_context:get(zotonic_dispatch, Context),
+ case z_notifier:first(#export_resource_header{id=Id, content_type=ContentType, dispatch=Dispatch}, Context) of
+ undefined ->
+ {<<>>, fun() -> do_body(undefined, Context) end};
+ {ok, Header, State} ->
+ {flatten_header(Header, ContentType, Context), fun() -> do_body(State, Context) end};
+ {ok, Header} ->
+ {flatten_header(Header, ContentType, Context), fun() -> do_body(undefined, Context) end}
+ end.
+
+flatten_header(Header, _ContentType, _Context) when is_binary(Header) ->
+ Header;
+flatten_header(Header, ContentType, Context) when is_list(Header) ->
+ case ContentType of
+ "text/csv" -> export_encode_csv:encode(Header, Context);
+ _ -> iolist_to_binary(Header)
+ end;
+flatten_header(Header, _ContentType, _Context) ->
+ z_convert:to_binary(Header).
+
+
+do_body(State, Context) ->
+ ContentType = z_context:get(content_type_mime, Context),
+ {Id, _} = get_id(Context),
+ Dispatch = z_context:get(zotonic_dispatch, Context),
+ case z_notifier:first(#export_resource_data{id=Id, content_type=ContentType, dispatch=Dispatch}, Context) of
+ undefined -> do_body_data([Id], State, Context);
+ {ok, List} -> do_body_data(List, State, Context);
+ {ok, List, NewState} -> do_body_data(List, NewState, Context)
+ end.
+
+do_body_data([], State, Context) ->
+ do_footer(State, Context);
+do_body_data(List, State, Context) ->
+ {Data, NewState} = lists:foldl(
+ fun(D, {Acc, AccState}) ->
+ {DEnc, AccState1} = do_body_encode(D, AccState, Context),
+ {[Acc, DEnc], AccState1}
+ end,
+ {[], State},
+ List),
+ DataBin = iolist_to_binary(Data),
+ case State of
+ undefined -> {DataBin, fun() -> do_footer(undefined, Context) end};
+ _ -> {DataBin, fun() -> do_body(NewState, Context) end}
+ end.
+
+do_body_encode(Item, State, Context) ->
+ ContentType = z_context:get(content_type_mime, Context),
+ {Id, _} = get_id(Context),
+ Dispatch = z_context:get(zotonic_dispatch, Context),
+ case z_notifier:first(#export_resource_encode{
+ id=Id,
+ dispatch=Dispatch,
+ content_type=ContentType,
+ data=Item,
+ state=State}, Context)
+ of
+ undefined -> {<<>>, State};
+ {ok, Enc} -> {Enc, State};
+ {ok, Enc, NewState} -> {Enc, NewState}
+ end.
+
+do_footer(State, Context) ->
+ ContentType = z_context:get(content_type_mime, Context),
+ {Id, _} = get_id(Context),
+ Dispatch = z_context:get(zotonic_dispatch, Context),
+ case z_notifier:first(#export_resource_footer{
+ id=Id,
+ dispatch=Dispatch,
+ content_type=ContentType,
+ state=State}, Context)
+ of
+ undefined -> {<<>>, done};
+ {ok, Enc} -> {Enc, done}
+ end.
+
+
+set_filename(Context) ->
+ ContentType = z_context:get(content_type_mime, Context),
+ {Id, _} = get_id(Context),
+ Dispatch = z_context:get(zotonic_dispatch, Context),
+ case z_notifier:first(#export_resource_filename{
+ id=Id,
+ dispatch=Dispatch,
+ content_type=ContentType}, Context)
+ of
+ undefined ->
+ Cat = m_rsc:p_no_acl(Id, category, Context),
+ Extension = case mimetypes:mime_to_exts(ContentType) of
+ undefined -> "bin";
+ Exts -> binary_to_list(hd(Exts))
+ end,
+ Filename = "export-"
+ ++z_convert:to_list(proplists:get_value(name, Cat))
+ ++"-"
+ ++z_convert:to_list(Id)
+ ++"."
+ ++Extension,
+ z_context:set_resp_header("Content-Disposition", "attachment; filename="++Filename, Context);
+ {ok, Filename} ->
+ Filename1 = z_convert:to_list(Filename),
+ z_context:set_resp_header("Content-Disposition", "attachment; filename="++Filename1, Context)
+ end.
+
+%% @doc Fetch the content type being served
+get_content_type(Id, Dispatch, Context) ->
+ case z_context:get(content_type, Context) of
+ csv ->
+ {ok, "text/csv"};
+ ContentType when is_list(ContentType) ->
+ {ok, ContentType};
+ undefined ->
+ case z_notifier:first(#export_resource_content_type{id=Id, dispatch=Dispatch}, Context) of
+ undefined -> {error, no_content_type};
+ Other -> Other
+ end
+ end.
+
+
+get_id(Context) ->
+ case z_context:get(id, Context) of
+ undefined ->
+ case z_context:get_q("id", Context) of
+ undefined ->
+ {undefined, Context};
+ [] ->
+ {undefined, Context};
+ Id ->
+ RscId = m_rsc:rid(Id, Context),
+ {RscId, z_context:set(id, {ok, RscId}, Context)}
+ end;
+ {ok, Id} ->
+ {Id, Context}
+ end.
View
3  modules/mod_export/dispatch/dispatch_export
@@ -0,0 +1,3 @@
+[
+ {export_rsc_csv, ["export", "csv", id], controller_export_resource, [ {content_type, "text/csv"} ]}
+].
View
70 modules/mod_export/mod_export.erl
@@ -0,0 +1,70 @@
+%% @author Marc Worrell <marc@worrell.nl>
+%% @copyright 2013 Marc Worrell
+%% @doc Generic export routines for data sources
+
+%% Copyright 2013 Marc Worrell
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(mod_export).
+-author("Marc Worrell <marc@worrell.nl>").
+
+-mod_title("Export Data").
+-mod_description("Exports data as CSV and other formats.").
+-mod_prio(800).
+-mod_depends([mod_base]).
+
+-export([
+ observe_content_types_dispatch/3,
+ observe_export_resource_data/2,
+ observe_export_resource_encode/2
+ ]).
+
+-include_lib("zotonic.hrl").
+
+%% @doc Add an extra content-type to the 'id' controller.
+observe_content_types_dispatch(#content_types_dispatch{}, Acc, _Context) ->
+ [{"text/csv", export_rsc_csv} | Acc].
+
+
+%% @doc Fetch all ids making up the export, handles collections and search queries.
+observe_export_resource_data(#export_resource_data{id=Id}, Context) ->
+ case m_rsc:is_a(Id, 'query', Context) of
+ true ->
+ {ok, z_search:query_([{id, Id}], Context)};
+ false ->
+ case m_rsc:is_a(Id, collection, Context) of
+ true -> {ok, m_edge:objects(Id, haspart, Context)};
+ false -> undefined
+ end
+ end.
+
+%% @doc Encode a "row" of data, according to the encoding requested
+observe_export_resource_encode(#export_resource_encode{content_type="text/csv", data=Id}, Context) when is_integer(Id) ->
+ Data = rsc_data(Id, Context),
+ {ok, export_encode_csv:encode(Data, Context)};
+observe_export_resource_encode(#export_resource_encode{}, _Context) ->
+ undefined.
+
+rsc_data(Id, Context) ->
+ [ Id | [ m_rsc:p(Id, Prop, Context) || Prop <- rsc_fields() ] ].
+
+rsc_fields() ->
+ [
+ title,
+ summary,
+ created,
+ modified,
+ page_url_abs
+ ].
+
View
78 modules/mod_export/support/export_encode_csv.erl
@@ -0,0 +1,78 @@
+%% @author Marc Worrell <marc@worrell.nl>
+%% @copyright 2013 Marc Worrell
+%% @doc Encode a list of values using CSV
+
+%% Copyright 2013 Marc Worrell
+%%
+%% Licensed under the Apache License, Version 2.0 (the "License");
+%% you may not use this file except in compliance with the License.
+%% You may obtain a copy of the License at
+%%
+%% http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing, software
+%% distributed under the License is distributed on an "AS IS" BASIS,
+%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+%% See the License for the specific language governing permissions and
+%% limitations under the License.
+
+-module(export_encode_csv).
+-author("Marc Worrell <marc@worrell.nl>").
+
+-export([
+ encode/2
+ ]).
+
+
+encode([], _Context) ->
+ <<"\r\n">>;
+encode([V], Context) ->
+ iolist_to_binary([
+ encode_value(V, Context),
+ <<"\r\n">>
+ ]);
+encode([V|Xs], Context) ->
+ iolist_to_binary([
+ encode_value(V, Context),
+ [ [$,, encode_value(X, Context)] || X <- Xs ],
+ <<"\r\n">>
+ ]).
+
+encode_value(undefined, _Context) ->
+ <<>>;
+encode_value(<<>>, _Context) ->
+ <<>>;
+encode_value(N, _Context) when is_integer(N) ->
+ z_convert:to_binary(N);
+encode_value(N, _Context) when is_float(N) ->
+ z_convert:to_binary(N);
+encode_value(B, _Context) when is_binary(B) ->
+ quote(escape(B));
+encode_value({Y,M,D} = Date, Context)
+ when is_integer(Y), is_integer(M), is_integer(D) ->
+ quote(erlydtl_dateformat:format({Date, {0,0,0}}, "Y-m-d", Context));
+encode_value({{Y,M,D}, {H,I,S}} = Date, Context)
+ when is_integer(Y), is_integer(M), is_integer(D),
+ is_integer(H), is_integer(I), is_integer(S) ->
+ quote(erlydtl_dateformat:format(Date, "Y-m-d H:i:s", Context));
+encode_value({trans, _} = Trans, Context) ->
+ encode_value(z_trans:lookup_fallback(Trans, Context), Context);
+encode_value(N, Context) ->
+ encode_value(z_convert:to_binary(N), Context).
+
+quote(B) -> [$", B, $"].
+
+% We need to recognize fields starting with a '=', as Excel thinks that is a formula.
+escape(<<$=, B/binary>>) -> escape(B, <<" =">>);
+escape(B) -> escape(B, <<>>).
+
+escape(<<>>, Acc) -> Acc;
+escape(<<$", B/binary>>, Acc) -> escape(B, <<Acc/binary, $", $">>);
+escape(<<10, B/binary>>, Acc) -> escape(B, <<Acc/binary, 32>>);
+escape(<<13, 10, B/binary>>, Acc) -> escape(B, <<Acc/binary, 32>>);
+escape(<<13, B/binary>>, Acc) -> escape(B, <<Acc/binary, 32>>);
+escape(<<X, B/binary>>, Acc) when X < 32 -> escape(B, Acc);
+escape(<<X, B/binary>>, Acc) -> escape(B, <<Acc/binary, X>>).
+
+
+
Please sign in to comment.
Something went wrong with that request. Please try again.