Permalink
Browse files

Implement "trans" tag with support for .po files

Includes a .po parser and generator, docs, and tests. See README_I18N.

Many thanks to David Garcia.
  • Loading branch information...
1 parent 449f2e7 commit 48fea229f2a1d886d35d7b391547c7489defdb08 Evan Miller committed Apr 28, 2010
View
@@ -1,3 +1,4 @@
{"src/erlydtl/*", [debug_info, {outdir, "ebin"}]}.
-{"src/tests/*", [debug_info, {outdir, "ebin"}]}.
-{"src/demo/*", [debug_info, {outdir, "ebin"}]}.
+{"src/erlydtl/i18n/*", [debug_info, {outdir, "ebin"}]}.
+{"src/tests/*", [debug_info, {outdir, "ebintest"}]}.
+{"src/demo/*", [debug_info, {outdir, "ebintest"}]}.
View
@@ -5,6 +5,7 @@ PARSER=src/erlydtl/erlydtl_parser
APP=erlydtl.app
all: $(PARSER).erl ebin/$(APP)
+ -mkdir -p ebintest
$(ERL) -make
ebin/$(APP): src/erlydtl/$(APP)
@@ -19,7 +20,7 @@ run:
test:
- $(ERL) -noshell -pa ebin \
+ $(ERL) -noshell -pa ebin -pa ebintest \
-s erlydtl_functional_tests run_tests \
-s erlydtl_dateformat_tests run_tests \
-s erlydtl_unittests run_tests \
@@ -28,5 +29,6 @@ test:
clean:
rm -fv ebin/*.beam
rm -fv ebin/$(APP)
+ rm -fv ebintest/*
rm -fv erl_crash.dump $(PARSER).erl
rm -fv examples/rendered_output/*
View
6 README
@@ -49,6 +49,12 @@ Options is a proplist possibly containing:
force_recompile - Recompile the module even if the source's checksum has not
changed. Useful for debugging.
+ locale - The locale used for template compile. Requires erlang_gettext. It
+ will ask gettext_server for the string value on the provided locale.
+ For example, adding {locale, "en_US"} will call {key2str, Key, "en_US"}
+ for all string marked as trans ({% trans "StringValue"%} on templates).
+ See README_I18N.
+
Usage (of a compiled template)
------------------------------
View
@@ -0,0 +1,47 @@
+Generate gettext infrastructure
+-------------------------------
+
+Erlydtl allows templates to use i18n features based on gettext. Standard po
+files can be used to generate i18ized templates. A template parser/po generator
+is also provided.
+
+ 1. In order to enable i18n you first, you'll need gettext library to be
+ available on your lib_path.
+
+ Library can be downloaded from http://github.com/noss/erlang-gettext
+
+ 2. Then you'll need to add a parse target on your makefile (or the script
+ used to trigger template reparsing) trans:
+
+ erl -pa ./ebin ./deps/*/ebin -noshell -s reloader -run i18n_manager \
+ generate_pos "en,es" "./views/*/*.html,./views/*.html"
+ rm -rf $(GETTEXT_DIR)/lang/default-old
+ mv $(GETTEXT_DIR)/lang/default $(GETTEXT_DIR)/lang/default-old
+ cp -rf $(GETTEXT_DIR)/lang/$(GETTEXT_TMP_NAME) $(GETTEXT_DIR)/lang/default
+ rm -rf $(GETTEXT_DIR)/lang/$(GETTEXT_TMP_NAME)/*
+
+ Mind that GETTEXT_DIR and GETTEXT_TMP_NAME must be bound to existing
+ directories. Args passed to i18n_manager:generate_pos are locales that
+ will be supported (generating dir structure and po files) and
+ directories where generator will search for template files including
+ trans tags.
+
+ 3. Before template parsing gettext server must be running and it must be
+ populated with the content of the po files. Consider adding this
+ snipplet to the code before template parsing
+
+ gettext_server:start(),
+ LoadPo =
+ fun(Lang)->
+ {_, Bin} = file:read_file("./lang/default/"++ Lang ++"/gettext.po"),
+ gettext:store_pofile(Lang, Bin)
+ end,
+ lists:map(LoadPo, ["es","en"]).
+
+ Here locales are the codes are provided to gettext. Those codes must be
+ a subset of the locales provided to po generation process.
+
+ 4. Update strings. Edit po files on $(GETTEXT_DIR)/lang/default/$(LOCALE)/gettext.po
+ translating msgstr to the translated version of their corresponding msgstr.
+
+ 5. Generate localized templates providing locale compile option.
View
@@ -0,0 +1 @@
+{% trans "Example String" %}
@@ -38,7 +38,7 @@
%% --------------------------------------------------------------------
%% Definitions
%% --------------------------------------------------------------------
--export([compile/2, compile/3]).
+-export([compile/2, compile/3, parse/1]).
-record(dtl_context, {
local_scopes = [],
@@ -51,7 +51,8 @@
reader = {file, read_file},
module = [],
compiler_options = [verbose, report_errors],
- force_recompile = false}).
+ force_recompile = false,
+ locale}).
-record(ast_info, {
dependencies = [],
@@ -148,7 +149,8 @@ init_dtl_context(File, Module, Options) ->
vars = proplists:get_value(vars, Options, Ctx#dtl_context.vars),
reader = proplists:get_value(reader, Options, Ctx#dtl_context.reader),
compiler_options = proplists:get_value(compiler_options, Options, Ctx#dtl_context.compiler_options),
- force_recompile = proplists:get_value(force_recompile, Options, Ctx#dtl_context.force_recompile)}.
+ force_recompile = proplists:get_value(force_recompile, Options, Ctx#dtl_context.force_recompile),
+ locale = proplists:get_value(locale, Options, Ctx#dtl_context.locale)}.
is_up_to_date(_, #dtl_context{force_recompile = true}) ->
@@ -321,6 +323,8 @@ body_ast(DjangoParseTree, Context, TreeWalker) ->
TreeWalkerAcc);
({'text', _Pos, String}, TreeWalkerAcc) ->
string_ast(String, TreeWalkerAcc);
+ ({'trans', {string_literal, _Pos, FormatString}}, TreeWalkerAcc) ->
+ translated_ast(FormatString, Context, TreeWalkerAcc);
({'string_literal', _Pos, String}, TreeWalkerAcc) ->
{{auto_escape(erl_syntax:string(unescape_string_literal(String)), Context),
#ast_info{}}, TreeWalkerAcc};
@@ -434,6 +438,12 @@ empty_ast(TreeWalker) ->
{{erl_syntax:list([]), #ast_info{}}, TreeWalker}.
+translated_ast(String,Context, TreeWalker) ->
+ NewStr = string:sub_string(String, 2, string:len(String) -1),
+ Locale = Context#dtl_context.locale,
+ LocalizedString = erlydtl_i18n:translate(NewStr,Locale),
+ {{erl_syntax:string(LocalizedString), #ast_info{}}, TreeWalker}.
+
string_ast(String, TreeWalker) ->
{{erl_syntax:string(String), #ast_info{}}, TreeWalker}. %% less verbose AST, better for development and debugging
% {{erl_syntax:binary([erl_syntax:binary_field(erl_syntax:integer(X)) || X <- String]), #ast_info{}}, TreeWalker}.
@@ -0,0 +1,17 @@
+%% Author: dave
+%% Created: Feb 25, 2010
+%% Description: Bridge between erlydtl compiler and gettext server
+-module(erlydtl_i18n).
+
+%%
+%% Include files
+%%
+%% Exported Functions
+%%
+-export([translate/2]).
+
+%%
+%% API Functions
+%%
+%% Makes i18n conversion using gettext
+translate(String, Locale) -> gettext:key2str(String, Locale).
@@ -93,7 +93,9 @@ Nonterminals
CustomTag
Args
-
+
+ TransTag
+
CallTag
CallWithTag.
@@ -135,13 +137,15 @@ Terminals
pipe
string_literal
text
+ trans_keyword
with_keyword.
Rootsymbol
Elements.
Elements -> '$empty' : [].
Elements -> Elements text : '$1' ++ ['$2'].
+Elements -> Elements TransTag : '$1' ++ ['$2'].
Elements -> Elements ValueBraced : '$1' ++ ['$2'].
Elements -> Elements ExtendsTag : '$1' ++ ['$2'].
Elements -> Elements IncludeTag : '$1' ++ ['$2'].
@@ -169,6 +173,8 @@ Value -> Literal : '$1'.
Variable -> identifier : {variable, '$1'}.
Variable -> Value dot identifier : {attribute, {'$3', '$1'}}.
+TransTag -> open_tag trans_keyword string_literal close_tag : {trans, '$3'}.
+
ExtendsTag -> open_tag extends_keyword string_literal close_tag : {extends, '$3'}.
IncludeTag -> open_tag include_keyword string_literal close_tag : {include, '$3'}.
NowTag -> open_tag now_keyword string_literal close_tag : {date, now, '$3'}.
@@ -62,7 +62,7 @@ scan([], Scanned, _, in_text) ->
"not", "or", "and", "comment", "endcomment", "cycle", "firstof",
"ifchanged", "ifequal", "endifequal", "ifnotequal", "endifnotequal",
"now", "regroup", "spaceless", "endspaceless", "ssi", "templatetag",
- "load", "call", "with"],
+ "load", "call", "with", "trans"],
Type = case lists:member(RevString, Keywords) of
true ->
list_to_atom(RevString ++ "_keyword");
@@ -0,0 +1,4 @@
+include ../../../../support/include.mk
+EBIN_DIR := ../../ebin
+
+all: $(EBIN_FILES_NO_DOCS)
@@ -0,0 +1,128 @@
+%% Author: dave
+%% Created: Feb 26, 2010
+%% Description: TODO: Add description to dets_generator
+-module(i18n_manager).
+
+%%
+%% Include files
+%%
+
+%% Exported Functions
+%%
+-export([generate_pos/1]).
+-define(EPOT_TABLE,epos).
+-define(EPOT_TABLE_FUZZY,epos_fuzzy).
+
+%%
+%% API Functions
+%%
+
+generate_pos([Lang,Files])->
+ io:format("~s -> ~s ~n",[Lang,Files]),
+ {ok, SplittedLocales} = string:tokens(Lang,","),
+ {ok, SplittedFiles} = string:tokens(Files, ","),
+ ProcessedFiles = sources_parser:parse(SplittedFiles),
+ io:format("Parsed tokens are ~p~n",[ProcessedFiles]),
+ BaseDir = "lang/default/",
+
+ PopulateTable = fun(Language)->
+ io:format("-------------------------Generating po file for ~s-------------------------~n",[Language]),
+ open_table(Language),
+ put(locale, Language),
+ insert_tokens(ProcessedFiles),
+
+ %%Recover already present translations
+ TranslationsForLanguage = po_scanner:scan(BaseDir ++ Language ++ "/gettext.po"),
+ io:format("Updating translations~n"),
+ insert_translations(TranslationsForLanguage),
+ Data = dets_data(),
+ io:format("Generating po file ~n"),
+ Fuzzy = dets_fuzzy(),
+ po_generator:generate_file(Language, Data, Fuzzy),
+ io:format("Closing files ~n"),
+ close_tables(Language),
+ io:format("All files closed ~n")
+ end,
+
+ lists:map(PopulateTable, SplittedLocales),
+ init:stop()
+ .
+
+
+%%
+%% Local Functions
+%%
+
+%% Open a temporal table for a given locale
+open_table(Locale)->
+ Dir = "./lang/tmp/" ++ Locale,
+ io:format("Creating dir ~s~n",[Dir]),
+ file:del_dir(Dir),
+ file:make_dir(Dir),
+ OpenTable = fun({TableName, TableFile}) ->
+ File = Dir ++ TableFile,
+ case dets:open_file(TableName, [{file, File}]) of
+ {ok,Ref} -> io:format("Opened DETS ~p ~p~n",[TableName,Ref]);
+ _Error -> io:format("Error opening DETS~p~n",[_Error])
+ end
+ end,
+
+ lists:map(OpenTable, [{?EPOT_TABLE,"/epot.dets"},{?EPOT_TABLE_FUZZY,"/epot_fuzzy.dets"}]).
+
+%%TODO better way to do cleanup
+close_tables(Locale) ->
+ %%dets:delete_all_objects(?EPOT_TABLE),
+ ok = dets:close(?EPOT_TABLE),
+ ok = dets:close(?EPOT_TABLE_FUZZY),
+ file:delete("./lang/tmp/" ++ Locale ++ "/epot.dets"),
+ file:delete("./lang/tmp/" ++ Locale ++ "/epot_fuzzy.dets").
+
+%%Get all data from dets table
+dets_data() -> dets:foldl(fun(E, Acc) -> [E|Acc] end, [], ?EPOT_TABLE).
+dets_fuzzy() -> dets:foldl(fun(E, Acc) -> [E|Acc] end, [], ?EPOT_TABLE_FUZZY).
+
+insert_tokens([]) -> noop;
+insert_tokens([{Id,{Fname,Line,_Col}}|Tail]) ->
+ insert_token(Id, Id, Fname, Line),
+ insert_tokens(Tail).
+
+insert_token(Id, Translation,Fname,Line)->
+ FileInfo = get_file_info(Id), %%File info are all files where this string is present
+ AllFileReferences = lists:sort( [{Fname,Line} | FileInfo] ),
+ dets:insert(?EPOT_TABLE, {Id, Translation,AllFileReferences}).
+
+insert_translations([]) -> noop;
+insert_translations(L = [H|T]) ->
+ %%io:format("Remaining ~p~n",[L]),
+ case H of
+ {comment, _} ->
+ %%Comments are skipped
+ insert_translations(T);
+ _Other ->
+ [{id,Id}, {str,Str}|Tail] = L,
+ insert_translation(Id,Str),
+ insert_translations(Tail)
+ end.
+
+insert_translation(Id, Translation) ->
+ io:format("Updating translation for ~p to ~p ~n",[Id,Translation]),
+ case Id of
+ [] ->
+ noop;
+ Id ->
+ case dets:lookup(?EPOT_TABLE,Id) of
+ [] ->
+ %%Fuzzy translation!
+ dets:insert(?EPOT_TABLE_FUZZY, {Id, Translation,fuzzy});
+ [{Id, _StoredTranslation,FileInfo}] ->
+ %%TODO check for translation unicity
+ io:format("Recovered translation for ~p ~p ~n",[Id,_StoredTranslation]),
+ dets:insert(?EPOT_TABLE, {Id, Translation,FileInfo})
+ end
+ end.
+
+get_file_info(Key) ->
+ case dets:lookup(?EPOT_TABLE, Key) of
+ [] -> [];
+ [{_,_,Finfo}|_] -> Finfo
+ end.
@@ -0,0 +1,57 @@
+%% Author: dave
+%% Created: Mar 1, 2010
+%% Description: Generates po files from dets tables, based on erlang gettext impl
+-module(po_generator).
+-define(ENDCOL, 72).
+%%
+%% Include files
+%%
+
+%%
+%% Exported Functions
+%%
+-export([generate_file/3]).
+
+%%
+%% API Functions
+%%
+generate_file(Lang,Items, Fuzzy) ->
+ Gettext_App_Name = "tmp",
+ GtxtDir = ".",
+ io:format("Opening po file"),
+ gettext_compile:open_po_file(Gettext_App_Name, GtxtDir, Lang),
+
+ gettext_compile:write_header(),
+ io:format("Writing entries~n"),
+ write_entries(Items),
+ io:format("Writing fuzzy entries~n"),
+ write_fuzzy_entries(Fuzzy),
+ gettext_compile:close_file().
+
+%%
+%% Local Functions
+%%
+write_entries(Items)->
+ Fd = get(fd),
+ F = fun({Id,Translation,Finfo}) ->
+ Fi = gettext_compile:fmt_fileinfo(Finfo),
+ io:format(Fd, "~n#: ~s~n", [Fi]),
+ file:write(Fd, "msgid \"\"\n"),
+ gettext_compile:write_pretty(Id),
+ file:write(Fd, "msgstr \"\"\n"),
+ gettext_compile:write_pretty(Translation)
+ end,
+ lists:foreach(F, Items).
+
+write_fuzzy_entries(Items) ->
+ Fd = get(fd),
+ file:write(Fd, "\n"),
+ F = fun({Id,Translation,_}) ->
+ file:write(Fd, "#, fuzzy\n"),
+ file:write(Fd, "msgid \"\"\n"),
+ gettext_compile:write_pretty(Id),
+ file:write(Fd, "msgstr \"\"\n"),
+ gettext_compile:write_pretty(Translation),
+ file:write(Fd, "\n")
+ end,
+ lists:foreach(F, Items).
Oops, something went wrong.

0 comments on commit 48fea22

Please sign in to comment.