@@ -4,3 +4,7 @@ lib/erlsdb/test/*.beam lib/*/ebin erlang_config.rb .depend_erlang.mf +bin/connect_local +bin/start_erl_local +bin/start_local +erl_crash.dump \ No newline at end of file .gitignore @@ -1,5 +1,10 @@ -ErlSDB README - +ErlSDB README +----------- + +That README is currently under construction. + +-- Eric + Summary ----------- Amazon SimpleDB is a web service for running queries on structured data @@ -44,29 +49,29 @@ Released under the GNU General Public License v2. Feedback -------- -Send patches, issues and comments to bhatti@plexobject.com. - +Send patches, issues and comments to bhatti@plexobject.com. + = BUILDING = -= Usage = -rake --tasks : to know all tasks - -You'll need a configuration in which you'll have to put things specific to your installation. -A template of file is generated the first time you launch 'rake' - -Here is some useful tasks: -rake erlang:modules: Compile all erlang source files (test included) -rake erlang:releases: Build a tarball for each rel file found in the directory tree -rake erlang:tests[name]: Run the eunit test for the application "name". If no name is given, - all applications are tested. If a file test.desc is find in the "app_name/test" directory, eunit will - use it as eunit test description. -rake erlang:edoc[name]: Build the document for an application or all if 'name' is omitted -rake otp:new_application[name]: Create a new application squeleton -rake otp:start_local[name]: Start a release in the developper environment - -rake otp:initial_targer[name,version]: Create a tarball with a 'ready' to deploy - an erlang system running the release. To use the system, you have to provide - 2 startup files in a directory named 'release_config'. Look at the sample in the - application sample_rake. -= Author = -Nicolas Charpentier <open_source@charpi.net> += Usage = +rake --tasks : to know all tasks + +You'll need a configuration in which you'll have to put things specific to your installation. +A template of file is generated the first time you launch 'rake' + +Here is some useful tasks: +rake erlang:modules: Compile all erlang source files (test included) +rake erlang:releases: Build a tarball for each rel file found in the directory tree +rake erlang:tests[name]: Run the eunit test for the application "name". If no name is given, + all applications are tested. If a file test.desc is find in the "app_name/test" directory, eunit will + use it as eunit test description. +rake erlang:edoc[name]: Build the document for an application or all if 'name' is omitted +rake otp:new_application[name]: Create a new application squeleton +rake otp:start_local[name]: Start a release in the developper environment + +rake otp:initial_targer[name,version]: Create a tarball with a 'ready' to deploy + an erlang system running the release. To use the system, you have to provide + 2 startup files in a directory named 'release_config'. Look at the sample in the + application sample_rake. += Author = +Nicolas Charpentier <open_source@charpi.net> README @@ -1,9 +1,10 @@ + %%%------------------------------------------------------------------- %%% @doc This header defines common record definitions %%% @end %%%------------------------------------------------------------------- --record(sdb_state, {access_key, secret_key, domain}). +-record(state, {access_key, secret_key}). -record(response, {box_usage, more_token, xml}). lib/erlsdb/include/erlsdb.hrl @@ -1,2 +1,2 @@ -ERL_FLAGS="-pa patches +K true -sname erlsdb -smp auto" +ERL_FLAGS="-pa patches +K true -sname erlsdb -smp auto " export ERL_FLAGS lib/erlsdb/release_config/startup.conf @@ -5,5 +5,5 @@ {modules, %MODULES%}, {mod, {erlsdb,[]}}, {registered, [erlsdb_sup]}, - {applications, [kernel, stdlib, sasl, crypto, xmerl]} + {applications, [kernel, stdlib, sasl, crypto, inets, xmerl]} ]}. lib/erlsdb/src/erlsdb.app.src @@ -72,9 +72,11 @@ %% @spec start(Type, StartArgs) -> {ok, Pid} | {ok, Pid, State} | {error, Reason} %% @end %%-------------------------------------------------------------------- -start(_Type, StartArgs) -> +start(_Type, _StartArgs) -> %case erlsdb_sup:start_link(StartArgs) of - case erlsdb_server:start_link(StartArgs) of + ID = param(access), + Secret = param(secret), + case erlsdb_server:start_link([ID, Secret]) of {ok, Pid} -> {ok, Pid}; Error -> @@ -258,3 +260,12 @@ delete_attributes(ItemName, AttributeNames) -> %%-------------------------------------------------------------------- stop(_State) -> ok. + + +param(Name, Default)-> + case application:get_env(?MODULE, Name) of + {ok, Value} -> Value; + _-> Default + end. + +param(Name) -> param(Name, nil). \ No newline at end of file lib/erlsdb/src/erlsdb.erl @@ -7,5 +7,6 @@ {sasl, "_"}, {crypto, "_"}, {xmerl, "_"}, + {inets, "_"}, {erlsdb, "_"}] }. lib/erlsdb/src/erlsdb.rel.src @@ -81,11 +81,9 @@ %% @spec start_link() -> {ok, pid()} | {error, Reason} %% @end %%-------------------------------------------------------------------- -start_link(InitialState) -> - crypto:start(), - inets:start(), +start_link(Args) -> %%?DEBUG("******* erlsdb_server:start_link/1 starting~n", [InitialState]), - gen_server:start_link({local, ?SERVER}, ?MODULE, InitialState, []). + gen_server:start_link({local, ?SERVER}, ?MODULE, Args, []). %%-------------------------------------------------------------------- %% @doc Stops the server. @@ -283,9 +281,9 @@ delete_attributes(ItemName, AttributeNames) -> %% ignore | %% {stop, Reason} %%-------------------------------------------------------------------- -init([InitialState]) -> - ?DEBUG("******* erlsdb_server:init/1 starting~n", [InitialState]), - {ok, InitialState}. +init([Access, Secret]) -> + ?DEBUG("******* erlsdb_server:init/1 starting~n", []), + {ok, #state{access_key=Access, secret_key=Secret}}. %%-------------------------------------------------------------------- %% Function: handle_call/3 @@ -297,7 +295,7 @@ init([InitialState]) -> %% {stop, Reason, Reply, State} | (terminate/2 is called) %% {stop, Reason, State} (terminate/2 is called) %%-------------------------------------------------------------------- -handle_call({list_domains, MoreToken, MaxNumberOfDomains}, _From, #sdb_state{access_key=AccessKey, secret_key = SecretKey} = State) -> +handle_call({list_domains, MoreToken, MaxNumberOfDomains}, _From, #state{access_key=AccessKey, secret_key = SecretKey} = State) -> ?DEBUG("******* erlsdb_server:list_domains BEGIN~n", []), Base = base_parameters("ListDomains", AccessKey), Base1 = if MoreToken == nil -> Base; true -> Base ++ [["MoreToken", MoreToken]] end, @@ -312,7 +310,7 @@ handle_call({list_domains, MoreToken, MaxNumberOfDomains}, _From, #sdb_state{acc {reply, Response, State}; -handle_call({get_attributes, ItemName, AttributeNames}, _From, #sdb_state{access_key=AccessKey, secret_key = SecretKey, domain = Domain} = State) -> +handle_call({get_attributes, Domain, ItemName, AttributeNames}, _From, #state{access_key=AccessKey, secret_key = SecretKey} = State) -> ?DEBUG("******* erlsdb_server:get_attributes~n", []), Base = [["DomainName", erlsdb_util:url_encode(Domain)], ["ItemName", erlsdb_util:url_encode(ItemName)]| @@ -355,36 +353,36 @@ handle_cast(stop, State) -> {stop, normal, State}; -handle_cast({create_domain}, #sdb_state{access_key=AccessKey, secret_key = SecretKey, domain = Domain} = State) -> +handle_cast({create_domain, Domain}, #state{access_key=AccessKey, secret_key = SecretKey } = State) -> ?DEBUG("******* erlsdb_server:create_domain ~p~n", [Domain]), - Base = [["DomainName", erlsdb_util:url_encode(Domain)]| + Base = [{"DomainName", erlsdb_util:url_encode(Domain)}| base_parameters("CreateDomain", AccessKey)], rest_request(SecretKey, Base, fun(_Xml) -> nil end), {noreply, State}; -handle_cast({delete_domain}, #sdb_state{access_key=AccessKey, secret_key = SecretKey, domain = Domain} = State) -> +handle_cast({delete_domain, Domain}, #state{access_key=AccessKey, secret_key = SecretKey } = State) -> ?DEBUG("******* erlsdb_server:delete_domain ~p~n", [Domain]), - Base = [["DomainName", erlsdb_util:url_encode(Domain)]| + Base = [{"DomainName", erlsdb_util:url_encode(Domain)}| base_parameters("DeleteDomain", AccessKey)], rest_request(SecretKey, Base, fun(_Xml) -> nil end), {noreply, State}; -handle_cast({put_attributes, ItemName, Attributes, Replace}, #sdb_state{access_key=AccessKey, secret_key = SecretKey, domain = Domain} = State) -> +handle_cast({put_attributes,Domain, ItemName, Attributes, Replace}, #state{access_key=AccessKey, secret_key = SecretKey } = State) -> ?DEBUG("******* erlsdb_server:put_attributes ~p~n", [Domain, ItemName, Attributes]), - Base = [["DomainName", erlsdb_util:url_encode(Domain)], - ["ItemName", erlsdb_util:url_encode(ItemName)]| + Base = [{"DomainName", erlsdb_util:url_encode(Domain)}, + {"ItemName", erlsdb_util:url_encode(ItemName)}| base_parameters("PutAttributes", AccessKey)] ++ erlsdb_util:encode_attributes(Attributes), Base1 = if Replace == false -> Base; true -> Base ++ [["Replace", "true"]] end, rest_request(SecretKey, Base1, fun(_Xml) -> nil end), {noreply, State}; -handle_cast({delete_attributes, ItemName, AttributeNames}, #sdb_state{access_key=AccessKey, secret_key = SecretKey, domain = Domain} = State) -> +handle_cast({delete_attributes, Domain, ItemName, AttributeNames}, #state{access_key=AccessKey, secret_key = SecretKey} = State) -> ?DEBUG("******* erlsdb_server:delete_attributes ~p~n", [Domain, ItemName, AttributeNames]), - Base = [["DomainName", erlsdb_util:url_encode(Domain)], - ["ItemName", erlsdb_util:url_encode(ItemName)]| + Base = [{"DomainName", erlsdb_util:url_encode(Domain)}, + {"ItemName", erlsdb_util:url_encode(ItemName)}| base_parameters("DeleteAttributes", AccessKey)] ++ erlsdb_util:encode_attribute_names(AttributeNames), rest_request(SecretKey, Base, fun(_Xml) -> nil end), @@ -443,7 +441,7 @@ rest_request(SecretKey, Params, XmlParserFunc) -> end; {error, ErrorMessage} -> case ErrorMessage of - timeout -> + Error when Error == nxdomain orelse Error == timeout -> ?DEBUG("URL ~p Timedout, retrying~n", [Url]), erlsdb_util:sleep(1000), rest_request(SecretKey, Params, XmlParserFunc); @@ -455,34 +453,34 @@ rest_request(SecretKey, Params, XmlParserFunc) -> query_string(SecretKey, Params) -> Params1 = lists:sort( - fun([Elem1, _], [Elem2, _]) -> - string:to_lower(Elem1) > string:to_lower(Elem2) end, + fun({A, _}, {X, _}) -> A < X end, Params), - {QueryStr, SignatureData} = - lists:foldr(fun query_string1/2, {"", ""}, Params1), - QueryStr ++ "Signature=" ++ erlsdb_util:url_encode(signature(SecretKey, SignatureData)). + QueryStr = + string:join(lists:foldr(fun query_string1/2, [], Params1), "&"), + SignatureData = "GET\nsdb.amazonaws.com\n/\n" ++ QueryStr, + QueryStr ++ "&Signature=" ++ erlsdb_util:url_encode(signature(SecretKey, SignatureData)). -query_string1([Key, Value], {QueryStr, SignatureData}) -> - QueryStr1 = QueryStr ++ Key ++ "=" ++ erlsdb_util:url_encode(Value) ++ "&", - SignatureData1 = SignatureData ++ Key ++ Value, - {QueryStr1, SignatureData1}. +query_string1({Key, Value}, Query) -> + [Key ++ "=" ++ erlsdb_util:url_encode(Value) | Query]. %%% % Returns HMAC encoded access key %%% signature(SecretKey, Data) -> - erlsdb_util:hmac(SecretKey, Data). + binary_to_list( + base64:encode(crypto:sha_mac(SecretKey, Data))). base_parameters(Action, AccessKey) -> - [["Action", Action], - ["AWSAccessKeyId", AccessKey], - ["Version", version()], - ["SignatureVersion", "1"], - ["Timestamp", erlsdb_util:timestamp()]]. + [{"Action", Action}, + {"AWSAccessKeyId", AccessKey}, + {"Version", version()}, + {"SignatureVersion", "2"}, + {"SignatureMethod", "HmacSHA1"}, + {"Timestamp", erlsdb_util:create_timestamp()}]. @@ -496,5 +494,5 @@ uri() -> version() -> %"2007-11-07". - "2007-02-09". + "2007-11-07". lib/erlsdb/src/erlsdb_server.erl @@ -32,9 +32,7 @@ encode_attribute_names/2, two_digit/1, abs_two_digit/1, - timestamp/0, - gmt_difference/0, - hmac/2, + create_timestamp/0, xml_values/1, xml_values/2, xml_names_values/1, @@ -120,55 +118,15 @@ abs_two_digit(X) when X >= 0 -> %% @spec encode_attributes(Attributes) -> ok %% @end %%-------------------------------------------------------------------- -timestamp() -> - {{_, _, _}, {_LocalHour, _LocalMin, _}} = LocalDateTime = calendar:local_time(), - [{{Year, Month, Day}, {Hour, Min, Sec}}] = - calendar:local_time_to_universal_time_dst(LocalDateTime), - Z = gmt_difference(), - integer_to_list(Year) ++ "-" ++ two_digit(Month) ++ "-" ++ two_digit(Day) - ++ "T" ++ two_digit(Hour) ++ ":" ++ two_digit(Min) ++ ":" ++ - two_digit(Sec) ++ Z. +% lifted from http://code.google.com/p/erlawys/source/browse/trunk/src/aws_util.erl +create_timestamp() -> create_timestamp(calendar:now_to_universal_time(now())). +create_timestamp({{Y, M, D}, {H, Mn, S}}) -> + to_str(Y) ++ "-" ++ to_str(M) ++ "-" ++ to_str(D) ++ "T" ++ + to_str(H) ++ ":" ++ to_str(Mn)++ ":" ++ to_str(S) ++ "Z". +add_zeros(L) -> if length(L) == 1 -> [$0|L]; true -> L end. +to_str(L) -> add_zeros(integer_to_list(L)). -%%-------------------------------------------------------------------- -%% @doc gmt_difference -- TODO -%% <pre> -%% Types: -%% </pre> -%% @spec gmt_difference() -> string -%% @end -%%-------------------------------------------------------------------- -gmt_difference() -> - UTC = calendar:universal_time(), - Local = calendar:universal_time_to_local_time(UTC), - gmt_difference((calendar:datetime_to_gregorian_seconds(Local) - calendar:datetime_to_gregorian_seconds(UTC)) / 3600). -gmt_difference(Diff) when Diff < 0 -> - gmt_difference(-Diff, "-"); -gmt_difference(Diff) -> - gmt_difference(Diff, "+"). - -gmt_difference(Diff, Sign) when Diff < 10 -> - gmt_difference1(float_to_list(Diff), Sign ++ "0"); -gmt_difference(Diff, Sign) -> - gmt_difference1(float_to_list(Diff), Sign). -gmt_difference1(StrDiff, SignZero) -> - Index = string:chr(StrDiff, $.), - SignZero ++ string:substr(StrDiff, 1, Index-1) ++ ":" ++ string:substr(StrDiff, Index+1, 2). - - -%%-------------------------------------------------------------------- -%% @doc hmac -%% <pre> -%% Types: -%% SecretKey = string -%% Data = string -%% </pre> -%% @spec hmac(SecretKey, Data) -> string -%% @end -%%-------------------------------------------------------------------- -hmac(SecretKey, Data) -> - base64:encode( - binary_to_list(crypto:sha_mac(SecretKey, Data))). %%-------------------------------------------------------------------- @@ -216,7 +174,7 @@ sleep(T) -> %%-------------------------------------------------------------------- -%% @doc url_encode - borrowed from CouchDB +%% @doc url_encode - lifted from the ever precious yaws_utils.erl %% <pre> %% Types: %% String @@ -232,17 +190,33 @@ url_encode([H|T]) -> [H|url_encode(T)]; H >= $0, $9 >= H -> [H|url_encode(T)]; - H == $_; H == $.; H == $-; H == $: -> + H == $_; H == $.; H == $-; H == $/ -> % FIXME: more.. [H|url_encode(T)]; true -> - case lists:flatten(io_lib:format("~.16.0B", [H])) of + case integer_to_hex(H) of [X, Y] -> [$%, X, Y | url_encode(T)]; [X] -> [$%, $0, X | url_encode(T)] end - end; + end; + url_encode([]) -> []. +integer_to_hex(I) -> + case catch erlang:integer_to_list(I, 16) of + {'EXIT', _} -> + old_integer_to_hex(I); + Int -> + Int + end. + +old_integer_to_hex(I) when I<10 -> + integer_to_list(I); +old_integer_to_hex(I) when I<16 -> + [I-10+$A]; +old_integer_to_hex(I) when I>=16 -> + N = trunc(I/16), + old_integer_to_hex(N) ++ old_integer_to_hex(I rem 16). lib/erlsdb/src/erlsdb_util.erl @@ -46,7 +46,7 @@ test_list_domains() -> erlsdb:list_domains(). test_create_test_domain() -> - erlsdb:create_domain(), + erlsdb:create_domain(test_domain()), {ok, List, _} = erlsdb:list_domains(), Domain = test_domain(), [Domain] = lists:filter( @@ -55,7 +55,7 @@ test_create_test_domain() -> test_delete_test_domain() -> - erlsdb:delete_domain(), + erlsdb:delete_domain(test_domain()), {ok, List, _} = erlsdb:list_domains(), Domain = test_domain(), [] = lists:filter( @@ -63,7 +63,7 @@ test_delete_test_domain() -> List). test_replace_get_attributes() -> - erlsdb:create_domain(), + erlsdb:create_domain(test_domain()), Attributes = lists:sort([ ["StreetAddress", "705 5th Ave"], ["City", "Seattle"], @@ -80,7 +80,7 @@ test_replace_get_attributes() -> end. test_replace_delete_attributes() -> - erlsdb:create_domain(), + erlsdb:create_domain(test_domain()), Attributes = lists:sort([ ["StreetAddress", "705 5th Ave"], ["City", "Seattle"], @@ -98,7 +98,7 @@ test_replace_delete_attributes() -> io:format("Unexpected response while getting attributes after delete ~p~n", [Response]) end. -test_gmt_difference([AccessKey, SecretKey]) -> +test_gmt_difference() -> "-08:00" = erlsdb_util:gmt_difference(). test([AccessKey, SecretKey]) -> @@ -106,14 +106,9 @@ test([AccessKey, SecretKey]) -> %debug_helper:trace(erlsdb_server), %%%application:load(erlsdb), %%%application:start(erlsdb), - Response = erlsdb:start(type, - [#sdb_state{ - access_key = AccessKey, - secret_key = SecretKey, - domain = test_domain() - } - ]), + Response = erlsdb:start(type, [AccessKey,SecretKey]), io:format("Test Started server ~p~n", [Response]), + test_list_domains(), test_create_test_domain(), test_delete_test_domain(), test_replace_get_attributes(), lib/erlsdb/test/erlsdb_test.erl a3576e1a71307dd06026a855058fd33b27c2218b Eric Cestari ecestari@mac.com http://github.com/cstar/erlsdb/commit/8d8ef3e6378c5521f03c177467acc775b22c21e0 8d8ef3e6378c5521f03c177467acc775b22c21e0 2009-02-23T03:36:31-08:00 2009-02-23T03:36:31-08:00 list_domains known to work updated signature to version 2 Now AccessKey and SecretKey are OTP application params 0fcd5952b24a9624f5c46db492b59cfdc9dd2b0a Eric Cestari ecestari@mac.com