@@ -2,3 +2,4 @@ require 'rake' task :default => ["erlang:compile", "erlang:releases"] +task :install => ["erlang:install"] \ No newline at end of file Rakefile @@ -8,7 +8,7 @@ -module(s3). -behaviour(application). - +-define(TIMEOUT, 15000). %%-------------------------------------------------------------------- %% External exports %%-------------------------------------------------------------------- @@ -21,7 +21,7 @@ %% API -export([ read_term/2, write_term/3, - list_buckets/0, create_bucket/1, delete_bucket/1, link_to/3, head/2, policy/1, get_object/3, get_objects/2, + list_buckets/0, create_bucket/1, delete_bucket/1, link_to/3, head/2, policy/1, get_object/3, get_objects/3, get_objects/2, list_objects/2, list_objects/1, write_object/4, write_object/5, read_object/2, read_object/3, delete_object/2 ]). %timer:tc(s3,get_objects,["ejabberd.conf", [{maxkeys, 20}]]). @@ -36,8 +36,8 @@ start(_Type, _StartArgs) -> ID = get(access, "AMAZON_ACCESS_KEY_ID"), Secret = get(secret, "AMAZON_SECRET_ACCESS_KEY"), - SSL = param(ssl, true), - N = param(workers, 5), + SSL = param(ssl, false), + N = param(workers, 2), Timeout = param(timeout, nil), Port = if SSL == true -> ssl:start(), @@ -104,23 +104,34 @@ delete_object (Bucket, Key) -> get_object (From, Bucket, Key) -> Pid = s3sup:get_random_pid(), - {ok, {Content, Headers}} = gen_server:call(Pid, {get, Bucket, Key}), - From ! {s3object, Content}. + {ok, {Content, _Headers}} = gen_server:call(Pid, {get, Bucket, Key}), + From ! {s3object, Key, Content}. % Gets objects in // from S3. %% option example: [{delimiter, "/"},{maxkeys,10},{prefix,"/foo"}] get_objects(Bucket, Options)-> + get_objects(Bucket, Options, ?TIMEOUT). +get_objects(Bucket, Options, Timeout)-> {ok, Objects} = list_objects (Bucket, Options ), + timer:send_after(Timeout, self(), timeout), lists:foreach(fun({object_info, {"Key", Key}, _, _, _})-> spawn(?MODULE, get_object,[self(), Bucket, Key ]) end, Objects), - R = lists:foldl(fun(_N, Acc)-> + R = lists:foldl( + fun (_N, timeout)->timeout; + (_N, Acc)-> receive - {s3object, R} -> - [R|Acc] + {s3object, K, R} -> + [{K, R}|Acc]; + timeout-> + timeout end + end,[], Objects), - {ok, R}. + case R of + timeout -> {error,timeout}; + R -> {ok, R} + end. %% option example: [{delimiter, "/"},{maxkeys,10},{prefix,"/foo"}] list_objects (Bucket, Options ) -> lib/s3/src/s3.erl @@ -26,7 +26,7 @@ -include("../include/s3.hrl"). -record(state, {ssl,access_key, secret_key, pending, timeout=?TIMEOUT}). --record(request, {pid, callback, code, headers=[], content=[]}). +-record(request, {pid, callback, started, code, headers=[], content=[]}). %%==================================================================== %% External functions %%==================================================================== @@ -168,8 +168,9 @@ handle_info({ibrowse_async_response,RequestId,Body },State = #state{pending=P}) end; handle_info({ibrowse_async_response_end,RequestId}, State = #state{pending=P})-> case gb_trees:lookup(RequestId,P) of - {value,R} -> + {value,#request{started=_Started}=R} -> handle_http_response(R), + %io:format("Query took ~p ms~n", [timer:now_diff(now(), Started)/1000]), {noreply,State#state{pending=gb_trees:delete(RequestId, P)}}; none -> {noreply,State} %% the requestid isn't here, probably the request was deleted after a timeout @@ -259,11 +260,16 @@ buildContentHeaders( Contents, AdditionalHeaders ) -> buildOptions(<<>>, _ContentType, SSL)-> [{stream_to, self()}, {is_ssl, SSL}, {ssl_options, []}]; buildOptions(Content, ContentType,SSL)-> - [{content_length, integer_to_list(size(Content))}, + [{content_length, content_length(Content)}, {content_type, ContentType}, {is_ssl, SSL},{ssl_options, []}, {stream_to, self()}]. - + +content_length(Content) when is_binary(Content)-> + integer_to_list(size(Content)); +content_length(Content) when is_list(Content)-> + integer_to_list(length(Content)). + genericRequest(From, #state{ssl=SSL, access_key=AKI, secret_key=SAK, timeout=Timeout, pending=P }=State, Method, Bucket, Path, QueryParams, AdditionalHeaders,Contents, ContentType, Callback ) -> Date = httpd_util:rfc1123_date(), @@ -279,10 +285,10 @@ genericRequest(From, #state{ssl=SSL, access_key=AKI, secret_key=SAK, timeout=Tim {"Date", Date } | OriginalHeaders ], Options = buildOptions(Contents, ContentType, SSL), - %io:format("Sending request ~p~n", [Url]), + io:format("Sending request ~p~n", [Url]), case ibrowse:send_req(Url, Headers, Method, Contents,Options, Timeout) of {ibrowse_req_id,RequestId} -> - Pendings = gb_trees:insert(RequestId,#request{pid=From,callback=Callback},P), + Pendings = gb_trees:insert(RequestId,#request{pid=From,started=now(), callback=Callback},P), {noreply, State#state{pending=Pendings}}; {error,E} when E =:= retry_later orelse E =:= conn_failed -> io:format("Waiting on retry Error : ~p, Pid : ~p~n", [E, self()]), lib/s3/src/s3server.erl @@ -1,2 +1,2 @@ -{vsn,"0.4"}. +{vsn,"0.5"}. {release_name,"initial"}. \ No newline at end of file lib/s3/vsn.config @@ -98,11 +98,16 @@ namespace :erlang do erlang_source_dependencies + erlang_test_dependencies end + def erlang_home + @erlang_home||=IO.popen("#{ERL_TOP}/bin/erl -noinput -noshell -eval 'io:format(code:root_dir()).' -s init stop").readlines[0] + end + def run_application_test(application, directories) application_name = application.pathmap("%f").ext("") run_script("run_test",["application", application_name] + directories) end - + + def run_description_test(description, directories) run_script("run_test",["file", description] + directories) end @@ -373,6 +378,16 @@ namespace :erlang do task :default => [:compile] + + desc "Installs in local erl repository : #{erlang_home}" + task :install => [:compile] do |t| + FileList.new('lib/*').each do |dir| + vsn = extract_version_information("#{dir}/vsn.config","vsn").gsub("\"","") + sh "cp -R #{dir} #{erlang_home}/#{dir}-#{vsn}" + end + end + + desc "Build Application packages" task :package => [:applications] do ERL_APPLICATIONS.each do |application| rakelib/erlang.rake 680185e4594f67b4e63cf98f84c73d2fea3edcd7 Eric Cestari ecestari@mac.com http://github.com/cstar/erls3/commit/6c44f47ede9ddc428d0305d254e1c8309115ca4b 6c44f47ede9ddc428d0305d254e1c8309115ca4b 2009-04-02T04:01:49-07:00 2009-04-02T04:01:49-07:00 Added rake install task. 8324328dfddccb4f2bd5651240435b7407076ea7 Eric Cestari ecestari@mac.com