Permalink
Browse files

merge

  • Loading branch information...
japerk committed May 20, 2010
2 parents d6a0c0d + 5a2452e commit 0abd90f2c27b2c83a789baccd3b91b4d2c7bab12
Showing with 125 additions and 54 deletions.
  1. +2 −3 README.txt
  2. +16 −0 src/erlaws.erl
  3. +104 −41 src/erlaws_s3.erl
  4. +1 −1 src/erlaws_sdb.erl
  5. +2 −9 src/erlaws_util.erl
View
@@ -1,15 +1,13 @@
Erlaws provides Erlang interfaces to various Amazon WebService offerings.
-This code is no longer maintained, so feel free to fork it.
-
-- original documentation from Google Code wiki --
= Description =
Erlaws is a collection of client implementations of Amazon's WebServices offerings. Currently there are clients for S3, SQS and SDB.
= Build =
-Check out the latest code from svn and issue {{{erl -make}}} to build the sources.
+Issue `erl -make` to build the sources.
= Usage =
@@ -36,6 +34,7 @@ Here a short overview:
* list_contents/1
* list_contents/2
* put_object/5
+ * put_file/5
* get_object/2
* info_object/2
* delete_object/2
View
@@ -0,0 +1,16 @@
+-module(erlaws).
+
+-behaviour(application).
+
+-export([start/0, start/2, stop/1]).
+
+start() ->
+ application:start(sasl),
+ crypto:start(),
+ inets:start().
+
+start(_Type, _Args) ->
+ erlaws:start().
+
+stop(_State) ->
+ ok.
View
@@ -10,19 +10,21 @@
%% API
-export([list_buckets/0, create_bucket/1, create_bucket/2, delete_bucket/1]).
--export([list_contents/1, list_contents/2, put_object/5, get_object/2]).
+-export([list_contents/1, list_contents/2, put_object/5, put_file/5, get_object/2]).
-export([info_object/2, delete_object/2]).
%% include record definitions
-include_lib("xmerl/include/xmerl.hrl").
+-include_lib("kernel/include/file.hrl").
-include("../include/erlaws.hrl").
%% macro definitions
--define( AWS_S3_HOST, "s3.amazonaws.com").
--define( NR_OF_RETRIES, 3).
--define( CALL_TIMEOUT, indefinite).
--define( S3_REQ_ID_HEADER, "x-amz-request-id").
--define( PREFIX_XPATH, "//CommonPrefixes/Prefix/text()").
+-define(AWS_S3_HOST, "s3.amazonaws.com").
+-define(NR_OF_RETRIES, 3).
+-define(CALL_TIMEOUT, indefinite).
+-define(S3_REQ_ID_HEADER, "x-amz-request-id").
+-define(PREFIX_XPATH, "//CommonPrefixes/Prefix/text()").
+-define(CHUNK_SIZE, 8 * 1024).
%% Returns a list of all of the buckets owned by the authenticated sender
%% of the request.
@@ -32,7 +34,7 @@
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
list_buckets() ->
- try genericRequest(get, "", "", "", [], "", <<>>) of
+ try genericRequest(get, "", "", [], [], [], <<>>) of
{ok, Headers, Body} ->
{XmlDoc, _Rest} = xmerl_scan:string(binary_to_list(Body)),
TextNodes = xmerl_xpath:string("//Bucket/Name/text()", XmlDoc),
@@ -55,7 +57,7 @@ list_buckets() ->
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
create_bucket(Bucket) ->
- try genericRequest(put, Bucket, "", "", [], "", <<>>) of
+ try genericRequest(put, Bucket, "", [], [], [], <<>>) of
{ok, Headers, _Body} ->
RequestId = case lists:keytake("x-amz-request-id", 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -82,7 +84,7 @@ create_bucket(Bucket, eu) ->
LCfg = <<"<CreateBucketConfiguration>
<LocationConstraint>EU</LocationConstraint>
</CreateBucketConfiguration>">>,
- try genericRequest(put, Bucket, "", "", [], "", LCfg) of
+ try genericRequest(put, Bucket, "", [], [], [], LCfg) of
{ok, Headers, _Body} ->
RequestId = case lists:keytake("x-amz-request-id", 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -100,7 +102,7 @@ create_bucket(Bucket, eu) ->
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
delete_bucket(Bucket) ->
- try genericRequest(delete, Bucket, "", "", [], "", <<>>) of
+ try genericRequest(delete, Bucket, "", [], [], [], <<>>) of
{ok, Headers, _Body} ->
RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -136,7 +138,7 @@ list_contents(Bucket) ->
%%
list_contents(Bucket, Options) when is_list(Options) ->
QueryParameters = [makeParam(X) || X <- Options],
- try genericRequest(get, Bucket, "", QueryParameters, [], "", <<>>) of
+ try genericRequest(get, Bucket, "", QueryParameters, [], [], <<>>) of
{ok, Headers, Body} ->
{XmlDoc, _Rest} = xmerl_scan:string(binary_to_list(Body)),
[Truncated| _Tail] = xmerl_xpath:string("//IsTruncated/text()",
@@ -157,16 +159,32 @@ list_contents(Bucket, Options) when is_list(Options) ->
{error, Descr}
end.
-%% Uploads data for key.
+%% Uploads data for key. Backwards-compatible version.
%%
%% Spec: put_object(Bucket::string(), Key::string(), Data::binary(),
%% ContentType::string(),
%% Metadata::[{Key::string(), Value::string()}]) ->
%% {ok, #s3_object_info(key=Key::string(), size=Size::integer())} |
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
-put_object(Bucket, Key, Data, ContentType, Metadata) ->
- try genericRequest(put, Bucket, Key, [], Metadata, ContentType, Data) of
+put_object(Bucket, Key, Data, ContentType, Metadata) when is_integer(hd(ContentType)) ->
+ put_object(Bucket, Key, Data, [{"Content-Type", ContentType}], Metadata);
+
+%% Uploads data for key. More general version.
+%%
+%% Spec: put_object(Bucket::string(), Key::string(), Data::binary(),
+%% HTTPHeaders::[{Key::string(), Value::string()}]
+%% Metadata::[{Key::string(), Value::string()}]) ->
+%% {ok, #s3_object_info(key=Key::string(), size=Size::integer())} |
+%% {error, {Code::string(), Msg::string(), ReqId::string()}}
+%%
+%% EXAMPLE:
+%% S3 = ?MODULE:new(...), % Fill it according to your preferences.
+%% S3:put_object("someBucket", "filename.js", <<"...">>, [{"Content-Type", "application/x-javascript; charset=\"utf-8\""},{"Cache-Control", "max-age=86400"},{"x-amz-acl", "public-read"}], [{"name", "metavalue"}]).
+%%
+%%
+put_object(Bucket, Key, Data, HTTPHeaders, Metadata) ->
+ try genericRequest(put, Bucket, Key, [], Metadata, HTTPHeaders, Data) of
{ok, Headers, _Body} ->
RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -176,6 +194,33 @@ put_object(Bucket, Key, Data, ContentType, Metadata) ->
throw:{error, Descr} ->
{error, Descr}
end.
+
+put_file(Bucket, Key, FileName, ContentType, Metadata) ->
+ Date = httpd_util:rfc1123_date(erlang:localtime()),
+ {FileSize, File} = openAndGetFileSize(FileName),
+ Headers =
+ buildContentHeaders(FileSize) ++
+ buildMetadataHeaders(Metadata),
+ Signature = sign(AWS_SEC_KEY,
+ stringToSign("PUT", "", ContentType, Date,
+ Bucket, Key, Headers)),
+ FinalHeaders = [ {"Authorization", "AWS " ++ AWS_KEY ++ ":" ++ Signature },
+ {"Host", buildHost(Bucket) },
+ {"Date", Date },
+ {"Content-Type", ContentType}
+ | Headers ],
+ Payload =
+ lists:append(
+ ["PUT /", Key, " HTTP/1.1\n",
+ lists:flatten([lists:append([K, ": ", V, "\n"]) ||
+ {K, V} <- lists:reverse(FinalHeaders)]),
+ "\n"]),
+ {ok, Socket} = gen_tcp:connect(?AWS_S3_HOST, 80,
+ [binary, {active, false}, {packet, 0}]),
+ gen_tcp:send(Socket, list_to_binary(Payload)),
+ sendData(Socket, File),
+ gen_tcp:close(Socket),
+ file:close(File).
%% Retrieves the data associated with the given key.
%%
@@ -184,7 +229,7 @@ put_object(Bucket, Key, Data, ContentType, Metadata) ->
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
get_object(Bucket, Key) ->
- try genericRequest(get, Bucket, Key, [], [], "", <<>>) of
+ try genericRequest(get, Bucket, Key, [], [], [], <<>>) of
{ok, Headers, Body} ->
RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -202,7 +247,7 @@ get_object(Bucket, Key) ->
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
info_object(Bucket, Key) ->
- try genericRequest(head, Bucket, Key, [], [], "", <<>>) of
+ try genericRequest(head, Bucket, Key, [], [], [], <<>>) of
{ok, Headers, _Body} ->
io:format("Headers: ~p~n", [Headers]),
MetadataList = [{string:substr(MKey, 12), Value} || {MKey, Value} <- Headers, string:str(MKey, "x-amz-meta") == 1],
@@ -222,7 +267,7 @@ info_object(Bucket, Key) ->
%% {error, {Code::string(), Msg::string(), ReqId::string()}}
%%
delete_object(Bucket, Key) ->
- try genericRequest(delete, Bucket, Key, [], [], "", <<>>) of
+ try genericRequest(delete, Bucket, Key, [], [], [], <<>>) of
{ok, Headers, _Body} ->
RequestId = case lists:keytake(?S3_REQ_ID_HEADER, 1, Headers) of
{value, {_, ReqId}, _} -> ReqId;
@@ -256,8 +301,7 @@ canonicalizeAmzHeaders( Headers ) ->
Strings = lists:map(
fun mkHdr/1,
collapse(XAmzHeaders)),
- erlaws_util:mkEnumeration( lists:map( fun (String) -> String ++ "\n" end,
- Strings), "").
+ erlaws_util:mkEnumeration( [[String, "\n"] || String <- Strings], "").
canonicalizeResource ( "", "" ) -> "/";
canonicalizeResource ( Bucket, "" ) -> "/" ++ Bucket ++ "/";
@@ -297,19 +341,19 @@ buildUrl(Bucket,Path,QueryParams) ->
buildProtocol() ++ Bucket ++ "." ++ ?AWS_S3_HOST ++ "/" ++ Path ++
erlaws_util:queryParams(QueryParams).
-buildContentHeaders( <<>>, _ ) -> [];
-buildContentHeaders( Contents, ContentType ) ->
+buildContentHeaders(Contents) when is_integer(Contents) ->
+ [{"Content-Length", integer_to_list(Contents)}];
+% Detect gzip header and put appropriate Content-Encoding. Questionable?..
+buildContentHeaders(<<16#1f, 16#8b, _/binary>> = Contents) ->
[{"Content-Length", integer_to_list(size(Contents))},
- {"Content-Type", ContentType}].
+ {"Content-Encoding", "gzip"}];
+buildContentHeaders(Contents) ->
+ [{"Content-Length", integer_to_list(size(Contents))}].
buildMetadataHeaders(Metadata) ->
- buildMetadataHeaders(Metadata, []).
-
-buildMetadataHeaders([], Acc) ->
- Acc;
-buildMetadataHeaders([{Key, Value}|Tail], Acc) ->
- buildMetadataHeaders(Tail, [{string:to_lower("x-amz-meta-"++Key), Value}
- | Acc]).
+ lists:foldl(fun({Key, Value}, Acc) ->
+ [{string:to_lower("x-amz-meta-"++Key), Value} | Acc]
+ end, [], Metadata).
buildContentMD5Header(ContentMD5) ->
case ContentMD5 of
@@ -325,16 +369,15 @@ stringToSign ( Verb, ContentMD5, ContentType, Date, Bucket, Path,
canonicalizeResource(Bucket, Path).
sign (Key,Data) ->
- %io:format("StringToSign:~n ~p~n", [Data]),
binary_to_list( base64:encode( crypto:sha_mac(Key,Data) ) ).
genericRequest( Method, Bucket, Path, QueryParams, Metadata,
- ContentType, Body ) ->
+ HTTPHeaders, Body ) ->
genericRequest( Method, Bucket, Path, QueryParams, Metadata,
- ContentType, Body, ?NR_OF_RETRIES).
+ HTTPHeaders, Body, ?NR_OF_RETRIES).
genericRequest( Method, Bucket, Path, QueryParams, Metadata,
- ContentType, Body, NrOfRetries) ->
+ HTTPHeaders, Body, NrOfRetries) ->
Date = httpd_util:rfc1123_date(erlang:localtime()),
MethodString = string:to_upper( atom_to_list(Method) ),
Url = buildUrl(Bucket,Path,QueryParams),
@@ -344,14 +387,21 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
_ -> binary_to_list(base64:encode(erlang:md5(Body)))
end,
- Headers = buildContentHeaders( Body, ContentType ) ++
+ Headers =
+ buildContentHeaders(Body) ++
buildMetadataHeaders(Metadata) ++
- buildContentMD5Header(ContentMD5),
+ buildContentMD5Header(ContentMD5) ++
+ HTTPHeaders,
+
+ ContentType = case [Value || {"Content-Type", Value} <- HTTPHeaders] of
+ [CT|_] -> CT;
+ [] -> ""
+ end,
{AccessKey, SecretAccessKey } = {AWS_KEY, AWS_SEC_KEY},
Signature = sign(SecretAccessKey,
- stringToSign( MethodString, ContentMD5, ContentType, Date,
+ stringToSign(MethodString, ContentMD5, ContentType, Date,
Bucket, Path, Headers )),
FinalHeaders = [ {"Authorization","AWS " ++ AccessKey ++ ":" ++ Signature },
@@ -370,8 +420,6 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
HttpOptions = [{autoredirect, true}],
Options = [ {sync,true}, {headers_as_is,true}, {body_format, binary} ],
- %%io:format("Request:~n ~p~n", [Request]),
-
Reply = http:request( Method, Request, HttpOptions, Options ),
%% {ok, {Status, ReplyHeaders, RBody}} = Reply,
@@ -384,15 +432,15 @@ genericRequest( Method, Bucket, Path, QueryParams, Metadata,
{ok, ResponseHeaders, ResponseBody};
{ok, {{_HttpVersion, Code, ReasonPhrase}, ResponseHeaders,
- _ResponseBody }} when Code=:=500, NrOfRetries > 0 ->
+ _ResponseBody }} when Code=:=500, NrOfRetries == 0 ->
throw ({error, "500", ReasonPhrase,
proplists:get_value(?S3_REQ_ID_HEADER, ResponseHeaders)});
{ok, {{_HttpVersion, Code, _ReasonPhrase}, _ResponseHeaders,
_ResponseBody }} when Code=:=500 ->
timer:sleep((?NR_OF_RETRIES-NrOfRetries)*500),
genericRequest(Method, Bucket, Path, QueryParams,
- Metadata, ContentType, Body, NrOfRetries-1);
+ Metadata, HTTPHeaders, Body, NrOfRetries-1);
{ok, {{_HttpVersion, _HttpCode, _ReasonPhrase}, ResponseHeaders,
ResponseBody }} ->
@@ -416,5 +464,20 @@ extractObjectInfo (Node) ->
#s3_object_info{key=Key#xmlText.value, lastmodified=LastModified#xmlText.value,
etag=ETag#xmlText.value, size=Size#xmlText.value}.
+openAndGetFileSize(FileName) ->
+ case file:open(FileName, [read, binary]) of
+ {ok, File} ->
+ {ok, #file_info{size=Size}} = file:read_file_info(FileName),
+ {Size, File};
+ _ ->
+ {error, no_file}
+ end.
-
+sendData(Socket, File) ->
+ case file:read(File, ?CHUNK_SIZE) of
+ {ok, Data} ->
+ gen_tcp:send(Socket, Data),
+ sendData(Socket, File);
+ eof ->
+ ok
+ end.
View
@@ -33,7 +33,7 @@ create_domain(Domain) ->
try genericRequest("CreateDomain",
Domain, "", [], []) of
{ok, Body} ->
- {XmlDoc, _Rest} = xmerl:scan_string(Body),
+ {XmlDoc, _Rest} = xmerl_scan:string(Body),
[#xmlText{value=RequestId}|_] =
xmerl_xpath:string("//ResponseMetadata/RequestId/text()", XmlDoc),
{ok, {requestId, RequestId}}
View
@@ -11,18 +11,11 @@ iso_8601_fmt(DateTime, Zone) ->
[Year, Month, Day, Hour, Min, Sec, Zone]).
mkEnumeration(Values, Separator) ->
- lists:flatten(lists:reverse(mkEnumeration(Values, Separator, []))).
-
-mkEnumeration([], _Separator, Acc) ->
- Acc;
-mkEnumeration([Head|[]], _Separator, Acc) ->
- [Head | Acc];
-mkEnumeration([Head|Tail], Separator, Acc) ->
- mkEnumeration(Tail, Separator, [Separator, Head | Acc]).
+ lists:flatten(string:join(Values, Separator)).
queryParams( [] ) -> "";
queryParams( ParamList ) ->
- "?" ++ mkEnumeration([url_encode(Param) ++ "=" ++ url_encode(Value)
+ "?" ++ mkEnumeration([[url_encode(Param), "=", url_encode(Value)]
|| {Param, Value} <- ParamList], "&" ).
%% The following code is taken from the ibrowse Http client

0 comments on commit 0abd90f

Please sign in to comment.