Support for basic OpenStack API operations and Keystone authentication

Fixes #373 #374 * Add rewrite rules for basic bucket and object operations from OpenStack Object Storage API * Add authentication module for Keystone authentication service * Make URL rewrite module configurable * Refactoring to further decouple API requests and responses from core object store functionality. * Add helper module for JSON processing * Refactor XML handling code * Isolate configuration functions into riak_cs_config module
basho · May 31, 2013 · 83f6ce7 · 83f6ce7
1 parent 6192547
commit 83f6ce7
Show file tree

Hide file tree

Showing 42 changed files with 1,653 additions and 496 deletions.
diff --git a/dialyzer.ignore-warnings b/dialyzer.ignore-warnings
@@ -163,11 +163,13 @@ riak_cs_gc.erl:.*: The pattern ..UUID, _... _. can never match the type ..$
 riak_cs_gc_d.erl:383: Function fetch_eligible_manifest_keys/2 will never be called
 riak_cs_gc_d.erl:387: Function eligible_manifest_keys/1 will never be called
 riak_cs_gc_d.erl:395: Function gc_index_query/2 will never be called
-^riak_cs_riakc_pool_worker.erl:48:
+^riak_cs_riakc_pool_worker.erl:54:
+riak_cs_storage_d.erl:346: Function fetch_user_list/1 will never be called
 riak_cs_storage_d.erl:384: Function fetch_user_list/1 will never be called
 riak_cs_wm_ping.erl:46: The pattern 'undefined' can never match the type pid()
 ############## All the others..........
 riak_cs_blockall_auth.erl:23: Callback info about the riak_cs_auth behaviour is not available
-riak_cs_passthru_auth.erl:23: Callback info about the riak_cs_auth behaviour is not available
+riak_cs_s3_passthru_auth.erl:23: Callback info about the riak_cs_auth behaviour is not available
 riak_cs_s3_auth.erl:23: Callback info about the riak_cs_auth behaviour is not available
+riak_cs_keystone_auth.erl:23: Callback info about the riak_cs_auth behaviour is not available
 riak_cs_s3_policy.erl:26: Callback info about the riak_cs_policy behaviour is not available
diff --git a/include/oos_api.hrl b/include/oos_api.hrl
@@ -0,0 +1,33 @@
+%% ---------------------------------------------------------------------
+%%
+%% Copyright (c) 2007-2013 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% ---------------------------------------------------------------------
+
+-define(DEFAULT_OS_AUTH_URL, "http://localhost:35357/v2.0/").
+-define(DEFAULT_TOKENS_RESOURCE, "tokens/").
+-define(DEFAULT_S3_TOKENS_RESOURCE, "s3tokens/").
+-define(DEFAULT_OS_USERS_RESOURCE, "users/").
+-define(DEFAULT_OS_ADMIN_TOKEN, "ADMIN").
+-define(DEFAULT_OS_OPERATOR_ROLES, [<<"admin">>, <<"swiftoperator">>]).
+
+-record(keystone_s3_auth_req_v1, {
+          access :: binary(),
+          signature :: binary(),
+          token :: binary()}).
+-type keystone_s3_auth_req() :: #keystone_s3_auth_req_v1{}.
+-define(KEYSTONE_S3_AUTH_REQ, #keystone_s3_auth_req_v1).
diff --git a/include/riak_cs.hrl b/include/riak_cs.hrl
@@ -86,8 +86,10 @@
                   submodule :: atom(),
                   exports_fun :: function(),
                   auth_module :: atom(),
+                  response_module :: atom(),
                   policy_module :: atom(),
-                  local_context :: term()
+                  local_context :: term(),
+                  api :: atom()
                  }).
 
 -record(key_context, {context :: #context{},
@@ -399,6 +401,10 @@
 -define(DEFAULT_PING_TIMEOUT, 5000).
 -define(JSON_TYPE, "application/json").
 -define(XML_TYPE, "application/xml").
+-define(S3_API_MOD, riak_cs_s3_rewrite).
+-define(OOS_API_MOD, riak_cs_oos_rewrite).
+-define(S3_RESPONSE_MOD, riak_cs_s3_response).
+-define(OOS_RESPONSE_MOD, riak_cs_oos_response).
 
 %% Major categories of Erlang-triggered DTrace probes
 %%

diff --git a/src/riak_cs_blockall_auth.erl → include/riak_cs_api.hrl b/src/riak_cs_blockall_auth.erl → include/riak_cs_api.hrl
@@ -18,18 +18,12 @@
 %%
 %% ---------------------------------------------------------------------
 
--module(riak_cs_blockall_auth).
+-record(list_buckets_response_v1, {
+          %% the user record
+          user :: rcs_user(),
 
--behavior(riak_cs_auth).
-
--include("riak_cs.hrl").
-
--export([identify/2,authenticate/4]).
-
--spec identify(term(), term()) -> {undefined, block_all}.
-identify(_RD,_Ctx) ->
-    {undefined, block_all}.
-
--spec authenticate(rcs_user(), term(), term(), term()) -> ok | {error, term()}.
-authenticate(_User, AuthData, _RD, _Ctx) ->
-    {error, AuthData}.
+          %% the list of bucket records
+          buckets :: [cs_bucket()]
+         }).
+-type list_buckets_response() :: #list_buckets_response_v1{}.
+-define(LBRESP, #list_buckets_response_v1).
diff --git a/rel/files/app.config b/rel/files/app.config
@@ -63,6 +63,10 @@
                 {bucket_list_pool, {{bucket_list_pool_tuple}} }
                ]},
 
+              %% == API and Authentication ==
+             {rewrite_module, {{rewrite_module}} },
+             {auth_module, {{auth_module}} },
+
               %% == Rolling upgrade support ==
 
               %% Riak CS version number. This is used to selectively

diff --git a/rel/vars.config b/rel/vars.config
@@ -26,6 +26,8 @@
 {request_pool_tuple, "{128, 0}"}.
 {bucket_list_pool_tuple, "{5, 0}"}.
 {cs_version,        010300}.
+{rewrite_module,    riak_cs_s3_rewrite}.
+{auth_module,       riak_cs_s3_auth}.
 
 %%
 %% etc/vm.args

diff --git a/rel/vars/dev1_vars.config b/rel/vars/dev1_vars.config
@@ -26,6 +26,8 @@
 {request_pool_tuple, "{128, 0}"}.
 {bucket_list_pool_tuple, "{5, 0}"}.
 {cs_version,        010300}.
+{rewrite_module,    riak_cs_s3_rewrite}.
+{auth_module,       riak_cs_s3_auth}.
 
 %%
 %% etc/vm.args

diff --git a/rel/vars/dev2_vars.config b/rel/vars/dev2_vars.config
@@ -26,6 +26,8 @@
 {request_pool_tuple, "{128, 0}"}.
 {bucket_list_pool_tuple, "{5, 0}"}.
 {cs_version,        010300}.
+{rewrite_module,    riak_cs_s3_rewrite}.
+{auth_module,       riak_cs_s3_auth}.
 
 %%
 %% etc/vm.args

diff --git a/rel/vars/dev3_vars.config b/rel/vars/dev3_vars.config
@@ -26,6 +26,8 @@
 {request_pool_tuple, "{128, 0}"}.
 {bucket_list_pool_tuple, "{5, 0}"}.
 {cs_version,        010300}.
+{rewrite_module,    riak_cs_s3_rewrite}.
+{auth_module,       riak_cs_s3_auth}.
 
 %%
 %% etc/vm.args

diff --git a/rel/vars/dev4_vars.config b/rel/vars/dev4_vars.config
@@ -26,6 +26,8 @@
 {request_pool_tuple, "{128, 0}"}.
 {bucket_list_pool_tuple, "{5, 0}"}.
 {cs_version,        010300}.
+{rewrite_module,    riak_cs_s3_rewrite}.
+{auth_module,       riak_cs_s3_auth}.
 
 %%
 %% etc/vm.args

diff --git a/src/riak_cs_api.erl b/src/riak_cs_api.erl
@@ -0,0 +1,60 @@
+%% ---------------------------------------------------------------------
+%%
+%% Copyright (c) 2007-2013 Basho Technologies, Inc.  All Rights Reserved.
+%%
+%% This file is provided to you under the Apache License,
+%% Version 2.0 (the "License"); you may not use this file
+%% except in compliance with the License.  You may obtain
+%% a copy of the License at
+%%
+%%   http://www.apache.org/licenses/LICENSE-2.0
+%%
+%% Unless required by applicable law or agreed to in writing,
+%% software distributed under the License is distributed on an
+%% "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+%% KIND, either express or implied.  See the License for the
+%% specific language governing permissions and limitations
+%% under the License.
+%%
+%% ---------------------------------------------------------------------
+
+-module(riak_cs_api).
+
+-export([list_buckets/1,
+         list_objects/5]).
+
+-include("riak_cs.hrl").
+-include("riak_cs_api.hrl").
+-include("list_objects.hrl").
+
+%% @doc Return a user's buckets.
+-spec list_buckets(rcs_user()) -> ?LBRESP{}.
+list_buckets(User=?RCS_USER{buckets=Buckets}) ->
+    ?LBRESP{user=User,
+            buckets=[Bucket || Bucket <- Buckets,
+                               Bucket?RCS_BUCKET.last_action /= deleted]}.
+
+-type options() :: [{atom(), 'undefined' | binary()}].
+-spec list_objects([string()], binary(), non_neg_integer(), options(), pid()) ->
+                          {ok, ?LORESP{}} | {error, term()}.
+list_objects([], _, _, _, _) ->
+    {error, no_such_bucket};
+list_objects(_UserBuckets, _Bucket, {error, _}=Error, _Options, _RiakPid) ->
+    Error;
+list_objects(_UserBuckets, Bucket, MaxKeys, Options, RiakPid) ->
+    ListKeysRequest = riak_cs_list_objects:new_request(Bucket,
+                                                       MaxKeys,
+                                                       Options),
+    BinPid = riak_cs_utils:pid_to_binary(self()),
+    CacheKey = << BinPid/binary, <<":">>/binary, Bucket/binary >>,
+    UseCache = riak_cs_list_objects_ets_cache:cache_enabled(),
+    case riak_cs_list_objects_fsm:start_link(RiakPid,
+                                             self(),
+                                             ListKeysRequest,
+                                             CacheKey,
+                                             UseCache) of
+        {ok, ListFSMPid} ->
+            riak_cs_list_objects_fsm:get_object_list(ListFSMPid);
+        {error, _}=Error ->
+            Error
+    end.
diff --git a/src/riak_cs_block_server.erl b/src/riak_cs_block_server.erl
@@ -174,11 +174,11 @@ handle_cast({get_block, ReplyPid, Bucket, Key, ClusterID, UUID, BlockNumber}, St
     {FullBucket, FullKey} = full_bkey(Bucket, Key, UUID, BlockNumber),
     StartTime = os:timestamp(),
     GetOptions = [{r, 1}, {notfound_ok, false}, {basic_quorum, false}],
-    LocalClusterID = riak_cs_utils:get_cluster_id(RiakcPid),
+    LocalClusterID = riak_cs_config:cluster_id(RiakcPid),
     %% don't use proxy get if it's a local get
     %% or proxy get is disabled
     UseProxyGet = ClusterID /= undefined
-                    andalso riak_cs_utils:proxy_get_active()
+                    andalso riak_cs_config:proxy_get_active()
                     andalso LocalClusterID /= ClusterID,
     Object =
         case UseProxyGet of