diff --git a/.travis.yml b/.travis.yml index 0b9be2fb7e5..0233d71b277 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,12 +22,12 @@ after_script: after_success: - make cover_report -services: redis-server +services: + - redis-server branches: only: - master -notifications: - email: mongoose-im@erlang-solutions.com + otp_release: - R16B03 env: @@ -37,6 +37,7 @@ env: - PRESET=odbc_pgsql_mnesia DB=pgsql REL_CONFIG=with-odbc - PRESET=pgsql_mnesia DB=pgsql REL_CONFIG=with-pgsql - PRESET=ldap_mnesia DB=mnesia + - PRESET=riak_mnesia DB=riak REL_CONFIG=with-riak - PRESET=external_mnesia DB=mnesia matrix: diff --git a/README.md b/README.md index 826121553de..7dc26a08b0a 100644 --- a/README.md +++ b/README.md @@ -123,6 +123,7 @@ with-mysql include mysql driver with-pgsql include pgsql driver with-odbc include standard ODBC driver shipped with Erlang/OTP with-redis include redis driver +with-riak include riak driver with-cassandra include cassandra driver full include all above deps ``` @@ -133,6 +134,9 @@ full include all above deps The `make configure` command has to be run only once (unless one need to change the relase config and include some other dependecies). + Take a look [here](http://mongooseim.readthedocs.org/en/latest/advanced-configuration/database-backends-configuration/) + for instructions how to setup the external databases. + `make rel` or `./rebar generate` commands will generate a self-contained OTP system image in the project's `rel/mongooseim` subdirectory. The contents of that directory are as follows: diff --git a/apps/ejabberd/src/ejabberd_app.erl b/apps/ejabberd/src/ejabberd_app.erl index 11b579d7041..3851638bf21 100644 --- a/apps/ejabberd/src/ejabberd_app.erl +++ b/apps/ejabberd/src/ejabberd_app.erl @@ -61,6 +61,7 @@ start(normal, _Args) -> mongoose_metrics:init(), ejabberd_system_monitor:add_handler(), ejabberd_rdbms:start(), + mongoose_riak:start(), ejabberd_auth:start(), cyrsasl:start(), %% Profiling diff --git a/apps/ejabberd/src/ejabberd_auth_http.erl b/apps/ejabberd/src/ejabberd_auth_http.erl index 397eba1bd36..3ce581bd27c 100644 --- a/apps/ejabberd/src/ejabberd_auth_http.erl +++ b/apps/ejabberd/src/ejabberd_auth_http.erl @@ -46,12 +46,13 @@ start(Host) -> PoolSize = proplists:get_value(connection_pool_size, AuthOpts, 10), Opts = proplists:get_value(connection_opts, AuthOpts, []), ChildMods = [fusco], - ChildMFA = {fusco, start_link, [AuthHost, Opts]}, + ChildMF = {fusco, start_link}, + ChildArgs = {for_all, [AuthHost, Opts]}, {ok, _} = supervisor:start_child(ejabberd_sup, {{ejabberd_auth_http_sup, Host}, {cuesport, start_link, - [pool_name(Host), PoolSize, ChildMods, ChildMFA]}, + [pool_name(Host), PoolSize, ChildMods, ChildMF, ChildArgs]}, transient, 2000, supervisor, [cuesport | ChildMods]}), ok. diff --git a/apps/ejabberd/src/ejabberd_auth_riak.erl b/apps/ejabberd/src/ejabberd_auth_riak.erl new file mode 100644 index 00000000000..bb58cb218af --- /dev/null +++ b/apps/ejabberd/src/ejabberd_auth_riak.erl @@ -0,0 +1,269 @@ +%%============================================================================== +%% Copyright 2015 Erlang Solutions Ltd. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%============================================================================== +-module(ejabberd_auth_riak). + +-behaviour(ejabberd_gen_auth). + +-include("ejabberd.hrl"). + +%% API +-export([start/1, + stop/1, + store_type/1, + login/2, + set_password/3, + check_password/3, + check_password/5, + try_register/3, + dirty_get_registered_users/0, + get_vh_registered_users/1, + get_vh_registered_users/2, + get_vh_registered_users_number/1, + get_vh_registered_users_number/2, + get_password/2, + get_password_s/2, + get_password/3, + does_user_exist/2, + remove_user/2, + remove_user/3, + plain_password_required/0]). + +-export([bucket_type/1]). + +-spec start(ejabberd:lserver()) -> ok. +start(_Host) -> + ok. + +-spec stop(ejabberd:lserver()) -> ok. +stop(_Host) -> + ok. + +-spec store_type(ejabberd:lserver()) -> plain | scram. +store_type(Host) -> + case scram:enabled(Host) of + false -> plain; + true -> scram + end. + +-spec set_password(ejabberd:luser(),ejabberd:lserver(), binary()) + -> ok | {error, not_allowed | invalid_jid}. +set_password(LUser, LServer, Password) -> + case prepare_password(LServer, Password) of + false -> + {error, invalid_password}; + Password -> + User = mongoose_riak:fetch_type(bucket_type(LServer), LUser), + do_set_password(User, LUser, LServer, Password) + end. + +-spec check_password(ejabberd:luser(), ejabberd:lserver(), binary()) -> boolean(). +check_password(LUser, LServer, Password) -> + case do_get_password(LUser, LServer) of + false -> + false; + #scram{} = Scram -> + scram:check_password(Password, Scram); + Password when is_binary(Password) -> + Password /= <<"">>; + _ -> + false + end. + +-spec check_password(ejabberd:luser(), + ejabberd:lserver(), + binary(), + binary(), + fun()) -> boolean(). +check_password(LUser, LServer, Password, Digest, DigestGen) -> + case do_get_password(LUser, LServer) of + false -> + false; + #scram{} = Scram -> + scram:check_digest(Scram, Digest, DigestGen, Password); + PassRiak when is_binary(PassRiak) -> + ejabberd_auth:check_digest(Digest, DigestGen, Password, PassRiak) + end. + +try_register(LUser, LServer, Password) -> + try_register_if_does_not_exist(LUser, LServer, Password). + +-spec dirty_get_registered_users() -> [ejabberd:simple_jid()]. +dirty_get_registered_users() -> + Servers = ejabberd_config:get_vh_by_auth_method(riak), + lists:flatmap( + fun(Server) -> + get_vh_registered_users(Server) + end, Servers). + +-spec get_vh_registered_users(ejabberd:lserver()) -> + [ejabberd:simple_jid()]. +get_vh_registered_users(LServer) -> + case mongoose_riak:list_keys(bucket_type(LServer)) of + {ok, Users} -> + [{User, LServer} || User <- Users]; + _ -> + [] + end. + +-spec get_vh_registered_users(ejabberd:lserver(), list()) -> + [ejabberd:simple_jid()]. +get_vh_registered_users(LServer, _Opts) -> + get_vh_registered_users(LServer). + +-spec get_vh_registered_users_number(ejabberd:lserver()) -> non_neg_integer(). +get_vh_registered_users_number(LServer) -> + length(get_vh_registered_users(LServer)). + +-spec get_vh_registered_users_number(ejabberd:lserver(), list()) -> non_neg_integer(). +get_vh_registered_users_number(LServer, _Opts) -> + get_vh_registered_users_number(LServer). + +-spec get_password(ejabberd:luser(), ejabberd:lserver()) -> binary() | false | scram(). +get_password(LUser, LServer) -> + case do_get_password(LUser, LServer) of + false -> + false; + #scram{} = Scram -> + scram:scram_to_tuple(Scram); + Password -> + Password + end. + +get_password_s(LUser, LServer) -> + case get_password(LUser, LServer) of + Password when is_binary(Password) -> + Password; + _ -> + <<"">> + end. + +get_password(_LUser, _LServer, _DefaultValue) -> + erlang:error(not_implemented). + +-spec does_user_exist(ejabberd:luser(), ejabberd:lserver()) -> boolean(). +does_user_exist(LUser, LServer) -> + case mongoose_riak:fetch_type(bucket_type(LServer), LUser) of + {ok, _} -> + true; + {error, {notfound, map}} -> + false + end. + +-spec remove_user(ejabberd:luser(), ejabberd:lserver()) -> + ok | {error, term()}. +remove_user(LUser, LServer) -> + mongoose_riak:delete(bucket_type(LServer), LUser). + +remove_user(_LUser, _LServer, _Password) -> + erlang:error(not_implemented). + +plain_password_required() -> + false. + +login(_LUser, _LServer) -> + erlang:error(not_implemented). + +-spec bucket_type(ejabberd:lserver()) -> {binary(), ejabberd:lserver()}. +bucket_type(LServer) -> + {<<"users">>, LServer}. + +%% ----------------------------------------------------------------------------- +%% Internal functions +%% ----------------------------------------------------------------------------- + +try_register_if_does_not_exist(LUser, LServer, _) + when LUser =:= error; LServer =:= error -> + {error, invalid_jid}; +try_register_if_does_not_exist(LUser, LServer, PasswordIn) -> + case does_user_exist(LUser, LServer) of + false -> + Password = prepare_password(LServer, PasswordIn), + try_register_with_password(LUser, LServer, Password); + true -> + {error, exists} + end. + +try_register_with_password(LUser, LServer, Password) -> + Now = integer_to_binary(now_to_seconds(os:timestamp())), + Ops = [{{<<"created">>, register}, + fun(R) -> riakc_register:set(Now, R) end}, + set_password_map_op(Password)], + UserMap = mongoose_riak:create_new_map(Ops), + case mongoose_riak:update_type(bucket_type(LServer), LUser, + riakc_map:to_op(UserMap)) of + {ok, _Map} -> + ok; + Error -> + Error + end. + +do_get_password(LUser, LServer) -> + case mongoose_riak:fetch_type(bucket_type(LServer), LUser) of + {ok, Map} -> + extract_password(Map); + _ -> + false + end. + +do_set_password({ok, Map}, LUser, LServer, Password) -> + Ops = [set_password_map_op(Password)], + UpdateMap = mongoose_riak:update_map(Map, Ops), + case mongoose_riak:update_type(bucket_type(LServer), LUser, + riakc_map:to_op(UpdateMap)) of + ok -> + ok; + Reason -> + Reason + end. + +prepare_password(Iterations, Password) when is_integer(Iterations) -> + Scram = scram:password_to_scram(Password, Iterations), + PassDetails = scram:serialize(Scram), + {<<"">>, PassDetails}; + +prepare_password(Server, Password) -> + case scram:enabled(Server) of + true -> + prepare_password(scram:iterations(Server), Password); + _ -> + Password + end. + +set_password_map_op({_, Scram}) -> + {{<<"scram">>, register}, fun(R) -> riakc_register:set(Scram, R) end}; +set_password_map_op(Password) -> + {{<<"password">>, register}, fun(R) -> riakc_register:set(Password, R) end}. + +extract_password(Map) -> + case riakc_map:find({<<"password">>, register}, Map) of + error -> + maybe_extract_scram_password(riakc_map:find({<<"scram">>, register}, Map)); + {ok, Password} -> + Password + end. + +maybe_extract_scram_password(false) -> + false; +maybe_extract_scram_password({ok, ScramSerialised}) -> + case scram:deserialize(ScramSerialised) of + {ok, Scram} -> + Scram; + _ -> + false + end. + +now_to_seconds({MegaSecs, Secs, _MicroSecs}) -> + MegaSecs * 1000000 + Secs. \ No newline at end of file diff --git a/apps/ejabberd/src/ejabberd_config.erl b/apps/ejabberd/src/ejabberd_config.erl index 637b1077f13..6222b4c8b0c 100644 --- a/apps/ejabberd/src/ejabberd_config.erl +++ b/apps/ejabberd/src/ejabberd_config.erl @@ -605,6 +605,8 @@ process_host_term(Term, Host, State) -> State; {odbc_server, ODBC_server} -> add_option({odbc_server, Host}, ODBC_server, State); + {riak_server, RiakConfig} -> + add_option(riak_server, RiakConfig, State); {Opt, Val} -> add_option({Opt, Host}, Val, State) end. @@ -1000,7 +1002,9 @@ handle_config_change({_Key, _OldValue, _NewValue}) -> %% ---------------------------------------------------------------- %% LOCAL CONFIG %% ---------------------------------------------------------------- -handle_local_config_add({Key, _Data} = El) -> +handle_local_config_add(#local_config{key = riak_server}) -> + mongoose_riak:start(); +handle_local_config_add(#local_config{key=Key} = El) -> case Key of true -> ok; @@ -1008,10 +1012,12 @@ handle_local_config_add({Key, _Data} = El) -> ?WARNING_MSG("local config add ~p option unhandled",[El]) end. +handle_local_config_del(#local_config{key = riak_server}) -> + mongoose_riak:stop(); handle_local_config_del(#local_config{key = node_start}) -> %% do nothing with it ok; -handle_local_config_del({Key, _Data} = El) -> +handle_local_config_del(#local_config{key=Key} = El) -> case can_be_ignored(Key) of true -> ok; @@ -1021,6 +1027,10 @@ handle_local_config_del({Key, _Data} = El) -> handle_local_config_change({listen, Old, New}) -> reload_listeners(compare_listeners(Old, New)); +handle_local_config_change({riak_server, _Old, _New}) -> + mongoose_riak:stop(), + mongoose_riak:start(), + ok; handle_local_config_change({Key, _Old, _New} = El) -> case can_be_ignored(Key) of diff --git a/apps/ejabberd/src/ejabberd_redis.erl b/apps/ejabberd/src/ejabberd_redis.erl index 97a308d08d0..19936d834c7 100644 --- a/apps/ejabberd/src/ejabberd_redis.erl +++ b/apps/ejabberd/src/ejabberd_redis.erl @@ -18,12 +18,13 @@ start_link(Opts) -> PoolSize = proplists:get_value(pool_size, Opts, 10), RedoOpts = proplists:get_value(worker_config, Opts, []), ChildMods = [redo, redo_redis_proto, redo_uri], - ChildMFA = {redo, start_link, [undefined, RedoOpts]}, + ChildMF = {redo, start_link}, + ChildArgs = {for_all, [undefined, RedoOpts]}, supervisor:start_child(ejabberd_sm_backend_sup, {ejabberd_redis_sup, {cuesport, start_link, - [?POOL_NAME, PoolSize, ChildMods, ChildMFA]}, + [?POOL_NAME, PoolSize, ChildMods, ChildMF, ChildArgs]}, transient, 2000, supervisor, [cuesport | ChildMods]}). -spec cmd(iolist()) -> binary() diff --git a/apps/ejabberd/src/mongoose_riak.erl b/apps/ejabberd/src/mongoose_riak.erl new file mode 100644 index 00000000000..83d88bac1e1 --- /dev/null +++ b/apps/ejabberd/src/mongoose_riak.erl @@ -0,0 +1,151 @@ +%%============================================================================== +%% Copyright 2015 Erlang Solutions Ltd. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%============================================================================== +-module(mongoose_riak). + +-include("ejabberd.hrl"). +-include_lib("riakc/include/riakc.hrl"). + +%% API +-export([start/0]). +-export([stop/0]). + +-export([start_worker/3]). + +-export([put/1, put/2]). +-export([get/2, get/3]). +-export([delete/2, delete/3]). +-export([update_type/3, update_type/4]). +-export([fetch_type/2, fetch_type/3]). +-export([list_keys/1]). +-export([get_worker/0]). +-export([create_new_map/1]). +-export([update_map/2]). + +-export([pool_name/0]). + +-compile({no_auto_import,[put/2]}). + +-define(CALL(F, Args), call_riak(F, Args)). + +-type riakc_map_op() :: {{binary(), riakc_map:datatype()}, + fun((riakc_datatype:datatype()) -> riakc_datatype:datatype())}. + +-spec start() -> {ok, pid()} | ignore. +start() -> + case ejabberd_config:get_local_option(riak_server) of + undefined -> + ignore; + RiakOpts -> + {_, RiakAddr} = lists:keyfind(address, 1, RiakOpts), + {_, RiakPort} = lists:keyfind(port, 1, RiakOpts), + Workers = proplists:get_value(pool_size, RiakOpts, 20), + RiakPBOpts = [auto_reconnect, keepalive], + mongoose_riak_sup:start(Workers, RiakAddr, RiakPort, RiakPBOpts) + end. +-spec stop() -> no_return(). +stop() -> + mongoose_riak_sup:stop(). + +-spec start_worker(riakc_pb_socket:address(), riakc_pb_socket:portnum(), + proplists:proplist()) + -> {ok, pid()} | {error, term()}. +start_worker(Address, Port, Opts) -> + riakc_pb_socket:start_link(Address, Port, Opts). + +-spec put(riakc_obj()) -> + ok | {ok, riakc_obj()} | {ok, key()} | {error, term()}. +put(Obj) -> + put(Obj, []). + +-spec put(riakc_obj(), timeout() | put_options()) -> + ok | {ok, riakc_obj()} | {ok, key()} | {error, term()}. +put(Obj, OptsOrTimeout) -> + ?CALL(put, [Obj, OptsOrTimeout]). + +-spec get(bucket(), key()) -> {ok, riakc_obj()} | {error, term()}. +get(Bucket, Key) -> + get(Bucket, Key, []). + +-spec get(bucket(), key(), get_options() | timeout()) -> + {ok, riakc_obj()} | {error, term()}. +get(Bucket, Key, OptsOrTimeout) -> + ?CALL(get, [Bucket, Key, OptsOrTimeout]). + + +-spec update_type({binary(), binary()}, binary(), riakc_datatype:update(term())) -> + ok. +update_type(Bucket, Key, Update) -> + update_type(Bucket, Key, Update, []). + +-spec update_type({binary(), binary()}, binary(), + riakc_datatype:update(term()), [proplists:property()]) -> + ok. +update_type(Bucket, Key, Update, Options) -> + ?CALL(update_type, [Bucket, Key, Update, Options]). + +-spec delete(bucket(), key()) -> + ok | {error, term()}. +delete(Bucket, Key) -> + delete(Bucket, Key, []). + +-spec delete(bucket(), key(), delete_options() | timeout()) -> + ok | {error, term()}. +delete(Bucket, Key, OptsOrTimeout) -> + ?CALL(delete, [Bucket, Key, OptsOrTimeout]). + +-spec fetch_type({binary(), binary()}, binary()) -> + {ok, riakc_datatype:datatype()} | {error, term()}. +fetch_type(Bucket, Key) -> + fetch_type(Bucket, Key, []). + +-spec fetch_type({binary(), binary()}, binary(), [proplists:property()]) -> + {ok, riakc_datatype:datatype()} | {error, term()}. +fetch_type(Bucket, Key, Opts) -> + ?CALL(fetch_type, [Bucket, Key, Opts]). + +-spec list_keys({binary(), binary()}) -> + {ok, [binary()]} | {error, term()}. +list_keys(Bucket) -> + ?CALL(list_keys, [Bucket]). + + +-spec create_new_map([riakc_map_op()]) -> riakc_map:crdt_map(). +create_new_map(Ops) -> + update_map(riakc_map:new(), Ops). + +-spec update_map(riakc_map:crdt_map(), [riakc_map_op()]) -> riakc_map:crdt_map(). +update_map(Map, Ops) -> + lists:foldl(fun update_map_op/2, Map, Ops). + +-spec get_worker() -> pid() | undefined. +get_worker() -> + case catch cuesport:get_worker(pool_name()) of + Pid when is_pid(Pid) -> + Pid; + _ -> + undefined + end. + +-spec pool_name() -> atom(). +pool_name() -> riak_pool. + +update_map_op({Field, Fun}, Map) -> + riakc_map:update(Field, Fun, Map). + +call_riak(F, ArgsIn) -> + Worker = get_worker(), + Args = [Worker | ArgsIn], + apply(riakc_pb_socket, F, Args). diff --git a/apps/ejabberd/src/mongoose_riak_sup.erl b/apps/ejabberd/src/mongoose_riak_sup.erl new file mode 100644 index 00000000000..a5d15c81cc5 --- /dev/null +++ b/apps/ejabberd/src/mongoose_riak_sup.erl @@ -0,0 +1,100 @@ +%%============================================================================== +%% Copyright 2015 Erlang Solutions Ltd. +%% +%% Licensed under the Apache License, Version 2.0 (the "License"); +%% you may not use this file except in compliance with the License. +%% You may obtain a copy of the License at +%% +%% http://www.apache.org/licenses/LICENSE-2.0 +%% +%% Unless required by applicable law or agreed to in writing, software +%% distributed under the License is distributed on an "AS IS" BASIS, +%% WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +%% See the License for the specific language governing permissions and +%% limitations under the License. +%%============================================================================== +-module(mongoose_riak_sup). + +-behaviour(supervisor). + +%% API +-export([start/4]). +-export([start_link/4]). +-export([stop/0]). + +%% Supervisor callbacks +-export([init/1]). + +-define(SERVER, ?MODULE). + +-include("ejabberd.hrl"). + +%%%=================================================================== +%%% API functions +%%%=================================================================== + +start(Workers, Addr, Port, PBOpts) -> + ChildSpec = {?MODULE, {?MODULE, start_link, [Workers, Addr, Port, PBOpts]}, + transient, infinity, supervisor, [?MODULE]}, + {ok, _} = supervisor:start_child(ejabberd_sup, ChildSpec). +%%-------------------------------------------------------------------- +%% @doc +%% Starts the supervisor +%% +%% @end +%%-------------------------------------------------------------------- +%-spec(start_link(integer(), inet:ip(), ) -> +% {ok, Pid :: pid()} | ignore | {error, Reason :: term()}). +start_link(Workers, Addr, Port, PBOpts) -> + supervisor:start_link({local, ?SERVER}, ?MODULE, [Workers, Addr, Port, PBOpts]). + +-spec stop() -> no_return(). +stop() -> + supervisor:terminate_child(ejabberd_sup, ?MODULE), + supervisor:delete_child(ejabberd_sup, ?MODULE). + +%%%=================================================================== +%%% Supervisor callbacks +%%%=================================================================== + +%%-------------------------------------------------------------------- +%% @private +%% @doc +%% Whenever a supervisor is started using supervisor:start_link/[2,3], +%% this function is called by the new process to find out about +%% restart strategy, maximum restart frequency and child +%% specifications. +%% +%% @end +%%-------------------------------------------------------------------- +-spec(init(Args :: term()) -> + {ok, {SupFlags :: {RestartStrategy :: supervisor:strategy(), + MaxR :: non_neg_integer(), MaxT :: non_neg_integer()}, + [ChildSpec :: supervisor:child_spec()] + }} | + ignore | + {error, Reason :: term()}). +init([Workers, Address, Port, PBOpts]) -> + RestartStrategy = one_for_one, + MaxRestarts = 10, + MaxSecondsBetweenRestarts = 1, + + SupFlags = {RestartStrategy, MaxRestarts, MaxSecondsBetweenRestarts}, + {ok, {SupFlags, [child_spec(Workers, [Address, Port, PBOpts])] }}. + +child_spec(Workers, RiakOpts) -> + Restart = transient, + Shutdown = 2000, + Type = supervisor, + ChildMods = [mongoose_riak, riakc_pb_socket], + ChildMF = {mongoose_riak, start_worker}, + RiakPoolName = mongoose_riak:pool_name(), + ChildArgs = {for_all, RiakOpts}, + AChild = {RiakPoolName, {cuesport, start_link, [RiakPoolName, Workers, ChildMods, ChildMF, ChildArgs]}, + Restart, Shutdown, Type, ChildMods}, + AChild. + +%%%=================================================================== +%%% Internal functions +%%%=================================================================== + diff --git a/doc/Basic-configuration.md b/doc/Basic-configuration.md index dd1f1174229..9f8637c837a 100644 --- a/doc/Basic-configuration.md +++ b/doc/Basic-configuration.md @@ -31,6 +31,12 @@ There are 2 types of options: params and features. Unlike params, features can b * **Description:** SQL DB connection configuration. Currently supported DB types are `mysql` and `pgsql`. To enable the connection, remove '%%' prefix from value. * **Syntax:** `"{odbc_server, {Type, Host, Port, DBName, Username, Password}}."` +* **riak_server** - feature + * **Description:** Riak connection pool configuration. Currently only one endpoint can be specified, to connect to more riak nodes you have to use load balancing techniques, for more details see: + [Load Balancing riak](http://docs.basho.com/riak/latest/ops/advanced/configs/load-balancing-proxy/) from basho. + To enable, remove '%%' prefix from value. + * **Syntax:** `"{riak_server, [{pool_size, Size}, {address, Host}, {port, Port}]}."` + * **auth_ldap** - feature * **Description:** Put [[LDAP configuration]] here. @@ -65,7 +71,7 @@ There are 2 types of options: params and features. Unlike params, features can b * **auth_method** - param * **Description:** Chooses authentication modules. Can be either a single module or a list of modules to be tried in sequence until one of them succeeds. - * **Valid values:** `internal`, `odbc`, `external`, `anonymous`, `ldap` + * **Valid values:** `internal`, `odbc`, `external`, `anonymous`, `ldap`, `riak` * `internal` means Mnesia-based * **Examples:** `"odbc"`, `"[internal, anonymous]"` diff --git a/doc/README.md b/doc/README.md index 219a2e6c5b2..02979d2d042 100644 --- a/doc/README.md +++ b/doc/README.md @@ -10,11 +10,13 @@ * Configuration * [Basic configuration](Basic-configuration.md) * [Advanced configuration](Advanced-configuration.md) + * [Database backends configuration](advanced-configuration/database-backends-configuration.md) * [Listener modules](advanced-configuration/Listener-modules.md) * [Extension modules](advanced-configuration/Modules.md) * [HTTP authentication module](advanced-configuration/HTTP-authentication-module.md) * Operation and maintenance * [Metrics](operation-and-maintenance/Mongoose-metrics.md) + * [Logging & monitoring](operation-and-maintenance/Logging-&-monitoring.md) * [Cluster configuration and node management](operation-and-maintenance/Cluster-configuration-and-node-management.md) * [Reloading configuration on a running system](operation-and-maintenance/Reloading-configuration-on-a-running-system.md) * For developers diff --git a/doc/advanced-configuration/database-backends-configuration.md b/doc/advanced-configuration/database-backends-configuration.md new file mode 100644 index 00000000000..217bd136935 --- /dev/null +++ b/doc/advanced-configuration/database-backends-configuration.md @@ -0,0 +1,127 @@ + +MongooseIM can work with several databases, both SQL and NoSQL ones. Some of +them require extra work before they can be used. For example the SQL databases require +defining schema. MongooseIM is tested with TravisCI, so the travis scripts can be used +as a reference. + +# MySQL + +**Can be used for**: + +* users (credentials) +* vcards +* roster +* private storage +* privacy lists +* last activity +* mam (message archive) + +**Setup** + +The schema files can be found in the `apps/ejabberd/priv` directory. The default +schema is defined in the `mysql.sql` file. + +For example, you can use the following command to apply it on localhost: + +```bash +mysql -h localhost -u user -p -e 'create database mongooseim' +mysql -h localhost -u user -p mongooseim < mysql.sql +``` + +# Postgres + +**Can be used for:** + +* users (credentials) +* vcards +* roster +* private storage +* privacy lists +* last activity +* mam (message archive) + +**Setup** + +The schema files can be found in the `apps/ejabberd/priv` directory. The default +schema is defined in the `pg.sql` file. + +For example, you can use the following command to apply it on localhost: + +```bash +psql -h localhost -U user -c "CREATE DATABASE mongooseim;" +psql -h localhost -U user -q -d mongooseim -f pg.sql +``` + +# MSSQL + +**Can be used for:** + +* users (credentials) +* vcards +* roster +* private storage +* privacy lists +* last activity +* mam (message archive) + +**Setup** + +# Riak (versions >=2.0) + +**Can be used for:** + +* users (credentials) + +**Setup** + +We are using the riak data types, so the minimal supported version is 2.0. To +be able to store **user credentials** one have to run the following command: + +```bash +riak-admin bucket-type create users '{"props":{"datatype":"map"}}' +riak-admin bucket-type activate users +``` + +This will create a bucket type required for storing **users credentials** and it will +activate it. + +# Cassandra + +**Can be used for:** + +* mam (Message archive) + +**Setup** + +The schema files can be found in the `apps/ejabberd/priv` directory. The default +schema is defined in the `cassandra.cql` file. + +For example, you can use the following command to apply schema on localhost: + +``` +cqlsh localhost 9160 -f cassandra.cql +``` + +# Redis + +**Can be used for:** + +* users sessions + +**Setup** + +No additional steps required. + +# LDAP + +**Can be used for:** + +* users (credentials) +* shared roster +* vcard + +**Setup** + +No additional steps required, the modules that are using LDAP are very customizable, +so they can be configured to support existsing schemas. + diff --git a/mkdocs.yml b/mkdocs.yml index f9b4361d1cb..44145ea3fad 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -13,6 +13,7 @@ pages: - 'Basic Configuration Overview': 'Basic-configuration.md' - 'Advanced Configuration Overview': 'Advanced-configuration.md' - 'Advanced Configuration': + - 'Database backends configuration': 'advanced-configuration/database-backends-configuration.md' - 'HTTP Authentication Module': 'advanced-configuration/HTTP-authentication-module.md' - 'Listener Modules': 'advanced-configuration/Listener-modules.md' - 'Extension Modules': 'advanced-configuration/Modules.md' diff --git a/rebar.config b/rebar.config index dbf27b41d3b..8e4339c3354 100644 --- a/rebar.config +++ b/rebar.config @@ -12,7 +12,7 @@ {deps, [ {base16, ".*", {git, "git://github.com/goj/base16.git", "1e9dd28bdbcfb"}}, - {cuesport, ".*", {git, "git://github.com/goj/cuesport.git", "3b16d99"}}, + {cuesport, ".*", {git, "git://github.com/esl/cuesport.git", "d82ff25"}}, {redo, ".*", {git, "git://github.com/JacobVorreuter/redo.git", "7c7eaef"}}, {exml, "2.1.5", {git, "git://github.com/esl/exml.git", "2.1.5"}}, {lager, "2.0.3", {git, "git://github.com/basho/lager.git", "2.0.3"}}, @@ -25,6 +25,7 @@ {idna, ".*", {git, "git://github.com/benoitc/erlang-idna.git", {tag, "1.0.1"}}}, {seestar, ".*", {git, "git://github.com/iamaleksey/seestar.git", "83e8099b617fffe5af86d4c91d84ce3608accd25"}}, + {riakc, ".*", {git, "https://github.com/basho/riak-erlang-client", "2.0.1"}}, {p1_utils, ".*", {git, "git://github.com/processone/p1_utils", "940f42ddfcdc0b7b2abf4d9ee292605a93699543"}}, {p1_cache_tab, ".*", {git, "git://github.com/processone/cache_tab", "7b89d6a"}}, @@ -39,6 +40,8 @@ {recon, "2.2.1", {git, "git://github.com/ferd/recon.git", {tag, "2.2.1"}}} ]}. +{pre_hooks, [{compile, "tools/compile_riak_pb.sh"}]}. + {ct_extra_params, "-pa apps/ejabberd/ebin " "-pa apps/stringprep/ebin " "-sasl sasl_error_logger false"}. diff --git a/rel/files/ejabberd.cfg b/rel/files/ejabberd.cfg index 41e79156522..681128e3064 100755 --- a/rel/files/ejabberd.cfg +++ b/rel/files/ejabberd.cfg @@ -418,6 +418,12 @@ %% %%{odbc_pool_size, 10}. +%%% ==================== +%%% RIAK SETUP +%%% ==================== + +{{riak_server}} + %% %% Interval to make a dummy SQL request to keep the connections to the %% database alive. Specify in seconds: for example 28800 means 8 hours diff --git a/rel/vars.config b/rel/vars.config index a436b662438..ddb2e0b0206 100644 --- a/rel/vars.config +++ b/rel/vars.config @@ -31,3 +31,4 @@ {https_config, "{cert, \"priv/ssl/fake_cert.pem\"}, {key, \"priv/ssl/fake_key.pem\"}, {key_pass, \"\"},"}. %% Applies to Websockets, BOSH and metrics; PEM format {zlib, "%%{zlib, 10000},"}. %% Second element of a tuple is inflated data size limit; 0 for no limit {registration_watchers, "%{registration_watchers, [\"admin@localhost\"]},"}. +{riak_server, "%%{riak_server, [{pool_size, 20}, {address, \"127.0.0.1\"}, {port, 8087}, {riak_pb_socket_opts, []}]}."}. diff --git a/tools/advanced.config b/tools/advanced.config new file mode 100644 index 00000000000..815bc989920 --- /dev/null +++ b/tools/advanced.config @@ -0,0 +1,3 @@ +[ + {riak_kv, [{delete_mode, immediate}]} +]. diff --git a/tools/compile_riak_pb.sh b/tools/compile_riak_pb.sh new file mode 100755 index 00000000000..cd4805703ae --- /dev/null +++ b/tools/compile_riak_pb.sh @@ -0,0 +1,3 @@ +#!/bin/bash + +cd deps/riak_pb; ./rebar compile deps_dir=.. \ No newline at end of file diff --git a/tools/configure b/tools/configure index 06221fa87f0..2f2cb04082b 100755 --- a/tools/configure +++ b/tools/configure @@ -21,6 +21,8 @@ analyze_opt("with-pgsql", {AppsToInclude, AppsToRun}) -> {[pgsql | AppsToInclude], AppsToRun}; analyze_opt("with-odbc", {AppsToInclude, AppsToRun}) -> {[odbc | AppsToInclude], AppsToRun}; +analyze_opt("with-riak", {AppsToInclude, AppsToRun}) -> + {[riakc, riak_pb, protobuffs | AppsToInclude], AppsToRun}; analyze_opt("with-redis", {AppsToInclude, AppsToRun}) -> {[redo | AppsToInclude], AppsToRun}; analyze_opt("with-cassandra", {AppsToInclude, AppsToRun}) -> @@ -35,6 +37,7 @@ all_opts_with_desc() -> [{"with-mysql", "include mysql driver"}, {"with-pgsql", "include pgsql driver"}, {"with-odbc", "include standard ODBC driver shipped with Erlang/OTP"}, + {"with-riak", "include riak client"}, {"with-redis", "include redis driver"}, {"with-cassandra", "include cassandra driver"}]. diff --git a/tools/setup_riak b/tools/setup_riak new file mode 100755 index 00000000000..fe4dfff8e09 --- /dev/null +++ b/tools/setup_riak @@ -0,0 +1,8 @@ +#!/bin/bash + +sudo cp tools/advanced.config /etc/riak/ + +service riak start + +${1}riak-admin bucket-type create users '{"props":{"datatype":"map"}}' +${1}riak-admin bucket-type activate users diff --git a/tools/travis-setup-db.sh b/tools/travis-setup-db.sh index 711641bb9e8..3bcc51aa7ab 100755 --- a/tools/travis-setup-db.sh +++ b/tools/travis-setup-db.sh @@ -35,4 +35,8 @@ Protocol = 9.3.5 Debug = 1 EOL +elif [ $DB = 'riak' ]; then + riak version + sudo tools/setup_riak + fi \ No newline at end of file