Merge pull request #1965 from esl/inbox-mssql

MSSQL support for Inbox

big_tests/run_common_test.erl

@@ -1,5 +1,5 @@
 %% During dev you would use something similar to:
-%% TEST_HOSTS="mim1" ./tools/travis-test.sh -e false -c false -s false -p odbc_mssql_mnesia
+%% TEST_HOSTS="mim" ./tools/travis-test.sh -e false -c false -s false -p odbc_mssql_mnesia
 %%
 %% If you also want to start just mim1 node use:
 %% DEV_NODES="mim1" TEST_HOSTS="mim" ./tools/travis-test.sh -e false -c false -s false -p odbc_mssql_mnesia

big_tests/tests/inbox_SUITE.erl

@@ -56,18 +56,12 @@
 %%--------------------------------------------------------------------
 
 all() ->
-  case is_odbc_enabled(domain()) of
-    true ->
-      %% TODO remove when supported
-      case is_mssql_enabled(domain()) of
-        true ->
-          {skip, mssql_not_supported};
-        _ ->
-          tests()
-      end;
-    false ->
-      {skip, require_odbc}
-  end.
+    case is_odbc_enabled(domain()) of
+      true ->
+          tests();
+      false ->
+          {skip, require_odbc}
+    end.
 
 tests() ->
   [
@@ -1006,7 +1000,3 @@ restore_inbox_option(Config) ->
   Args = proplists:get_value(inbox_opts, Config),
   dynamic_modules:restart(Host, mod_inbox, Args).
 
-is_mssql_enabled(Host) ->
-  Engine = rpc(mongoose_rdbms,db_engine,[Host]),
-  %% According to the code, it will return "odbc" if there is mssql string configuration applied to 'odbc_server' tuple
-  Engine =:= odbc.

doc/modules/mod_inbox.md

@@ -19,6 +19,15 @@ Only changes that affect the user directly will be stored in their inbox.
 If true, the inbox conversation is removed for a user when they are removed from the groupchat.
 * **iqdisc** (atom, default: `no_queue`)
 
+### Note about supported RDBMS
+
+`mod_inbox` executes upsert queries, which have different syntax in every supported RDBMS.
+Inbox currently supports the following DBs:
+
+* MySQL via native driver
+* PgSQL via native driver
+* MSSQL via ODBC driver
+
 ### Example Request
 
 ```

priv/mssql2012.sql

@@ -453,6 +453,23 @@ GO
 CREATE INDEX i_muc_light_blocking ON muc_light_blocking(luser, lserver);
 GO
 
+-- luser, lserver and remote_bare_jid have 250 characters in MySQL
+-- but here we are limited by index size (900 bytes)
+CREATE TABLE dbo.inbox(
+    luser NVARCHAR(150) NOT NULL,
+    lserver NVARCHAR(150) NOT NULL,
+    remote_bare_jid NVARCHAR(150) NOT NULL,
+    content VARBINARY(max) NOT NULL,
+    unread_count INT NOT NULL,
+    msg_id NVARCHAR(250) NOT NULL,
+    CONSTRAINT PK_inbox PRIMARY KEY CLUSTERED(
+        luser ASC,
+        lserver ASC,
+        remote_bare_jid ASC
+    )
+)
+GO
+
 SET ANSI_PADDING OFF
 GO
 ALTER TABLE [dbo].[offline_message] ADD  DEFAULT (NULL) FOR [expire]

src/inbox/mod_inbox_odbc.erl

@@ -21,8 +21,16 @@
          remove_inbox/3,
          clear_inbox/2]).
 
--define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))).
-init(_VHost, _Options) ->
+%% For specific backends
+-export([esc_string/1]).
+
+%% ----------------------------------------------------------------------
+%% API
+%% ----------------------------------------------------------------------
+
+init(VHost, _Options) ->
+    %% To verify if current ODBC backend is supported
+    odbc_specific_backend(VHost),
     ok.
 
 -spec get_inbox(LUsername :: jid:luser(),
@@ -41,7 +49,7 @@ get_inbox_rdbms(LUser, Server) ->
     mongoose_rdbms:sql_query(
         Server,
         ["select remote_bare_jid, content, unread_count from inbox "
-        "where luser=", ?ESC(LUser), " and lserver=", ?ESC(Server), ";"]).
+        "where luser=", esc_string(LUser), " and lserver=", esc_string(Server), ";"]).
 
 -spec set_inbox(Username, Server, ToBareJid, Content, Count, MsgId) -> inbox_write_res() when
                 Username :: jid:luser(),
@@ -73,9 +81,9 @@ remove_inbox(Username, Server, ToBareJid) ->
                          ToBareJid :: binary()) -> query_result().
 remove_inbox_rdbms(Username, Server, ToBareJid) ->
     mongoose_rdbms:sql_query(Server, ["delete from inbox where luser=",
-        ?ESC(Username), " and lserver=", ?ESC(Server),
+        esc_string(Username), " and lserver=", esc_string(Server),
         " and remote_bare_jid=",
-        ?ESC(ToBareJid), ";"]).
+        esc_string(ToBareJid), ";"]).
 
 -spec set_inbox_incr_unread(Username :: binary(),
                             Server :: binary(),
@@ -108,8 +116,8 @@ reset_unread(Username, Server, ToBareJid, MsgId) ->
                                MsgId :: binary()) -> query_result().
 reset_inbox_unread_rdbms(Username, Server, ToBareJid, MsgId) ->
     mongoose_rdbms:sql_query(Server, ["update inbox set unread_count=0 where luser=",
-        ?ESC(Username), " and lserver=", ?ESC(Server), " and remote_bare_jid=",
-        ?ESC(ToBareJid), " and msg_id=", ?ESC(MsgId), ";"]).
+        esc_string(Username), " and lserver=", esc_string(Server), " and remote_bare_jid=",
+        esc_string(ToBareJid), " and msg_id=", esc_string(MsgId), ";"]).
 
 -spec clear_inbox(Username :: binary(), Server :: binary()) -> ok.
 clear_inbox(Username, Server) ->
@@ -118,10 +126,18 @@ clear_inbox(Username, Server) ->
     Res = clear_inbox_rdbms(LUsername, LServer),
     check_result(Res).
 
+-spec esc_string(binary() | string()) -> mongoose_rdbms:sql_query_part().
+esc_string(String) ->
+    mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(String)).
+
+%% ----------------------------------------------------------------------
+%% Internal functions
+%% ----------------------------------------------------------------------
+
 -spec clear_inbox_rdbms(Username :: jid:luser(), Server :: jid:lserver()) -> query_result().
 clear_inbox_rdbms(Username, Server) ->
     mongoose_rdbms:sql_query(Server, ["delete from inbox where luser=",
-        ?ESC(Username), " and lserver=", ?ESC(Server), ";"]).
+        esc_string(Username), " and lserver=", esc_string(Server), ";"]).
 
 -spec decode_row(host(), {username(), binary(), count()}) -> inbox_res().
 decode_row(LServer, {Username, Content, Count}) ->
@@ -132,16 +148,13 @@ decode_row(LServer, {Username, Content, Count}) ->
 
 
 odbc_specific_backend(Host) ->
-    Type = mongoose_rdbms:db_engine(Host),
-    try
-        list_to_existing_atom("mod_inbox_odbc_" ++ atom_to_list(Type))
-    catch Err:Type ->
-        ?ERROR_MSG(
-            "Error when creating inbox backend module ~p, err:~p:~p",
-            [Type, Err, Type])
+    case {mongoose_rdbms:db_engine(Host), mongoose_rdbms_type:get()} of
+        {mysql, _} -> mod_inbox_odbc_mysql;
+        {pgsql, _} -> mod_inbox_odbc_pgsql;
+        {odbc, mssql} -> mod_inbox_odbc_mssql;
+        NotSupported -> erlang:error({rdbms_not_supported, NotSupported})
     end.
 
-
 count_to_bin(Count) when is_integer(Count) -> integer_to_binary(Count);
 count_to_bin(Count) when is_binary(Count) -> Count.
 
@@ -162,4 +175,4 @@ check_result(Result, _) ->
 check_result({updated, _}) ->
     ok;
 check_result(Result) ->
-    {error, {bad_result, Result}}.
+    {error, {bad_result, Result}}.

src/inbox/mod_inbox_odbc_mssql.erl

@@ -0,0 +1,80 @@
+%%%-------------------------------------------------------------------
+%%% @author piotr.nosek@erlang-solutions.com
+%%% @copyright (C) 2018, Erlang Solutions Ltd.
+%%% @doc
+%%%
+%%% @end
+%%% Created : 5. Jul 2018
+%%%-------------------------------------------------------------------
+
+-module(mod_inbox_odbc_mssql).
+-author("piotr.nosek@erlang-solutions.com").
+
+-include("mod_inbox.hrl").
+
+-export([set_inbox/6, set_inbox_incr_unread/5]).
+
+-import(mod_inbox_odbc, [esc_string/1]).
+
+%% -----------------------------------------------------------
+%% API
+%% -----------------------------------------------------------
+
+-spec set_inbox(Username :: jid:luser(),
+                Server :: jid:lserver(),
+                ToBareJid :: binary(),
+                Content :: binary(),
+                Count :: binary(),
+                MsgId :: binary()) -> query_result().
+set_inbox(Username, Server, ToBareJid, Content, Count, MsgId) ->
+    Query = build_query(Username, Server, ToBareJid, Content, Count, MsgId),
+    mongoose_rdbms:sql_query(Server, Query).
+
+
+-spec set_inbox_incr_unread(Username :: jid:luser(),
+                            Server :: jid:lserver(),
+                            ToBareJid :: binary(),
+                            Content :: binary(),
+                            MsgId :: binary()) -> query_result().
+set_inbox_incr_unread(Username, Server, ToBareJid, Content, MsgId) ->
+    Query = build_query(Username, Server, ToBareJid, Content, increment, MsgId),
+    mongoose_rdbms:sql_query(Server, Query).
+
+%% -----------------------------------------------------------
+%% Internal functions
+%% -----------------------------------------------------------
+
+build_query(Username, Server, ToBareJid, Content, increment, MsgId) ->
+    CountUpdate = "target.unread_count + 1",
+    build_query(Username, Server, ToBareJid, Content, MsgId, esc_string("1"), CountUpdate);
+build_query(Username, Server, ToBareJid, Content, CountValue, MsgId) ->
+    ECount = esc_string(CountValue),
+    build_query(Username, Server, ToBareJid, Content, MsgId, ECount, ECount).
+
+build_query(Username, Server, ToBareJid, Content, MsgId, CountInsert, CountUpdate) ->
+    ELUser = esc_string(Username),
+    ELServer = esc_string(Server),
+    EToBareJid = esc_string(ToBareJid),
+    EContent = mongoose_rdbms:use_escaped_binary(mongoose_rdbms:escape_binary(Server, Content)),
+    EMsgId = esc_string(MsgId),
+
+    ["MERGE INTO inbox WITH (SERIALIZABLE) AS target"
+      " USING (SELECT ",
+                    ELUser, " as luser, ",
+                    ELServer, " as lserver, ",
+                    EToBareJid, " as remote_bare_jid, ",
+                    EContent, " as content, ",
+                    CountInsert, " as unread_count, ",
+                    EMsgId, " as msg_id)"
+      " AS source (luser, lserver, remote_bare_jid, content, unread_count, msg_id)"
+          " ON (target.luser = source.luser"
+          " AND target.lserver = source.lserver"
+          " AND target.remote_bare_jid = source.remote_bare_jid)"
+      " WHEN MATCHED THEN UPDATE"
+          " SET content = ", EContent, ","
+              " unread_count = ", CountUpdate, ","
+              " msg_id = ", EMsgId,
+      " WHEN NOT MATCHED THEN INSERT"
+          " (luser, lserver, remote_bare_jid, content, unread_count, msg_id)"
+      " VALUES (", ELUser, ", ", ELServer, ", ", EToBareJid, ", ", EContent, ", ",
+                CountInsert, ", ", EMsgId, ");"].

src/inbox/mod_inbox_odbc_mysql.erl

@@ -7,12 +7,18 @@
 %%% Created : 16. May 2018 12:51
 %%%-------------------------------------------------------------------
 -module(mod_inbox_odbc_mysql).
+
 -author("ludwikbukowski").
+
 -include("mod_inbox.hrl").
-%% API
+
 -export([set_inbox/6, set_inbox_incr_unread/5]).
--define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))).
 
+-import(mod_inbox_odbc, [esc_string/1]).
+
+%% --------------------------------------------------------
+%% API
+%% --------------------------------------------------------
 
 -spec set_inbox(Username :: jid:luser(),
                 Server :: jid:lserver(),
@@ -21,11 +27,15 @@
                 Count :: binary(),
                 MsgId :: binary()) -> query_result().
 set_inbox(Username, Server, ToBareJid, Content, Count, MsgId) ->
-    mongoose_rdbms:sql_query(Server, ["insert into inbox(luser, lserver, remote_bare_jid, content,
-    unread_count, msg_id) values (", ?ESC(Username), ",", ?ESC(Server), ",", ?ESC(ToBareJid), ",",
-        ?ESC(Content), ",", ?ESC(Count), ",", ?ESC(MsgId),
-        ") on duplicate key update content=",
-        ?ESC(Content), ", unread_count=", ?ESC(Count), ",msg_id=", ?ESC(MsgId), ";"]).
+    mongoose_rdbms:sql_query(Server,
+        ["insert into inbox(luser, lserver, remote_bare_jid, content, unread_count, msg_id) "
+               "values (", esc_string(Username), ",", esc_string(Server), ",",
+                           esc_string(ToBareJid), ",", esc_string(Content), ",",
+                           esc_string(Count), ",", esc_string(MsgId),
+                       ") on duplicate key "
+               "update content=", esc_string(Content),
+               ", unread_count=", esc_string(Count),
+               ", msg_id=", esc_string(MsgId), ";"]).
 
 
 -spec set_inbox_incr_unread(Username :: jid:luser(),
@@ -34,12 +44,12 @@ set_inbox(Username, Server, ToBareJid, Content, Count, MsgId) ->
                             Content :: binary(),
                             MsgId :: binary()) -> query_result().
 set_inbox_incr_unread(Username, Server, ToBareJid, Content, MsgId) ->
-    mongoose_rdbms:sql_query(Server, ["insert into inbox(luser, lserver, remote_bare_jid,
-    content, unread_count, msg_id) values (", ?ESC(Username), ", ", ?ESC(Server), ",",
-        ?ESC(ToBareJid), ", ", ?ESC(Content), ", ", "1", ", ",
-        ?ESC(MsgId), ") on duplicate key update content=",
-        ?ESC(Content), ", unread_count=inbox.unread_count + 1, msg_id=",
-        ?ESC(MsgId), ";"]).
-
-
+    mongoose_rdbms:sql_query(Server,
+        ["insert into inbox(luser, lserver, remote_bare_jid, content, unread_count, msg_id) "
+               "values (", esc_string(Username), ", ", esc_string(Server), ",",
+                           esc_string(ToBareJid), ", ", esc_string(Content), ", ", "1", ", ",
+                           esc_string(MsgId), ") on duplicate key "
+               "update content=", esc_string(Content),
+               ", unread_count=inbox.unread_count + 1, "
+               "msg_id=", esc_string(MsgId), ";"]).
 

src/inbox/mod_inbox_odbc_pgsql.erl

@@ -7,12 +7,18 @@
 %%% Created : 16. May 2018 12:51
 %%%-------------------------------------------------------------------
 -module(mod_inbox_odbc_pgsql).
+
 -author("ludwikbukowski").
+
 -include("mod_inbox.hrl").
-%% API
+
 -export([set_inbox/6, set_inbox_incr_unread/5]).
 
--define(ESC(T), mongoose_rdbms:use_escaped_string(mongoose_rdbms:escape_string(T))).
+-import(mod_inbox_odbc, [esc_string/1]).
+
+%% ---------------------------------------------------------
+%% API
+%% ---------------------------------------------------------
 
 -spec set_inbox(Username :: jid:luser(),
                 Server :: jid:lserver(),
@@ -21,21 +27,28 @@
                 Count :: binary(),
                 MsgId :: binary()) -> query_result().
 set_inbox(Username, Server, ToBareJid, Content, Count, MsgId) ->
-    mongoose_rdbms:sql_query(Server, ["insert into inbox(luser, lserver, remote_bare_jid, content,
-    unread_count, msg_id) values (", ?ESC(Username), ",", ?ESC(Server), ",", ?ESC(ToBareJid), ",",
-        ?ESC(Content), ",", ?ESC(Count), ",", ?ESC(MsgId),
-        ") on conflict(luser, lserver, remote_bare_jid)do update set content=",
-        ?ESC(Content), ", unread_count=", ?ESC(Count), ",msg_id=", ?ESC(MsgId), ";"]).
+    mongoose_rdbms:sql_query(Server,
+        ["insert into inbox(luser, lserver, remote_bare_jid, content, unread_count, msg_id) "
+                "values (", esc_string(Username), ",", esc_string(Server), ",",
+                            esc_string(ToBareJid), ",", esc_string(Content), ",",
+                            esc_string(Count), ",", esc_string(MsgId),
+                       ") on conflict (luser, lserver, remote_bare_jid) do "
+            "update set content=", esc_string(Content),
+                     ", unread_count=", esc_string(Count),
+                     ", msg_id=", esc_string(MsgId), ";"]).
 
 -spec set_inbox_incr_unread(Username :: jid:luser(),
                             Server :: jid:lserver(),
                             ToBareJid :: binary(),
                             Content :: binary(),
                             MsgId :: binary()) -> query_result().
 set_inbox_incr_unread(Username, Server, ToBareJid, Content, MsgId) ->
-    mongoose_rdbms:sql_query(Server, ["insert into inbox(luser, lserver, remote_bare_jid,
-    content, unread_count, msg_id) values (", ?ESC(Username), ", ", ?ESC(Server), ",",
-        ?ESC(ToBareJid), ", ", ?ESC(Content), ", ", "1", ", ",
-        ?ESC(MsgId), ") on conflict(luser, lserver, remote_bare_jid) do update set content=",
-        ?ESC(Content), ", unread_count=inbox.unread_count + 1, msg_id=",
-        ?ESC(MsgId), ";"]).
+    mongoose_rdbms:sql_query(Server,
+        ["insert into inbox(luser, lserver, remote_bare_jid, content, unread_count, msg_id) "
+                "values (", esc_string(Username), ", ", esc_string(Server), ",",
+                            esc_string(ToBareJid), ", ", esc_string(Content), ", ", "1", ", ",
+                            esc_string(MsgId), ") on conflict (luser, lserver, remote_bare_jid) do "
+            "update set content=", esc_string(Content), ", "
+                       "unread_count=inbox.unread_count + 1, "
+                       "msg_id=", esc_string(MsgId), ";"]).
+