Merge 27fb6de into 48a339f

.travis.yml

@@ -84,11 +84,11 @@ env:
     - RUN_SMALL_TESTS=false
   matrix:
       # When changing jobs, update EXAMPLES in tools/test-runner.sh
-    - PRESET=small_tests RUN_SMALL_TESTS=true SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
-    - PRESET=internal_mnesia DB=mnesia REL_CONFIG=with-all TLS_DIST=yes
+    #- PRESET=small_tests RUN_SMALL_TESTS=true SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
+    #- PRESET=internal_mnesia DB=mnesia REL_CONFIG=with-all TLS_DIST=yes
     - PRESET=mysql_redis DB=mysql REL_CONFIG="with-mysql with-redis"
     - PRESET=odbc_mssql_mnesia DB=mssql REL_CONFIG=with-odbc
-    - PRESET=ldap_mnesia DB=mnesia REL_CONFIG=with-none
+    #- PRESET=ldap_mnesia DB=mnesia REL_CONFIG=with-none
     - PRESET=elasticsearch_and_cassandra_mnesia DB="elasticsearch cassandra"
       REL_CONFIG="with-elasticsearch with-cassandra" TESTSPEC=mam.spec
       ELASTICSEARCH_VERSION=5.6.9 CASSANDRA_VERSION=3.9
@@ -106,14 +106,14 @@ matrix:
            RUN_SMALL_TESTS=true
     - otp_release: 21.0
       env: PRESET=riak_mnesia DB=riak REL_CONFIG="with-riak with-jingle-sip" RUN_SMALL_TESTS=true
-    - language: generic
-      env: PRESET=pkg pkg_PLATFORM=centos7
-           SKIP_COMPILE=1 SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
-           SKIP_COV=1 SKIP_REPORT_UPLOAD=1
-    - language: generic
-      env: PRESET=pkg pkg_PLATFORM=debian_stretch
-           SKIP_COMPILE=1 SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
-           SKIP_COV=1 SKIP_REPORT_UPLOAD=1
+    #- language: generic
+    #  env: PRESET=pkg pkg_PLATFORM=centos7
+    #       SKIP_COMPILE=1 SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
+    #       SKIP_COV=1 SKIP_REPORT_UPLOAD=1
+    #- language: generic
+    #  env: PRESET=pkg pkg_PLATFORM=debian_stretch
+    #       SKIP_COMPILE=1 SKIP_RELEASE=1 SKIP_BUILD_TESTS=1
+    #       SKIP_COV=1 SKIP_REPORT_UPLOAD=1
 
 notifications:
     webhooks:

big_tests/default.spec

@@ -11,70 +11,11 @@
 %% do not remove below SUITE if testing mongoose
 {suites, "tests", mongoose_sanity_checks_SUITE}.
 
-{suites, "tests", rdbms_SUITE}.
-{suites, "tests", race_conditions_SUITE}.
-{suites, "tests", acc_e2e_SUITE}.
-{suites, "tests", accounts_SUITE}.
-{suites, "tests", adhoc_SUITE}.
-{suites, "tests", amp_big_SUITE}.
-{suites, "tests", anonymous_SUITE}.
-{suites, "tests", bosh_SUITE}.
-{suites, "tests", carboncopy_SUITE}.
-{suites, "tests", cluster_commands_SUITE}.
-{suites, "tests", conf_reload_SUITE}.
-{suites, "tests", connect_SUITE}.
-{suites, "tests", component_SUITE}.
-{suites, "tests", disco_and_caps_SUITE}.
-{suites, "tests", ejabberdctl_SUITE}.
-{suites, "tests", inbox_SUITE}.
-{suites, "tests", jingle_SUITE}.
-{suites, "tests", last_SUITE}.
-{suites, "tests", login_SUITE}.
-{suites, "tests", mam_SUITE}.
-{suites, "tests", metrics_api_SUITE}.
-{suites, "tests", metrics_c2s_SUITE}.
-{suites, "tests", metrics_roster_SUITE}.
-{suites, "tests", metrics_register_SUITE}.
-{suites, "tests", metrics_session_SUITE}.
-{suites, "tests", mod_aws_sns_SUITE}.
-{suites, "tests", mod_blocking_SUITE}.
-{suites, "tests", mod_http_notification_SUITE}.
-{suites, "tests", mod_http_upload_SUITE}.
-{suites, "tests", mod_ping_SUITE}.
-{suites, "tests", mod_time_SUITE}.
-{suites, "tests", mod_version_SUITE}.
-{suites, "tests", muc_SUITE}.
-{suites, "tests", muc_light_SUITE}.
-{suites, "tests", muc_light_legacy_SUITE}.
-{suites, "tests", muc_http_api_SUITE}.
-{suites, "tests", muc_light_http_api_SUITE}.
-{suites, "tests", oauth_SUITE}.
-{suites, "tests", offline_SUITE}.
-{suites, "tests", pep_SUITE}.
-{suites, "tests", presence_SUITE}.
-{suites, "tests", privacy_SUITE}.
-{suites, "tests", private_SUITE}.
-{suites, "tests", pubsub_SUITE}.
-{suites, "tests", push_SUITE}.
-{suites, "tests", push_http_SUITE}.
-{suites, "tests", push_integration_SUITE}.
-{suites, "tests", push_pubsub_SUITE}.
-{suites, "tests", rest_SUITE}.
-{suites, "tests", rest_client_SUITE}.
-{suites, "tests", s2s_SUITE}.
-{suites, "tests", sasl_SUITE}.
-{suites, "tests", shared_roster_SUITE}.
-{suites, "tests", sic_SUITE}.
-{suites, "tests", sm_SUITE}.
-{suites, "tests", users_api_SUITE}.
-{suites, "tests", vcard_simple_SUITE}.
-{suites, "tests", vcard_SUITE}.
-{suites, "tests", websockets_SUITE}.
-{suites, "tests", xep_0352_csi_SUITE}.
-{suites, "tests", mod_global_distrib_SUITE}.
-{suites, "tests", mongoose_cassandra_SUITE}.
-{suites, "tests", mongoose_elasticsearch_SUITE}.
-{suites, "tests", sasl_external_SUITE}.
+{groups, "tests", mam_SUITE, [rsm04, muc_rsm04],
+ {cases, [
+          server_returns_item_not_found_for_before_filter_with_nonexistent_id,
+          server_returns_item_not_found_for_after_filter_with_nonexistent_id
+         ]}}.
 
 {config, ["test.config"]}.
 {logdir, "ct_report"}.

big_tests/tests/mam_SUITE.erl

@@ -75,6 +75,8 @@
          pagination_last25_opt_count_all/1,
          pagination_offset5_opt_count/1,
          pagination_offset5_opt_count_all/1,
+         server_returns_item_not_found_for_before_filter_with_nonexistent_id/1,
+         server_returns_item_not_found_for_after_filter_with_nonexistent_id/1,
          %% complete_flag_cases tests
          before_complete_false_last5/1,
          before_complete_false_before10/1,
@@ -459,7 +461,10 @@ rsm_cases() ->
        %% opt_count cases with all messages on the page
        pagination_first25_opt_count_all,
        pagination_last25_opt_count_all,
-       pagination_offset5_opt_count_all].
+       pagination_offset5_opt_count_all,
+       %% item_not_found response for nonexistent message ID in before/after filters
+       server_returns_item_not_found_for_before_filter_with_nonexistent_id,
+       server_returns_item_not_found_for_after_filter_with_nonexistent_id].
 
 complete_flag_cases() ->
     [before_complete_false_last5,
@@ -2410,6 +2415,31 @@ pagination_last_after_id5_before_id11(Config) ->
         end,
     parallel_story(Config, [{alice, 1}], F).
 
+server_returns_item_not_found_for_before_filter_with_nonexistent_id(Config) ->
+    NonexistentID = <<"AV25E9SCO50K">>,
+    RSM = #rsm_in{max = 5, direction = 'before', id = NonexistentID},
+    StanzaID = <<"before-nonexistent-id">>,
+    server_returns_item_not_found_for_nonexistent_id(Config, RSM, StanzaID).
+
+server_returns_item_not_found_for_after_filter_with_nonexistent_id(Config) ->
+    NonexistentID = <<"AV25E9SCO50K">>,
+    RSM = #rsm_in{max = 5, direction = 'after', id = NonexistentID},
+    StanzaID = <<"after-nonexistent-id">>,
+    server_returns_item_not_found_for_nonexistent_id(Config, RSM, StanzaID).
+
+server_returns_item_not_found_for_nonexistent_id(Config, RSM, StanzaID) ->
+    P = ?config(props, Config),
+    F = fun(Alice) ->
+        rsm_send(Config, Alice,
+                 stanza_page_archive_request(P, StanzaID, RSM)),
+        Res = escalus:wait_for_stanza(Alice),
+        ct:pal("res: ~p\n", [Res]),
+        escalus:assert(is_iq_error, Res),
+        escalus:assert(is_error, [<<"cancel">>, <<"item-not-found">>], Res),
+        ok
+        end,
+    parallel_story(Config, [{alice, 1}], F).
+
 
 %% Test cases for "complete" attribute
 %% Complete attribute is used for pagination, telling when to stop paginating.

src/mam/mod_mam.erl

@@ -661,6 +661,8 @@ return_error_iq(IQ, {Reason, {stacktrace, _Stacktrace}}) ->
     return_error_iq(IQ, Reason);
 return_error_iq(IQ, timeout) ->
     {error, timeout, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:service_unavailable()]}};
+return_error_iq(IQ, item_not_found) ->
+    {error, item_not_found, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:item_not_found()]}};
 return_error_iq(IQ, not_implemented) ->
     {error, not_implemented, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:feature_not_implemented()]}};
 return_error_iq(IQ, Reason) ->

src/mam/mod_mam_cassandra_arch.erl

@@ -30,6 +30,7 @@
 -import(mod_mam_utils,
         [maybe_min/2,
          maybe_max/2,
+         maybe_last/1,
          bare_jid/1,
          full_jid/1
          ]).
@@ -388,16 +389,29 @@ lookup_messages_before_id(PoolName, Host, UserJID,
                           PageSize, Filter) ->
     TotalCount = calc_count(PoolName, UserJID, Host, Filter),
     Offset = calc_offset(PoolName, UserJID, Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(PoolName, UserJID, Host, before_id(ID, Filter), PageSize, true),
-    {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows)}}.
+    MessageRows = extract_messages(PoolName, UserJID, Host, to_id(ID, Filter),
+                                   PageSize + 1, true),
+    case maybe_last(MessageRows) of
+        {ok, #{id := ID}} = _IntervalEndpoint ->
+            Page = lists:sublist(MessageRows, PageSize),
+            {ok, {TotalCount, Offset, rows_to_uniform_format(Page)}};
+        undefined ->
+            {error, item_not_found}
+    end.
 
 lookup_messages_after_id(PoolName, Host, UserJID,
                          RSM = #rsm_in{direction = aft, id = ID},
                          PageSize, Filter) ->
     TotalCount = calc_count(PoolName, UserJID, Host, Filter),
     Offset = calc_offset(PoolName, UserJID, Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(PoolName, UserJID, Host, after_id(ID, Filter), PageSize, false),
-    {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows)}}.
+    MessageRows0 = extract_messages(PoolName, UserJID, Host, from_id(ID, Filter),
+                                    PageSize + 1, false),
+    case MessageRows0 of
+        [#{id := ID} = _IntervalEndpoint | MessageRows] ->
+            {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows)}};
+        _ ->
+            {error, item_not_found}
+    end.
 
 
 after_id(ID, Filter = #mam_ca_filter{start_id = AfterID}) ->
@@ -497,8 +511,7 @@ calc_before(PoolName, UserJID, Host, Filter, MessID) ->
 calc_count(PoolName, UserJID, _Host, Filter) ->
     QueryName = {calc_count_query, select_filter(Filter)},
     Params = eval_filter_params(Filter),
-    {ok, [#{count := Count}]} = mongoose_cassandra:cql_read(PoolName, UserJID, ?MODULE, QueryName,
-                                              Params),
+    {ok, [#{count := Count}]} = mongoose_cassandra:cql_read(PoolName, UserJID, ?MODULE, QueryName, Params),
     Count.
 
 %% @doc Convert offset to index of the first entry
@@ -633,10 +646,10 @@ prepare_filter_cql(StartID, EndID) ->
         undefined -> "";
         _ -> " AND id >= :start_id"
     end ++
-        case EndID of
-            undefined -> "";
-            _ -> " AND id <= :end_id"
-        end.
+    case EndID of
+        undefined -> "";
+        _ -> " AND id <= :end_id"
+    end.
 
 filter_to_cql() ->
     [{select_filter(StartID, EndID), prepare_filter_cql(StartID, EndID)}

src/mam/mod_mam_muc.erl

@@ -575,6 +575,8 @@ return_error_iq(IQ, {Reason, {stacktrace, _Stacktrace}}) ->
     return_error_iq(IQ, Reason);
 return_error_iq(IQ, timeout) ->
     {error, timeout, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:service_unavailable()]}};
+return_error_iq(IQ, item_not_found) ->
+    {error, item_not_found, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:item_not_found()]}};
 return_error_iq(IQ, not_implemented) ->
     {error, not_implemented, IQ#iq{type = error, sub_el = [mongoose_xmpp_errors:feature_not_implemented()]}};
 return_error_iq(IQ, missing_with_jid) ->

src/mam/mod_mam_muc_cassandra_arch.erl

@@ -33,6 +33,7 @@
 -import(mod_mam_utils,
         [maybe_min/2,
          maybe_max/2,
+         maybe_last/1,
          apply_start_border/2,
          apply_end_border/2]).
 
@@ -441,17 +442,30 @@ lookup_messages_before_id(PoolName, Host, RoomJID,
                           PageSize, Filter) ->
     TotalCount = calc_count(PoolName, RoomJID, Host, Filter),
     Offset = calc_offset(PoolName, RoomJID, Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(PoolName, RoomJID, Host, before_id(ID, Filter), PageSize, true),
-    {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows, RoomJID)}}.
+    MessageRows = extract_messages(PoolName, RoomJID, Host, to_id(ID, Filter),
+                                   PageSize + 1, true),
+    case maybe_last(MessageRows) of
+        {ok, #{id := ID}} = _IntervalEndpoint ->
+            Page = lists:sublist(MessageRows, PageSize),
+            {ok, {TotalCount, Offset, rows_to_uniform_format(Page, RoomJID)}};
+        undefined ->
+            {error, item_not_found}
+    end.
 
 lookup_messages_after_id(PoolName, Host, RoomJID,
                          RSM = #rsm_in{direction = aft, id = ID},
                          PageSize, Filter) ->
     PoolName = pool_name(RoomJID),
     TotalCount = calc_count(PoolName, RoomJID, Host, Filter),
     Offset = calc_offset(PoolName, RoomJID, Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(PoolName, RoomJID, Host, after_id(ID, Filter), PageSize, false),
-    {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows, RoomJID)}}.
+    MessageRows0 = extract_messages(PoolName, RoomJID, Host, from_id(ID, Filter),
+                                    PageSize + 1, false),
+    case MessageRows0 of
+        [#{id := ID} = _IntervalEndpoint | MessageRows] ->
+            {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows, RoomJID)}};
+        _ ->
+            {error, item_not_found}
+    end.
 
 
 after_id(ID, Filter = #mam_muc_ca_filter{start_id = AfterID}) ->

src/mam/mod_mam_muc_rdbms_arch.erl

@@ -39,7 +39,8 @@
 %% Other
 -import(mod_mam_utils,
         [apply_start_border/2,
-         apply_end_border/2]).
+         apply_end_border/2,
+         maybe_last/1]).
 
 -import(mongoose_rdbms,
         [escape_string/1,
@@ -307,22 +308,31 @@ lookup_messages(Host, RoomID, RoomJID = #jid{},
                 RSM = #rsm_in{direction = aft, id = ID}, Borders,
                 Start, End, _Now, WithJID, SearchText,
                 PageSize, _) ->
-    Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText),
-    TotalCount = calc_count(Host, Filter),
-    Offset     = calc_offset(Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(Host, after_id(ID, Filter), 0, PageSize, false),
-    {ok, {TotalCount, Offset,
-          rows_to_uniform_format(MessageRows, Host, RoomJID)}};
+    Filter       = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText),
+    TotalCount   = calc_count(Host, Filter),
+    Offset       = calc_offset(Host, Filter, PageSize, TotalCount, RSM),
+    MessageRows0 = extract_messages(Host, from_id(ID, Filter), 0, PageSize + 1, false),
+    case MessageRows0 of
+        [#{id := ID} = _IntervalEndpoint | MessageRows] ->
+            {ok, {TotalCount, Offset, rows_to_uniform_format(MessageRows, Host, RoomJID)}};
+        _ ->
+            {error, item_not_found}
+    end;
 lookup_messages(Host, RoomID, RoomJID = #jid{},
                 RSM = #rsm_in{direction = before, id = ID},
                 Borders, Start, End, _Now, WithJID, SearchText,
                 PageSize, _) ->
     Filter = prepare_filter(RoomID, Borders, Start, End, WithJID, SearchText),
     TotalCount = calc_count(Host, Filter),
     Offset     = calc_offset(Host, Filter, PageSize, TotalCount, RSM),
-    MessageRows = extract_messages(Host, before_id(ID, Filter), 0, PageSize, true),
-    {ok, {TotalCount, Offset,
-          rows_to_uniform_format(MessageRows, Host, RoomJID)}};
+    MessageRows = extract_messages(Host, to_id(ID, Filter), 0, PageSize + 1, true),
+    case maybe_last(MessageRows) of
+        {ok, #{id := ID}} = _IntervalEndpoint ->
+            Page = lists:sublist(MessageRows, PageSize),
+            {ok, {TotalCount, Offset, rows_to_uniform_format(Page, Host, RoomJID)}};
+        undefined ->
+            {error, item_not_found}
+    end;
 lookup_messages(Host, RoomID, RoomJID = #jid{},
                 RSM, Borders,
                 Start, End, _Now, WithJID, SearchText,
@@ -347,6 +357,16 @@ before_id(ID, Filter) ->
     SID = escape_message_id(ID),
     [Filter, " AND id < ", use_escaped_integer(SID)].
 
+-spec from_id(ID :: escaped_message_id(), Filter :: filter()) -> filter().
+from_id(ID, Filter) ->
+    SID = escape_message_id(ID),
+    [Filter, " AND id >= ", use_escaped_integer(SID)].
+
+-spec to_id(ID :: escaped_message_id(), Filter :: filter()) -> filter().
+to_id(ID, Filter) ->
+    SID = escape_message_id(ID),
+    [Filter, " AND id <= ", use_escaped_integer(SID)].
+
 
 -spec rows_to_uniform_format([raw_row()], jid:server(), jid:jid()) ->
                                     [mod_mam_muc:row()].