Merge branch 'master' into toml-config

.circleci/config.yml

@@ -141,7 +141,7 @@ orbs:
           - run:
               name: Run Small Tests
               command: |
-                SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/travis-test.sh -p small_tests -s true
+                SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 ./tools/travis-test.sh -p small_tests -s true -e true
           - run:
               name: Coverage
               when: on_success

.gitignore

@@ -67,6 +67,7 @@ compile_commands.json
 Mnesia.mongooseim@*
 compile.log
 ct.log
+eunit.log
 deps.log
 dialyzer
 configure.out

.travis.yml

@@ -45,7 +45,7 @@ before_script:
         - if [ $PRESET = 'ldap_mnesia' ]; then sudo tools/travis-setup-ldap.sh; fi
         - if [ $PRESET = 'mysql_redis' ]; then sudo tools/travis-setup-rmq.sh; fi
 script:
-    - SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 tools/travis-test.sh -p $PRESET -s $RUN_SMALL_TESTS
+    - SKIP_AUTO_COMPILE=true KEEP_COVER_RUNNING=1 tools/travis-test.sh -p $PRESET -s $RUN_SMALL_TESTS -e $RUN_SMALL_TESTS
 
 after_failure:
         - if [ -s _build/mim1/rel/mongooseim/log/crash.log ]; then cat _build/mim1/rel/mongooseim/log/crash.log; fi

Makefile

@@ -23,6 +23,9 @@ ct:
 		then $(RUN) $(REBAR) ct --dir test --suite $(SUITE) ; \
 		else $(RUN) $(REBAR) ct $(REBAR_CT_EXTRA_ARGS); fi)
 
+eunit:
+	@$(RUN) $(REBAR) eunit
+
 rel: certs configure.out rel/vars.config rel/vars-toml.config
 	. ./configure.out && $(REBAR) as prod release
 

big_tests/tests/mod_http_notification_SUITE.erl

@@ -40,7 +40,6 @@ all_tests() ->
      simple_message,
      simple_message_no_listener,
      simple_message_failing_listener,
-     listener_up_and_down_and_up_again,
      proper_http_message_encode_decode
     ].
 
@@ -136,35 +135,6 @@ do_simple_message(Config0, Msg) ->
     escalus_new_assert:mix_match([is_presence, is_chat(Msg)], Stanzas),
     escalus_client:stop(Config, Bob).
 
-listener_up_and_down_and_up_again(Config) ->
-    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}],
-        fun(Alice, Bob) ->
-            escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Msg1">>)),
-            escalus:wait_for_stanza(Bob),
-
-            Body1 = get_http_request(),
-            http_request_body_as(Body1, <<"Msg1">>),
-
-            stop_http_listener(?FUNCTION_NAME),
-
-            escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Msg2">>)),
-            escalus:wait_for_stanza(Bob),
-
-            start_http_listener(?FUNCTION_NAME, get_prefix(Config)),
-
-            escalus:send(Alice, escalus_stanza:chat_to(Bob, <<"Msg3">>)),
-            escalus:wait_for_stanza(Bob),
-
-            Body3 = get_http_request(),
-            http_request_body_as(Body3, <<"Msg3">>)
-
-        end).
-
-http_request_body_as(Body, ExpectedMsg) ->
-    Expected = {<<"message">>, ExpectedMsg},
-    ExtractedAndDecoded = rpc(mim(), cow_qs, parse_qs, [Body]),
-    ?assertEqual(Expected, lists:keyfind(<<"message">>, 1, ExtractedAndDecoded)).
-
 %%%===================================================================
 %%% Helpers
 %%%===================================================================
@@ -217,8 +187,6 @@ start_http_listener(simple_message_no_listener, _) ->
 start_http_listener(simple_message_failing_listener, Prefix) ->
     http_helper:start(http_notifications_port(), Prefix, fun(Req) -> Req end);
 start_http_listener(proper_http_message_encode_decode, Prefix) ->
-    http_helper:start(http_notifications_port(), Prefix, fun process_notification/1);
-start_http_listener(listener_up_and_down_and_up_again, Prefix) ->
     http_helper:start(http_notifications_port(), Prefix, fun process_notification/1).
 
 stop_http_listener(simple_message_no_listener) ->

big_tests/tests/mod_http_upload_SUITE.erl

@@ -14,7 +14,7 @@
 -define(S3_HOSTNAME, "http://bucket.s3-eu-east-25.example.com").
 -define(S3_OPTS, ?MOD_HTTP_UPLOAD_OPTS(?S3_HOSTNAME, true)).
 
--define(MINIO_HOSTNAME, "http://localhost:9000/mybucket/").
+-define(MINIO_HOSTNAME, "http://127.0.0.1:9000/mybucket/").
 -define(MINIO_OPTS(AddAcl), ?MOD_HTTP_UPLOAD_OPTS(?MINIO_HOSTNAME, AddAcl)).
 
 -define(MINIO_TEST_DATA, "qwerty").

big_tests/tests/pubsub_s2s_SUITE.erl

@@ -109,7 +109,7 @@ publish_test(Config) ->
               Node = pubsub_tools:pubsub_node(),
               pubsub_tools:create_node(Alice, Node, []),
               pubsub_tools:publish(Alice, <<"item1">>, Node, []),
-              pubsub_tools:publish(Alice2, <<"item2">>, Node, [{expected_error_type, <<"cancel">>}]),
+              pubsub_tools:publish(Alice2, <<"item2">>, Node, [{expected_error_type, <<"auth">>}]),
               pubsub_tools:delete_node(Alice, Node, [])
       end).
 
@@ -121,7 +121,7 @@ publish_without_node_attr_test(Config) ->
               Node = pubsub_tools:pubsub_node(),
               pubsub_tools:create_node(Alice, Node, []),
               pubsub_tools:publish(Alice, <<"item1">>, Node, []),
-              pubsub_tools:publish_without_node_attr(Alice2, <<"item2">>, Node, [{expected_error_type, <<"cancel">>}]),
+              pubsub_tools:publish_without_node_attr(Alice2, <<"item2">>, Node, [{expected_error_type, <<"modify">>}]),
               pubsub_tools:delete_node(Alice, Node, [])
       end).
 

big_tests/tests/push_http_SUITE.erl

@@ -191,7 +191,7 @@ check_default_format(From, To, Body, Msg) ->
 
 start_pool() ->
     PoolOpts = [{strategy, random_worker}, {call_timeout, 5000}, {workers, 10}],
-    HTTPOpts = [{path_prefix, "/"}, {http_opts, #{}}, {server, http_notifications_host()}],
+    HTTPOpts = [{path_prefix, "/"}, {http_opts, []}, {server, http_notifications_host()}],
     Pool = {http, host, http_pool, PoolOpts, HTTPOpts},
     ejabberd_node_utils:call_fun(mongoose_wpool, start_configured_pools,
                                  [[Pool], [<<"localhost">>]]).

big_tests/tests/push_integration_SUITE.erl

@@ -128,7 +128,7 @@ init_per_suite(Config) ->
     Port = mongoose_push_mock:port(),
 
     PoolOpts = [{strategy, available_worker}, {workers, 20}],
-    HTTPOpts = [{server, {"127.0.0.1", Port}}, {http_opts, #{transport => tls}}],
+    HTTPOpts = [{server, "https://localhost:" ++ integer_to_list(Port)}],
     rpc(?RPC_SPEC, mongoose_wpool, start_configured_pools,
         [[{http, global, mongoose_push_http, PoolOpts, HTTPOpts}]]),
     ConfigWithModules = dynamic_modules:save_modules(domain(), Config),

big_tests/tests/rest_SUITE.erl

@@ -20,6 +20,7 @@
 -include_lib("escalus/include/escalus.hrl").
 -include_lib("common_test/include/ct.hrl").
 -include_lib("eunit/include/eunit.hrl").
+-include_lib("exml/include/exml.hrl").
 
 -import(rest_helper,
         [assert_inlist/2,
@@ -83,6 +84,7 @@ test_cases() ->
      sessions_are_listed,
      session_can_be_kicked,
      messages_are_sent_and_received,
+     stanzas_are_sent_and_received,
      messages_are_archived,
      messages_can_be_paginated,
      password_can_be_changed
@@ -205,14 +207,15 @@ messages_are_sent_and_received(Config) ->
         escalus:assert(is_chat_message, [maps:get(body, M2)], Res1)
     end).
 
-send_messages(Alice, Bob) ->
-        AliceJID = escalus_utils:jid_to_lower(escalus_client:short_jid(Alice)),
-        BobJID = escalus_utils:jid_to_lower(escalus_client:short_jid(Bob)),
-        M = #{caller => BobJID, to => AliceJID, body => <<"hello from Bob">>},
-        {?NOCONTENT, _} = post(admin, <<"/messages">>, M),
-        M1 = #{caller => AliceJID, to => BobJID, body => <<"hello from Alice">>},
-        {?NOCONTENT, _} = post(admin, <<"/messages">>, M1),
-        {M, M1}.
+stanzas_are_sent_and_received(Config) ->
+%%    this is to test the API for sending arbitrary stanzas, e.g. message with extra elements
+    escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
+        send_extended_message(Alice, Bob),
+        Res = escalus:wait_for_stanza(Bob),
+        ?assertEqual(<<"attribute">>, exml_query:attr(Res, <<"extra">>)),
+        ?assertEqual(<<"inside the sibling">>, exml_query:path(Res, [{element, <<"sibling">>}, cdata])),
+        ok
+                                                        end).
 
 messages_are_archived(Config) ->
     escalus:fresh_story(Config, [{alice, 1}, {bob, 1}], fun(Alice, Bob) ->
@@ -432,6 +435,30 @@ befriend_and_alienate_auto(Config) ->
 %% Helpers
 %%--------------------------------------------------------------------
 
+send_messages(Alice, Bob) ->
+    AliceJID = escalus_utils:jid_to_lower(escalus_client:short_jid(Alice)),
+    BobJID = escalus_utils:jid_to_lower(escalus_client:short_jid(Bob)),
+    M = #{caller => BobJID, to => AliceJID, body => <<"hello from Bob">>},
+    {?NOCONTENT, _} = post(admin, <<"/messages">>, M),
+    M1 = #{caller => AliceJID, to => BobJID, body => <<"hello from Alice">>},
+    {?NOCONTENT, _} = post(admin, <<"/messages">>, M1),
+    {M, M1}.
+
+send_extended_message(From, To) ->
+    M = #xmlel{name = <<"message">>,
+               attrs = [{<<"from">>, escalus_client:full_jid(From)},
+                        {<<"to">>, escalus_client:full_jid(To)},
+                        {<<"extra">>, <<"attribute">>}],
+               children = [#xmlel{name = <<"body">>,
+                                  children = [#xmlcdata{content = <<"the body">>}]},
+                           #xmlel{name = <<"sibling">>,
+                                  children = [#xmlcdata{content = <<"inside the sibling">>}]}
+               ]
+    },
+    M1 = #{stanza => exml:to_binary(M)},
+    {?NOCONTENT, _} = post(admin, <<"/stanzas">>, M1),
+    ok.
+
 check_roster(Path, Jid, Subs, Ask) ->
     {?OK, R} = gett(admin, Path),
     S = atom_to_binary(Subs, latin1),

doc/advanced-configuration/outgoing-connections.md

@@ -207,21 +207,20 @@ Recommended `PoolOptions` for `HTTP` pools are:
 
 `ConnectionOptions` can take the following `{key, value}` pairs:
 
-* `{server, HostName}` - string - the URL of the destination HTTP server (including a port number if needed).
-* `{path_prefix, Prefix}` - string, default: `"/"` - the part of the destination URL that is appended to the host name (`Host` option).
+* `{server, HostName}` - string, default: `"http://localhost"` - the URL of the destination HTTP server (including a port number if needed).
+* `{path_prefix, Prefix}` - string, default: `"/"` - the part of the destination URL that is appended to the host name (`host` option).
 * `{request_timeout, TimeoutValue}` - non-negative integer, default: `2000` - maximum number of milliseconds to wait for the HTTP response.
-* `{http_opts, HTTPOptions}` - extra parameters to the library responsible for making the HTTP calls.
-Defaults to the map: `#{retry => 1, retry_timeout => 1000}`.
-  More details about the possible `http_opts` can be found in [gun]'s documentation.
+* `{http_opts, HTTPOptions}` - list, default: `[]` - can be used to pass extra parameters which are passed to [fusco], the library used for making the HTTP calls.
+  More details about the possible `http_opts` can be found in [fusco]'s documentation.
 
-[gun]: https://ninenines.eu/docs/en/gun/2.0/manual/
+[fusco]: https://github.com/esl/fusco
 
 ##### Example configuration
 
 ```Erlang
 {outgoing_pools, [
   {http, global, http_auth,
-   [{strategy, available_worker}], [{server, {"https://my_server", 8080}}]}
+   [{strategy, available_worker}], [{server, "https://my_server:8080"}]}
 ]}.
 ```
 
@@ -232,14 +231,14 @@ If peer certificate verification is required, the pool can be configured in the
   {http, global, mongoose_push_http,
    [{workers, 50}],
    [{server, "https://localhost:8443"},
-    {http_opts, #{
-                     transport_opts => [{verify, verify_peer}]
-                 }}
+    {http_opts, [
+                 {connect_options, [{verify, verify_peer}]}
+                 ]}
    ]}
 ]}.
 ```
 
-Please note the `transport_opts` passed to [gun] via the pool's `http_opts` parameter.
+Please note the `connect_options` passed to [fusco] via the pool's `http_opts` parameter.
 
 
 ## Redis connection setup

doc/migrations/3.7.0_3.7.x.md

@@ -61,16 +61,3 @@ The equivalent Logger configuration is
 
 [lager]: https://github.com/erlang-lager/lager
 [Logger]: https://erlang.org/doc/apps/kernel/logger_chapter.html
-
-## Migrating the HTTP pool configuration
-
-As we upgraded the underlying library, in order to achieve compatibility with the HTTP/2 protocol,
-its configuration has slightly changed, mainly the options passed directly to the underlying
-library.
-
-With fusco, this was done with the option `{http_opts, Opts}`, where `Opts` was a list of key-value
-pairs. For gun, `Opts` instead takes the form of a map. More details about each library
-configuration can be found in their respective documentations: [gun](gun) and [fusco](fusco)
-
-[fusco]: https://github.com/esl/fusco
-[gun]: https://ninenines.eu/docs/en/gun/2.0/manual/

doc/rest-api/Administration-backend_swagger.yml

@@ -260,6 +260,28 @@ paths:
           description: The messages archived for the given user and the other party.
           schema:
             $ref: '#/definitions/messageList'
+  /stanzas:
+    post:
+      tags:
+        - "One-to-one messages"
+      parameters:
+        - name: message
+          in: body
+          description: The chat stanza
+          required: true
+          schema:
+            title: message
+            type: object
+            properties:
+              stanza:
+                description: the complete stanza
+                type: string
+                required: true
+                example: "<message from=\"alice@wonderlant.lit\" to=\"bob@wonderlant.lit\"><body>whatever</body></message>"
+      description: Send an arbitrary stanza (attributes `from` and `to` are required).
+      responses:
+        204:
+          description: Stanza was sent
   /contacts/{user}:
     get:
       description: "Returns all contacts from the user's roster"

rebar.config

@@ -40,7 +40,6 @@
   {fast_scram, {git, "https://github.com/esl/fast_scram.git", {branch, "master"}}},
   {jiffy, "1.0.5"},
   {fusco, {git, "https://github.com/esl/fusco.git", {branch, "master"}}},
-  {gun, "1.3.2"},
   {idna, "6.0.1"},
   {uuid, "1.8.0", {pkg, uuid_erl}},
   {fast_tls, {git, "https://github.com/processone/fast_tls.git", {ref, "ceb277f"}}},

rebar.lock

@@ -74,7 +74,10 @@
   0},
  {<<"gen_fsm_compat">>,{pkg,<<"gen_fsm_compat">>,<<"0.3.0">>},0},
  {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
- {<<"gun">>,{pkg,<<"gun">>,<<"1.3.2">>},0},
+ {<<"gun">>,
+  {git,"https://github.com/ninenines/gun.git",
+       {ref,"427230d6f94f5b8a396fd504a73d73d1d65ab0a7"}},
+  1},
  {<<"hackney">>,{pkg,<<"hackney">>,<<"1.13.0">>},1},
  {<<"hamcrest">>,
   {git,"https://github.com/basho/hamcrest-erlang.git",
@@ -189,7 +192,6 @@
  {<<"erlcloud">>, <<"4DE8DC3CBA64A59D57E4A32606DB6AA5CAC371CA0ADB8361571B2E6A74295A56">>},
  {<<"gen_fsm_compat">>, <<"5903549F67D595F58A7101154CBE0FDD46955FBFBE40813F1E53C23A970FF5F4">>},
  {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
- {<<"gun">>, <<"542064CBB9F613650B8A8100B3A927505F364FBE198B7A5A112868FF43F3E477">>},
  {<<"hackney">>, <<"24EDC8CD2B28E1C652593833862435C80661834F6C9344E84B6A2255E7AEEF03">>},
  {<<"hut">>, <<"08D46679523043424870723923971889E8A34D63B2F946A35B46CF921D1236E7">>},
  {<<"idna">>, <<"1D038FB2E7668CE41FBF681D2C45902E52B3CB9E9C77B55334353B222C2EE50C">>},

rel/fed1.vars.config

@@ -14,7 +14,7 @@
 {hosts, "[ \"fed1\" ]"}.
 {sm_backend, "{mnesia, []}"}.
 {auth_method, "internal"}.
-{s2s_addr, "{ {s2s_addr, \"localhost\"}, {127,0,0,1} }. { {s2s_addr, \"localhost.bis\"}, {127,0,0,1} }."}.
+{s2s_addr, "{ {s2s_addr, \"localhost\"}, {127,0,0,1} }. { {s2s_addr, \"pubsub.localhost\"}, {127,0,0,1} }. { {s2s_addr, \"muc.localhost\"}, {127,0,0,1} }. { {s2s_addr, \"localhost.bis\"}, {127,0,0,1} }."}.
 {s2s_default_policy, allow}.
 {highload_vm_args, ""}.
 {ejabberd_service, ""}.

src/ejabberd_s2s_out.erl

@@ -378,7 +378,7 @@ wait_for_validation({xmlstreamelement, El}, StateData) ->
     case is_verify_res(El) of
         {result, To, From, Id, Type} ->
             ?LOG_DEBUG(#{what => s2s_receive_result,
-                         from => From, to => To, messag_id => Id, type => Type}),
+                         from => From, to => To, message_id => Id, type => Type}),
             case {Type, StateData#state.tls_enabled, StateData#state.tls_required} of
                 {<<"valid">>, Enabled, Required} when (Enabled==true) or (Required==false) ->
                     send_queue(StateData, StateData#state.queue),
@@ -400,7 +400,7 @@ wait_for_validation({xmlstreamelement, El}, StateData) ->
             end;
         {verify, To, From, Id, Type} ->
             ?LOG_DEBUG(#{what => s2s_receive_verify,
-                         from => From, to => To, messag_id => Id, type => Type}),
+                         from => From, to => To, message_id => Id, type => Type}),
             case StateData#state.verify of
                 false ->
                     NextState = wait_for_validation,
@@ -613,7 +613,7 @@ stream_established({xmlstreamelement, El}, StateData) ->
     case is_verify_res(El) of
         {verify, VTo, VFrom, VId, VType} ->
             ?LOG_DEBUG(#{what => s2s_recv_verify,
-                         to => VTo, from => VFrom, messag_id => VId, type => VType,
+                         to => VTo, from => VFrom, message_id => VId, type => VType,
                          myname => StateData#state.myname, server => StateData#state.server}),
             case StateData#state.verify of
                 {VPid, _VKey, _SID} ->