Merge pull request #7 from tgrk/fix-ingest-url

Fix ingest API

.circleci/config.yml

@@ -26,15 +26,15 @@ jobs:
           name: Display webapp coverage report
           command: rebar3 cover -v
       - run:
-          name: Convert coverage report to codecov supported format
-          command:  bin/covertool -cover _build/test/cover/eunit.coverdata -appname lager_humio_backend -output cobertura.xml
+          name: Generate coverage report to codecov supported format
+          command:  rebar3 covertool generate -p2
       - run:
           name: Upload coverage report to Codecov
           command: |
             python3 -m venv venv
             . venv/bin/activate
             pip3 install --upgrade codecov
-            codecov
+            codecov -f _build/test/covertool/lager_humio_backend.covertool.xml
       - save_cache:
           key: build-cache-{{ checksum "rebar.lock" }}
           paths:

README.md

@@ -34,10 +34,9 @@ First you have to sign up with [Humio][1] service to get all required informatio
 | ------------------ |:--------:| ---------------------------------------------- |
 | `host`             | Yes      | Hostname of the Humio server (e.g. go.humio.com) |
 | `token`            | Yes      | Humio Ingestion API token (from [Settings][2]) |
-| `dataspace`        | Yes      | Humio dataspace (from [Settings][2])           |
 | `source`           | Yes      | Humio log source for log grouping and filtering  |
 | `level`            | Yes      | Minimal log level to use (defaults to `debug`) |
-| `formatter`        | No       | The module to use when formatting log messages (defaults to `lager_default_formatter') |
+| `formatter`        | No       | The module to use when formatting log messages (defaults to `lager_default_formatter`)  |
 | `formatter_config` | No       | The format configuration string (defaults to `time [ severity ] message`) |
 | `metadata_filter`  | No       | A list of excluded metadata keys |
 | `retry_interval`   | No       | Intervarl for retry in case endpoint is not available (defaults to 60 seconds) |
@@ -49,11 +48,10 @@ Sample configuration:
 {lager, [
   {handlers, [
     {lager_console_backend, debug},
-    {lager_humio_backend,   [{host, "go.humio.com"},
-                             {token, "YOUR_INGESTION_API_TOKEN"},
-                             {dataspace, "YOUR_DATASPACE"},
-                             {source,    "YOUR_APPLICATION"},
-                             {level, info}
+    {lager_humio_backend,   [{host,  "cloud.humio.com"},
+                             {token,  "YOUR_INGESTION_API_TOKEN"},
+                             {source, "YOUR_APPLICATION"},
+                             {level,  info}
     ]}
   ]}
 }

rebar.config

@@ -17,11 +17,12 @@
 
 {minimum_otp_vsn, "19"}.
 
-{deps, [ {lager,            "3.6.1"}
-       , {jiffy,            "0.15.1"}
-       , {iso8601,          "1.2.3"}
-       , {meck,             "0.8.9"}
+{deps, [ {lager,            "3.6.4"}
+       , {jiffy,            "0.15.2"}
+       , {iso8601,          "1.3.1"}
+       , {meck,             "0.8.11"}
        , {eunit_formatters, "0.5.0"}
+			 , {covertool,        "2.0.0"}
 ]}.
 
 {overrides,
@@ -36,12 +37,13 @@
           }]
       }
   ]}
+
+
 ]}.
 
-{cover_enabled, true}.
-{cover_print_enabled, true}.
-{eunit_opts, [verbose,
-              no_tty,
-              {report, {eunit_progress, [colored]}}]}.
+{plugins, [covertool]}.
+{cover_export_enabled, true}.
+{covertool_eunit, {".eunit/eunit.coverdata", "eunit.coverage.xml"}}.
+{covertool_prefix_len, 2}.
 
 {xref_checks, [undefined_function_calls]}.

rebar.config.script

@@ -6,13 +6,13 @@ case erlang:function_exported(rebar3, main, 1) of
         %% rebar 2.x or older
         NewConf = lists:keystore(deps_dir, 1, CONFIG, {deps_dir, "deps"}),
         Deps = [{lager, ".*",
-                 {git, "https://github.com/erlang-lager/lager.git", {tag, "3.6.1"}}},
+                 {git, "https://github.com/erlang-lager/lager.git", {tag, "3.6.4"}}},
                 {jiffy, "0.*",
-                 {git, "https://github.com/davisp/jiffy.git", {tag, "0.15.1"}}},
+                 {git, "https://github.com/davisp/jiffy.git", {tag, "1.0.1"}}},
                 {iso8601, ".*",
                   {git, "https://github.com/erlsci/iso8601.git", {tag, "1.2.3"}}},
                 {meck, ".*",
-                 {git, "https://github.com/eproxus/meck.git", {tag, "0.8.9"}}},
+                 {git, "https://github.com/eproxus/meck.git", {tag, "0.8.11"}}},
                 {eunit_formatters, ".*",
                  {git, "https://github.com/seancribbs/eunit_formatters.git", {tag, "v0.5.0"}}}
                ],

rebar.lock

@@ -1,16 +1,18 @@
 {"1.1.0",
-[{<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
+[{<<"covertool">>,{pkg,<<"covertool">>,<<"2.0.0">>},0},
+ {<<"eunit_formatters">>,{pkg,<<"eunit_formatters">>,<<"0.5.0">>},0},
  {<<"goldrush">>,{pkg,<<"goldrush">>,<<"0.1.9">>},1},
- {<<"iso8601">>,{pkg,<<"iso8601">>,<<"1.2.3">>},0},
- {<<"jiffy">>,{pkg,<<"jiffy">>,<<"0.15.1">>},0},
- {<<"lager">>,{pkg,<<"lager">>,<<"3.6.1">>},0},
- {<<"meck">>,{pkg,<<"meck">>,<<"0.8.9">>},0}]}.
+ {<<"iso8601">>,{pkg,<<"iso8601">>,<<"1.3.1">>},0},
+ {<<"jiffy">>,{pkg,<<"jiffy">>,<<"0.15.2">>},0},
+ {<<"lager">>,{pkg,<<"lager">>,<<"3.6.4">>},0},
+ {<<"meck">>,{pkg,<<"meck">>,<<"0.8.11">>},0}]}.
 [
 {pkg_hash,[
+ {<<"covertool">>, <<"2E0172F35D95AA573F113970E077FBA644DD809F450C27E88C173CA3A66FDB42">>},
  {<<"eunit_formatters">>, <<"6A9133943D36A465D804C1C5B6E6839030434B8879C5600D7DDB5B3BAD4CCB59">>},
  {<<"goldrush">>, <<"F06E5D5F1277DA5C413E84D5A2924174182FB108DABB39D5EC548B27424CD106">>},
- {<<"iso8601">>, <<"56D173EC568D42E95908797A9B7D27BCF0B377443277416733EB1A544A4A47C2">>},
- {<<"jiffy">>, <<"BE83B09388DA1A6C7E798207C9D6A1C4D71BB95FCC387D37D35861788F49AB97">>},
- {<<"lager">>, <<"9D29C5FF7F926D25ECD9899990867C9152DCF34EEE65BAC8EC0DFC0D16A26E0C">>},
- {<<"meck">>, <<"64C5C0BD8BCCA3A180B44196265C8ED7594E16BCC845D0698EC6B4E577F48188">>}]}
+ {<<"iso8601">>, <<"D1CEE73F56D71C35590C6B2DB2074873BF410BABAAB768F6EA566366D8CA4810">>},
+ {<<"jiffy">>, <<"DE266C390111FD4EA28B9302F0BC3D7472468F3B8E0ACEABFBEFA26D08CD73B7">>},
+ {<<"lager">>, <<"CED6E98070FB4E58EE93174D006D46479C79844DF7FC17FA4FEFC1049A320D88">>},
+ {<<"meck">>, <<"2C39E15EC87D847DA6CF69B4A1C4AF3FD850AE2A272E719E0E8751A7FE54771F">>}]}
 ].

src/lager_humio_backend.app.src

@@ -1,6 +1,6 @@
 {application, lager_humio_backend,
  [{description, "Lager logging backend for Humio.com"},
-  {vsn, "1.2.4"},
+  {vsn, "1.3.5"},
   {registered, []},
   {applications,
    [kernel,

src/lager_humio_backend.erl

@@ -6,7 +6,6 @@
 %%% <ul>
 %%%    <li>`host' - Hostname of the Humio server (e.g. go.humio.com)</li>
 %%%    <li>`token' - Humio Ingestion API token (from Settings)</li>
-%%%    <li>`dataspace' - Humio dataspace (from Settings)</li>
 %%%    <li>`source' - Humio log source for log grouping and filtering</li>
 %%%    <li>`level' - log level to use</li>
 %%%    <li>`formatter' - the module to use when formatting log messages.
@@ -39,7 +38,6 @@
 
 -record(state, { host            :: string()
                , token           :: string()
-               , dataspace       :: string()
                , source          :: string()
                , level           :: integer()
                , formatter       :: atom()
@@ -55,7 +53,7 @@
 %% Exported for testing
 -ifdef(TEST).
 -export([ call_ingest_api/4
-        , create_httpc_request/4
+        , create_httpc_request/3
         , get_configuration/1
         , validate_options/1
         , create_event/4
@@ -91,8 +89,7 @@ handle_event({log, Message}, #state{level = MinLevel} = State) ->
     case lager_util:is_loggable(Message, MinLevel, ?MODULE) of
         true ->
             Payload = jiffy:encode(create_payload(Message, State)),
-            Request = create_httpc_request(Payload, State#state.host,
-                                           State#state.token, State#state.dataspace),
+            Request = create_httpc_request(Payload, State#state.host, State#state.token),
             RetryInterval = State#state.retry_interval,
             MaxRetries = State#state.max_retries,
             Opts = State#state.httpc_opts,
@@ -161,16 +158,20 @@ call_ingest_api(Request, Retries, Interval, Opts) ->
     case httpc:request(post, Request, Opts, []) of
         {ok, {{_, 200, _}, _H, _B}} ->
             ok;
+        {ok, {{_, 401, _}, _H, Response}} ->
+						error_logger:error_msg("HUMIO: ~s!~n", [Response]),
+            timer:sleep(Interval),
+            call_ingest_api(Request, Retries - 1, Interval, Opts);
         _Other ->
             timer:sleep(Interval),
             call_ingest_api(Request, Retries - 1, Interval, Opts)
     end.
 
-create_httpc_request(Payload, Host, Token, DS) ->
-    {get_uri(Host, DS), get_headers(Token), "application/json", Payload}.
+create_httpc_request(Payload, Host, Token) ->
+    {get_uri(Host), get_headers(Token), "application/json", Payload}.
 
-get_uri(Host, DS) ->
-    "https://" ++ Host ++ "/api/v1/dataspaces/" ++ DS ++ "/ingest".
+get_uri(Host) ->
+		"https://" ++ Host ++ "/api/v1/ingest/humio-structured".
 
 get_headers(Token) ->
     [{"Authorization", "Bearer " ++ Token}].
@@ -222,9 +223,8 @@ validate_options([H | _]) ->
     {error, {bad_config, H}}.
 
 get_configuration(Options) ->
-    #state{ host            = get_option(host, Options, "go.humio.com")
+    #state{ host            = get_option(host, Options, "cloud.humio.com")
           , token           = get_option(token, Options, "")
-          , dataspace       = get_option(dataspace, Options, "")
           , source          = get_option(source, Options, "unknown")
           , level           = lager_util:level_to_num(
                                 get_option(level, Options, debug))

test/lager_humio_backend_tests.erl

@@ -34,7 +34,6 @@ test_integration() ->
     HumioConfig = {lager_humio_backend,
                    [{host, "testhost"},
                     {token, "foo"},
-                    {dataspace, "bar"},
                     {source, "foobar"},
                     {level, info}]
                   },
@@ -66,7 +65,7 @@ test_integration() ->
     ok.
 
 assert_request({Url, Headers, ContentType, Payload}) ->
-    ?assertEqual("https://testhost/api/v1/dataspaces/bar/ingest", Url),
+    ?assertEqual("https://testhost/api/v1/ingest/humio-structured", Url),
     ?assertEqual([{"Authorization","Bearer foo"}], Headers),
     ?assertEqual("application/json", ContentType),
 
@@ -90,7 +89,7 @@ assert_request({Url, Headers, ContentType, Payload}) ->
     ok.
 
 test_call_ingest_api_retry() ->
-    Request = lager_humio_backend:create_httpc_request(<<"{}">>, "foo", "bar", "baz"),
+    Request = lager_humio_backend:create_httpc_request(<<"{}">>, "foo", "bar"),
 
     ok = meck:expect(
            httpc, request,
@@ -114,7 +113,6 @@ test_call_ingest_api_retry() ->
 test_get_configuration() ->
     Options = [ {host, ""}
               , {token, ""}
-              , {dataspace, ""}
               , {source, ""}
               , {level, debug}
               , {formatter, lager_default_formatter}
@@ -126,14 +124,13 @@ test_get_configuration() ->
               ],
 
     ?assertEqual(
-       {state, [], [], [], [], 128, lager_default_formatter, [], [], 60*1000, 10, []},
+       {state, [], [], [], 128, lager_default_formatter, [], [], 60*1000, 10, []},
        lager_humio_backend:get_configuration(Options)
       ).
 
 test_validate_options() ->
     ValidOptions = [ {host, "test.com"}
                    , {token, "foo"}
-                   , {dataspace, "bar"}
                    , {level, debug}
                    , {formatter, lager_default_formatter}
                    , {format_config, []}
@@ -148,7 +145,6 @@ test_validate_options() ->
 
     Invalid = [ {{host, ""},            {error, missing_host}}
               , {{token, ""},           {error, missing_token}}
-              , {{dataspace, ""},       {error, missing_dataspace}}
               , {{source, ""},          {error, missing_source}}
               , {{level, unknown},      {error, {bad_level, unknown}}}
               , {{retry_interval, foo}, {error, {bad_config, {retry_interval, foo}}}}