From 6daaf0e48147ea90a1fdfce0a3824a5fc6cdfc08 Mon Sep 17 00:00:00 2001 From: Jon Carstens Date: Sat, 10 Feb 2024 11:06:40 -0700 Subject: [PATCH] Revert to Cowboy as the webserver While using Bandit, it was discovered that a herd of _disconnects_ could cause catastrophic failures leading to node crashes (in our testing, these were decently sized 4-core, 8GB instances with 5-6k devices connected). When testing again with Cowboy, the problem was resolved. This change reverts to use cowboy to allow progression of other fixes and features while Bandit/Thousand Island is investigated to find the root cause --- config/config.exs | 2 - config/dev.exs | 37 ++++++------ config/runtime.exs | 41 ++++++------- config/test.exs | 18 +++--- lib/nerves_hub/device_ssl_transport.ex | 79 -------------------------- lib/nerves_hub/ssl.ex | 68 ++++++++++++---------- mix.exs | 3 +- mix.lock | 8 ++- 8 files changed, 87 insertions(+), 169 deletions(-) delete mode 100644 lib/nerves_hub/device_ssl_transport.ex diff --git a/config/config.exs b/config/config.exs index 089205188..a5fb58a4d 100644 --- a/config/config.exs +++ b/config/config.exs @@ -24,7 +24,6 @@ config :nerves_hub, # NervesHub Device # config :nerves_hub, NervesHubWeb.DeviceEndpoint, - adapter: Bandit.PhoenixAdapter, render_errors: [view: NervesHubWeb.ErrorView, accepts: ~w(html json)], pubsub_server: NervesHub.PubSub @@ -32,7 +31,6 @@ config :nerves_hub, NervesHubWeb.DeviceEndpoint, # NervesHub Web # config :nerves_hub, NervesHubWeb.Endpoint, - adapter: Bandit.PhoenixAdapter, secret_key_base: "ZH9GG2S5CwIMWXBg92wUuoyKFrjgqaAybHLTLuUk1xZO0HeidcJbnMBSTHDcyhSn", live_view: [ signing_salt: "Kct3W8U7uQ6KAczYjzNbiYS6A8Pbtk3f" diff --git a/config/dev.exs b/config/dev.exs index e447ed784..c357425b0 100644 --- a/config/dev.exs +++ b/config/dev.exs @@ -43,27 +43,22 @@ config :nerves_hub, NervesHubWeb.DeviceEndpoint, ip: {0, 0, 0, 0}, port: 4001, otp_app: :nerves_hub, - thousand_island_options: [ - transport_module: NervesHub.DeviceSSLTransport, - transport_options: [ - # Enable client SSL - # Older versions of OTP 25 may break using using devices - # that support TLS 1.3 or 1.2 negotiation. To mitigate that - # potential error, we enforce TLS 1.2. If you're using OTP >= 25.1 - # on all devices, then it is safe to allow TLS 1.3 by removing - # the versions constraint and setting `certificate_authorities: false` - # See https://github.com/erlang/otp/issues/6492#issuecomment-1323874205 - # - # certificate_authorities: false, - versions: [:"tlsv1.2"], - verify: :verify_peer, - verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, - fail_if_no_peer_cert: true, - keyfile: Path.join(ssl_dir, "device.nerves-hub.org-key.pem"), - certfile: Path.join(ssl_dir, "device.nerves-hub.org.pem"), - cacertfile: Path.join(ssl_dir, "ca.pem") - ] - ] + # Enable client SSL + # Older versions of OTP 25 may break using using devices + # that support TLS 1.3 or 1.2 negotiation. To mitigate that + # potential error, we enforce TLS 1.2. If you're using OTP >= 25.1 + # on all devices, then it is safe to allow TLS 1.3 by removing + # the versions constraint and setting `certificate_authorities: false` + # See https://github.com/erlang/otp/issues/6492#issuecomment-1323874205 + # + # certificate_authorities: false, + versions: [:"tlsv1.2"], + verify: :verify_peer, + verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, + fail_if_no_peer_cert: true, + keyfile: Path.join(ssl_dir, "device.nerves-hub.org-key.pem"), + certfile: Path.join(ssl_dir, "device.nerves-hub.org.pem"), + cacertfile: Path.join(ssl_dir, "ca.pem") ] ## diff --git a/config/runtime.exs b/config/runtime.exs index bd75735bb..d44a525f2 100644 --- a/config/runtime.exs +++ b/config/runtime.exs @@ -105,29 +105,24 @@ if config_env() == :prod do https: [ port: https_port, otp_app: :nerves_hub, - thousand_island_options: [ - transport_module: NervesHub.DeviceSSLTransport, - transport_options: [ - # Enable client SSL - # Older versions of OTP 25 may break using using devices - # that support TLS 1.3 or 1.2 negotiation. To mitigate that - # potential error, we enforce TLS 1.2. If you're using OTP >= 25.1 - # on all devices, then it is safe to allow TLS 1.3 by removing - # the versions constraint and setting `certificate_authorities: false` - # since we don't expect devices to send full chains to the server - # See https://github.com/erlang/otp/issues/6492#issuecomment-1323874205 - # - # certificate_authorities: false, - versions: [:"tlsv1.2"], - verify: :verify_peer, - verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, - fail_if_no_peer_cert: true, - keyfile: keyfile, - certfile: certfile, - cacertfile: CAStore.file_path(), - hibernate_after: 15_000 - ] - ] + # Enable client SSL + # Older versions of OTP 25 may break using using devices + # that support TLS 1.3 or 1.2 negotiation. To mitigate that + # potential error, we enforce TLS 1.2. If you're using OTP >= 25.1 + # on all devices, then it is safe to allow TLS 1.3 by removing + # the versions constraint and setting `certificate_authorities: false` + # since we don't expect devices to send full chains to the server + # See https://github.com/erlang/otp/issues/6492#issuecomment-1323874205 + # + # certificate_authorities: false, + versions: [:"tlsv1.2"], + verify: :verify_peer, + verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, + fail_if_no_peer_cert: true, + keyfile: keyfile, + certfile: certfile, + cacertfile: CAStore.file_path(), + hibernate_after: 15_000 ] end end diff --git a/config/test.exs b/config/test.exs index bcf495d6f..759dfdbd6 100644 --- a/config/test.exs +++ b/config/test.exs @@ -25,17 +25,13 @@ config :nerves_hub, NervesHubWeb.DeviceEndpoint, https: [ port: 4101, otp_app: :nerves_hub, - thousand_island_options: [ - transport_options: [ - # Enable client SSL - verify: :verify_peer, - verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, - fail_if_no_peer_cert: true, - keyfile: Path.join([__DIR__, "../test/fixtures/ssl/device.nerves-hub.org-key.pem"]), - certfile: Path.join([__DIR__, "../test/fixtures/ssl/device.nerves-hub.org.pem"]), - cacertfile: Path.join([__DIR__, "../test/fixtures/ssl/ca.pem"]) - ] - ] + # Enable client SSL + verify: :verify_peer, + verify_fun: {&NervesHub.SSL.verify_fun/3, nil}, + fail_if_no_peer_cert: true, + keyfile: Path.join([__DIR__, "../test/fixtures/ssl/device.nerves-hub.org-key.pem"]), + certfile: Path.join([__DIR__, "../test/fixtures/ssl/device.nerves-hub.org.pem"]), + cacertfile: Path.join([__DIR__, "../test/fixtures/ssl/ca.pem"]) ] ## diff --git a/lib/nerves_hub/device_ssl_transport.ex b/lib/nerves_hub/device_ssl_transport.ex deleted file mode 100644 index 0d6414ca2..000000000 --- a/lib/nerves_hub/device_ssl_transport.ex +++ /dev/null @@ -1,79 +0,0 @@ -defmodule NervesHub.DeviceSSLTransport do - @moduledoc """ - SSL transport for device certificate authentication - - This transport exists to rate limit incoming SSL connections _before_ any - ssl work has started. This let's us shed incoming devices before we waste - a lot of resources on denying them midway through the SSL connection in - the `NervesHub.SSL.verify_fun/3` - - See `handshake/1` for the main change. All other function are delegated back to - `ThousandIsland.Transports.SSL` - """ - - @behaviour ThousandIsland.Transport - - @impl ThousandIsland.Transport - defdelegate listen(port, user_options), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate accept(listener_socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - def handshake(socket) do - if NervesHub.RateLimit.increment() do - :telemetry.execute([:nerves_hub, :rate_limit, :accepted], %{count: 1}) - - ThousandIsland.Transports.SSL.handshake(socket) - else - :telemetry.execute([:nerves_hub, :rate_limit, :rejected], %{count: 1}) - - {:error, :closed} - end - end - - @impl ThousandIsland.Transport - defdelegate upgrade(socket, opts), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate controlling_process(socket, pid), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate recv(socket, length, timeout), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate send(socket, data), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate sendfile(socket, filename, offset, length), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate getopts(socket, options), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate setopts(socket, options), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate shutdown(socket, way), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate close(socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate sockname(socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate peername(socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate peercert(socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate secure?(), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate getstat(socket), to: ThousandIsland.Transports.SSL - - @impl ThousandIsland.Transport - defdelegate negotiated_protocol(socket), to: ThousandIsland.Transports.SSL -end diff --git a/lib/nerves_hub/ssl.ex b/lib/nerves_hub/ssl.ex index 130f918ef..0264f2d4b 100644 --- a/lib/nerves_hub/ssl.ex +++ b/lib/nerves_hub/ssl.ex @@ -1,6 +1,7 @@ defmodule NervesHub.SSL do alias NervesHub.Devices alias NervesHub.Certificate + alias NervesHub.RateLimit @type pkix_path_validation_reason :: :cert_expired @@ -33,9 +34,13 @@ defmodule NervesHub.SSL do # or the signer cert was included by the client and is valid # for the peer (device) cert def verify_fun(otp_cert, :valid_peer, state) do - :telemetry.execute([:nerves_hub, :rate_limit, :accepted], %{count: 1}) - - do_verify(otp_cert, state) + if RateLimit.increment() do + :telemetry.execute([:nerves_hub, :rate_limit, :accepted], %{count: 1}) + do_verify(otp_cert, state) + else + :telemetry.execute([:nerves_hub, :rate_limit, :rejected], %{count: 1}) + {:fail, :rate_limit} + end end def verify_fun(_certificate, :valid, state) do @@ -43,32 +48,37 @@ defmodule NervesHub.SSL do end def verify_fun(otp_cert, {:bad_cert, err}, state) when err in [:unknown_ca, :cert_expired] do - :telemetry.execute([:nerves_hub, :rate_limit, :accepted], %{count: 1}) - - aki = Certificate.get_aki(otp_cert) - ski = Certificate.get_ski(otp_cert) - - cond do - aki == ski -> - # If the signer CA is also the root, then AKI == SKI. We can skip - # checking as it will be validated later on if the device needs - # registration - {:valid, state} - - is_binary(ski) and match?({:ok, _db_ca}, Devices.get_ca_certificate_by_ski(ski)) -> - # Signer CA sent with the device certificate, but is an intermediary - # so the chain is incomplete labeling it as unknown_ca. - # - # Since we have this CA registered, validate so we can move on to the device - # cert next and expiration will be checked later if registration of a new - # device cert needs to happen. - {:valid, state} - - true -> - # The signer CA was not included in the request, so this is most - # likely a device cert that needs verification. If it isn't, then - # this is some other unknown CA that will fail - do_verify(otp_cert, state) + if RateLimit.increment() do + :telemetry.execute([:nerves_hub, :rate_limit, :accepted], %{count: 1}) + + aki = Certificate.get_aki(otp_cert) + ski = Certificate.get_ski(otp_cert) + + cond do + aki == ski -> + # If the signer CA is also the root, then AKI == SKI. We can skip + # checking as it will be validated later on if the device needs + # registration + {:valid, state} + + is_binary(ski) and match?({:ok, _db_ca}, Devices.get_ca_certificate_by_ski(ski)) -> + # Signer CA sent with the device certificate, but is an intermediary + # so the chain is incomplete labeling it as unknown_ca. + # + # Since we have this CA registered, validate so we can move on to the device + # cert next and expiration will be checked later if registration of a new + # device cert needs to happen. + {:valid, state} + + true -> + # The signer CA was not included in the request, so this is most + # likely a device cert that needs verification. If it isn't, then + # this is some other unknown CA that will fail + do_verify(otp_cert, state) + end + else + :telemetry.execute([:nerves_hub, :rate_limit, :rejected], %{count: 1}) + {:fail, :rate_limit} end end diff --git a/mix.exs b/mix.exs index b902b1fd7..34c516687 100644 --- a/mix.exs +++ b/mix.exs @@ -51,12 +51,12 @@ defmodule NervesHub.MixProject do [ {:mix_test_watch, "~> 1.0", only: :test, runtime: false}, {:recon, "~> 2.5"}, - {:bandit, "~> 1.0"}, {:base62, "~> 1.2"}, {:bcrypt_elixir, "~> 3.0"}, {:castore, "~> 1.0"}, {:circular_buffer, "~> 0.4.1"}, {:comeonin, "~> 5.3"}, + {:cowboy, "~> 2.0"}, {:crontab, "~> 1.1"}, {:decorator, "~> 1.2"}, {:ecto, "~> 3.8", override: true}, @@ -85,6 +85,7 @@ defmodule NervesHub.MixProject do {:phoenix_swoosh, "~> 1.0"}, {:phoenix_view, "~> 2.0"}, {:plug, "~> 1.7"}, + {:plug_cowboy, "~> 2.1"}, {:postgrex, "~> 0.14"}, {:scrivener_ecto, "~> 2.7"}, {:scrivener_html, git: "https://github.com/nerves-hub/scrivener_html", branch: "phx-1.5"}, diff --git a/mix.lock b/mix.lock index a11acc07b..d759abc8c 100644 --- a/mix.lock +++ b/mix.lock @@ -1,5 +1,4 @@ %{ - "bandit": {:hex, :bandit, "1.1.3", "0c504f50029381f41203788851df8e43554d79b0a073e993b424b5897ee2fb8d", [:mix], [{:hpax, "~> 0.1.1", [hex: :hpax, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:thousand_island, "~> 1.0", [hex: :thousand_island, repo: "hexpm", optional: false]}, {:websock, "~> 0.5", [hex: :websock, repo: "hexpm", optional: false]}], "hexpm", "5953cd4e924c85d61a3afbac298bfa76a1b3b9eae2cee192e23f2e5aaa4d5b73"}, "base62": {:hex, :base62, "1.2.2", "85c6627eb609317b70f555294045895ffaaeb1758666ab9ef9ca38865b11e629", [:mix], [{:custom_base, "~> 0.2.1", [hex: :custom_base, repo: "hexpm", optional: false]}], "hexpm", "d41336bda8eaa5be197f1e4592400513ee60518e5b9f4dcf38f4b4dae6f377bb"}, "bcrypt_elixir": {:hex, :bcrypt_elixir, "3.1.0", "0b110a9a6c619b19a7f73fa3004aa11d6e719a67e672d1633dc36b6b2290a0f7", [:make, :mix], [{:comeonin, "~> 5.3", [hex: :comeonin, repo: "hexpm", optional: false]}, {:elixir_make, "~> 0.6", [hex: :elixir_make, repo: "hexpm", optional: false]}], "hexpm", "2ad2acb5a8bc049e8d5aa267802631912bb80d5f4110a178ae7999e69dca1bf7"}, "castore": {:hex, :castore, "1.0.5", "9eeebb394cc9a0f3ae56b813459f990abb0a3dedee1be6b27fdb50301930502f", [:mix], [], "hexpm", "8d7c597c3e4a64c395980882d4bca3cebb8d74197c590dc272cfd3b6a6310578"}, @@ -7,6 +6,9 @@ "circular_buffer": {:hex, :circular_buffer, "0.4.1", "477f370fd8cfe1787b0a1bade6208bbd274b34f1610e41f1180ba756a7679839", [:mix], [], "hexpm", "633ef2e059dde0d7b89bbab13b1da9d04c6685e80e68fbdf41282d4fae746b72"}, "combine": {:hex, :combine, "0.10.0", "eff8224eeb56498a2af13011d142c5e7997a80c8f5b97c499f84c841032e429f", [:mix], [], "hexpm", "1b1dbc1790073076580d0d1d64e42eae2366583e7aecd455d1215b0d16f2451b"}, "comeonin": {:hex, :comeonin, "5.4.0", "246a56ca3f41d404380fc6465650ddaa532c7f98be4bda1b4656b3a37cc13abe", [:mix], [], "hexpm", "796393a9e50d01999d56b7b8420ab0481a7538d0caf80919da493b4a6e51faf1"}, + "cowboy": {:hex, :cowboy, "2.10.0", "ff9ffeff91dae4ae270dd975642997afe2a1179d94b1887863e43f681a203e26", [:make, :rebar3], [{:cowlib, "2.12.1", [hex: :cowlib, repo: "hexpm", optional: false]}, {:ranch, "1.8.0", [hex: :ranch, repo: "hexpm", optional: false]}], "hexpm", "3afdccb7183cc6f143cb14d3cf51fa00e53db9ec80cdcd525482f5e99bc41d6b"}, + "cowboy_telemetry": {:hex, :cowboy_telemetry, "0.4.0", "f239f68b588efa7707abce16a84d0d2acf3a0f50571f8bb7f56a15865aae820c", [:rebar3], [{:cowboy, "~> 2.7", [hex: :cowboy, repo: "hexpm", optional: false]}, {:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "7d98bac1ee4565d31b62d59f8823dfd8356a169e7fcbb83831b8a5397404c9de"}, + "cowlib": {:hex, :cowlib, "2.12.1", "a9fa9a625f1d2025fe6b462cb865881329b5caff8f1854d1cbc9f9533f00e1e1", [:make, :rebar3], [], "hexpm", "163b73f6367a7341b33c794c4e88e7dbfe6498ac42dcd69ef44c5bc5507c8db0"}, "crontab": {:hex, :crontab, "1.1.13", "3bad04f050b9f7f1c237809e42223999c150656a6b2afbbfef597d56df2144c5", [:mix], [{:ecto, "~> 1.0 or ~> 2.0 or ~> 3.0", [hex: :ecto, repo: "hexpm", optional: true]}], "hexpm", "d67441bec989640e3afb94e123f45a2bc42d76e02988c9613885dc3d01cf7085"}, "custom_base": {:hex, :custom_base, "0.2.1", "4a832a42ea0552299d81652aa0b1f775d462175293e99dfbe4d7dbaab785a706", [:mix], [], "hexpm", "8df019facc5ec9603e94f7270f1ac73ddf339f56ade76a721eaa57c1493ba463"}, "db_connection": {:hex, :db_connection, "2.6.0", "77d835c472b5b67fc4f29556dee74bf511bbafecdcaf98c27d27fa5918152086", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "c2f992d15725e721ec7fbc1189d4ecdb8afef76648c746a8e1cad35e3b8a35f3"}, @@ -56,9 +58,10 @@ "phoenix_template": {:hex, :phoenix_template, "1.0.4", "e2092c132f3b5e5b2d49c96695342eb36d0ed514c5b252a77048d5969330d639", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}], "hexpm", "2c0c81f0e5c6753faf5cca2f229c9709919aba34fab866d3bc05060c9c444206"}, "phoenix_view": {:hex, :phoenix_view, "2.0.3", "4d32c4817fce933693741deeb99ef1392619f942633dde834a5163124813aad3", [:mix], [{:phoenix_html, "~> 2.14.2 or ~> 3.0 or ~> 4.0", [hex: :phoenix_html, repo: "hexpm", optional: true]}, {:phoenix_template, "~> 1.0", [hex: :phoenix_template, repo: "hexpm", optional: false]}], "hexpm", "cd34049af41be2c627df99cd4eaa71fc52a328c0c3d8e7d4aa28f880c30e7f64"}, "plug": {:hex, :plug, "1.15.3", "712976f504418f6dff0a3e554c40d705a9bcf89a7ccef92fc6a5ef8f16a30a97", [:mix], [{:mime, "~> 1.0 or ~> 2.0", [hex: :mime, repo: "hexpm", optional: false]}, {:plug_crypto, "~> 1.1.1 or ~> 1.2 or ~> 2.0", [hex: :plug_crypto, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4.3 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "cc4365a3c010a56af402e0809208873d113e9c38c401cabd88027ef4f5c01fd2"}, + "plug_cowboy": {:hex, :plug_cowboy, "2.7.0", "3ae9369c60641084363b08fe90267cbdd316df57e3557ea522114b30b63256ea", [:mix], [{:cowboy, "~> 2.7.0 or ~> 2.8.0 or ~> 2.9.0 or ~> 2.10.0", [hex: :cowboy, repo: "hexpm", optional: false]}, {:cowboy_telemetry, "~> 0.3", [hex: :cowboy_telemetry, repo: "hexpm", optional: false]}, {:plug, "~> 1.14", [hex: :plug, repo: "hexpm", optional: false]}], "hexpm", "d85444fb8aa1f2fc62eabe83bbe387d81510d773886774ebdcb429b3da3c1a4a"}, "plug_crypto": {:hex, :plug_crypto, "2.0.0", "77515cc10af06645abbfb5e6ad7a3e9714f805ae118fa1a70205f80d2d70fe73", [:mix], [], "hexpm", "53695bae57cc4e54566d993eb01074e4d894b65a3766f1c43e2c61a1b0f45ea9"}, "postgrex": {:hex, :postgrex, "0.17.4", "5777781f80f53b7c431a001c8dad83ee167bcebcf3a793e3906efff680ab62b3", [:mix], [{:db_connection, "~> 2.1", [hex: :db_connection, repo: "hexpm", optional: false]}, {:decimal, "~> 1.5 or ~> 2.0", [hex: :decimal, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: true]}, {:table, "~> 0.1.0", [hex: :table, repo: "hexpm", optional: true]}], "hexpm", "6458f7d5b70652bc81c3ea759f91736c16a31be000f306d3c64bcdfe9a18b3cc"}, - "ranch": {:hex, :ranch, "2.1.0", "2261f9ed9574dcfcc444106b9f6da155e6e540b2f82ba3d42b339b93673b72a3", [:make, :rebar3], [], "hexpm", "244ee3fa2a6175270d8e1fc59024fd9dbc76294a321057de8f803b1479e76916"}, + "ranch": {:hex, :ranch, "1.8.0", "8c7a100a139fd57f17327b6413e4167ac559fbc04ca7448e9be9057311597a1d", [:make, :rebar3], [], "hexpm", "49fbcfd3682fab1f5d109351b61257676da1a2fdbe295904176d5e521a2ddfe5"}, "recon": {:hex, :recon, "2.5.4", "05dd52a119ee4059fa9daa1ab7ce81bc7a8161a2f12e9d42e9d551ffd2ba901c", [:mix, :rebar3], [], "hexpm", "e9ab01ac7fc8572e41eb59385efeb3fb0ff5bf02103816535bacaedf327d0263"}, "scrivener": {:hex, :scrivener, "2.7.2", "1d913c965ec352650a7f864ad7fd8d80462f76a32f33d57d1e48bc5e9d40aba2", [:mix], [], "hexpm", "7866a0ec4d40274efbee1db8bead13a995ea4926ecd8203345af8f90d2b620d9"}, "scrivener_ecto": {:hex, :scrivener_ecto, "2.7.0", "cf64b8cb8a96cd131cdbcecf64e7fd395e21aaa1cb0236c42a7c2e34b0dca580", [:mix], [{:ecto, "~> 3.3", [hex: :ecto, repo: "hexpm", optional: false]}, {:scrivener, "~> 2.4", [hex: :scrivener, repo: "hexpm", optional: false]}], "hexpm", "e809f171687806b0031129034352f5ae44849720c48dd839200adeaf0ac3e260"}, @@ -72,7 +75,6 @@ "telemetry_metrics": {:hex, :telemetry_metrics, "0.6.2", "2caabe9344ec17eafe5403304771c3539f3b6e2f7fb6a6f602558c825d0d0bfb", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "9b43db0dc33863930b9ef9d27137e78974756f5f198cae18409970ed6fa5b561"}, "telemetry_metrics_statsd": {:hex, :telemetry_metrics_statsd, "0.7.0", "92732fae63db31ef2508df6faee7d81401883e33f2976715a82f296a33a45cee", [:mix], [{:nimble_options, "~> 0.4 or ~> 1.0", [hex: :nimble_options, repo: "hexpm", optional: false]}, {:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}, {:telemetry_metrics, "~> 0.6", [hex: :telemetry_metrics, repo: "hexpm", optional: false]}], "hexpm", "797e34a856376dfd4e96347da0f747fcff4e0cadf6e6f0f989598f563cad05ff"}, "telemetry_poller": {:hex, :telemetry_poller, "1.0.0", "db91bb424e07f2bb6e73926fcafbfcbcb295f0193e0a00e825e589a0a47e8453", [:rebar3], [{:telemetry, "~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "b3a24eafd66c3f42da30fc3ca7dda1e9d546c12250a2d60d7b81d264fbec4f6e"}, - "thousand_island": {:hex, :thousand_island, "1.3.2", "bc27f9afba6e1a676dd36507d42e429935a142cf5ee69b8e3f90bff1383943cd", [:mix], [{:telemetry, "~> 0.4 or ~> 1.0", [hex: :telemetry, repo: "hexpm", optional: false]}], "hexpm", "0e085b93012cd1057b378fce40cbfbf381ff6d957a382bfdd5eca1a98eec2535"}, "timex": {:hex, :timex, "3.7.11", "bb95cb4eb1d06e27346325de506bcc6c30f9c6dea40d1ebe390b262fad1862d1", [:mix], [{:combine, "~> 0.10", [hex: :combine, repo: "hexpm", optional: false]}, {:gettext, "~> 0.20", [hex: :gettext, repo: "hexpm", optional: false]}, {:tzdata, "~> 1.1", [hex: :tzdata, repo: "hexpm", optional: false]}], "hexpm", "8b9024f7efbabaf9bd7aa04f65cf8dcd7c9818ca5737677c7b76acbc6a94d1aa"}, "tzdata": {:hex, :tzdata, "1.1.1", "20c8043476dfda8504952d00adac41c6eda23912278add38edc140ae0c5bcc46", [:mix], [{:hackney, "~> 1.17", [hex: :hackney, repo: "hexpm", optional: false]}], "hexpm", "a69cec8352eafcd2e198dea28a34113b60fdc6cb57eb5ad65c10292a6ba89787"}, "unicode_util_compat": {:hex, :unicode_util_compat, "0.7.0", "bc84380c9ab48177092f43ac89e4dfa2c6d62b40b8bd132b1059ecc7232f9a78", [:rebar3], [], "hexpm", "25eee6d67df61960cf6a794239566599b09e17e668d3700247bc498638152521"},