From 60893444d712c97d80d5500d9d90a4da2906f915 Mon Sep 17 00:00:00 2001 From: Kian-Meng Ang Date: Wed, 17 Mar 2021 21:08:08 +0800 Subject: [PATCH] Misc doc changes Besides other changes, this commit ensures the generated HTML doc for HexDocs.pm will become the main source doc for this Elixir library which leverage on ExDoc features. List of changes: * Set and use latest ex_doc * Set readme as main html page * Add changelog to html doc * Fix markdowns * Update gitignore * Update formatter config * Add license section * Refactor project config * Badges and more badges! --- .formatter.exs | 8 +- .gitignore | 33 ++++- CHANGELOG.md | 6 +- LICENSE | 2 +- README.md | 237 ++++++++++++++----------------- lib/exq/adapters/queue.ex | 4 +- lib/exq/adapters/queue/mock.ex | 4 +- lib/exq/adapters/queue/redis.ex | 4 +- lib/exq/api.ex | 70 +++++---- lib/exq/api/server.ex | 4 +- lib/exq/dequeue/behaviour.ex | 6 +- lib/exq/enqueue_api.ex | 5 +- lib/exq/enqueuer.ex | 2 +- lib/exq/enqueuer/server.ex | 8 +- lib/exq/manager/server.ex | 4 +- lib/exq/middleware/pipeline.ex | 7 +- lib/exq/middleware/server.ex | 52 +++---- lib/exq/middleware/telemetry.ex | 87 ++++++------ lib/exq/mock.ex | 12 +- lib/exq/redis/job_queue.ex | 1 + lib/exq/support/job.ex | 2 +- lib/exq/support/mode.ex | 6 +- lib/exq/support/opts.ex | 8 +- lib/exq/support/process.ex | 6 +- lib/exq/support/randomize.ex | 4 + lib/exq/support/time.ex | 4 + lib/exq/worker/metadata.ex | 9 +- lib/exq/worker/server.ex | 29 ++-- lib/exq/worker/supervisor.ex | 4 + lib/exq/worker_drainer/server.ex | 12 +- mix.exs | 54 ++++--- mix.lock | 10 +- 32 files changed, 384 insertions(+), 320 deletions(-) diff --git a/.formatter.exs b/.formatter.exs index 33223f19..d2cda26e 100644 --- a/.formatter.exs +++ b/.formatter.exs @@ -1,8 +1,4 @@ +# Used by "mix format" [ - inputs: [ - "config/**/*.{ex,exs}", - "lib/**/*.{ex,exs}", - "test/**/*.{ex,exs}", - "mix.exs" - ] + inputs: ["{mix,.formatter}.exs", "{config,lib,test}/**/*.{ex,exs}"] ] diff --git a/.gitignore b/.gitignore index 06d5ee8b..687d0c08 100644 --- a/.gitignore +++ b/.gitignore @@ -1,13 +1,36 @@ +# The directory Mix will write compiled artifacts to. +/_build/ + +# If you run "mix test --cover", coverage assets end up here. +/cover/ + +# The directory Mix downloads your dependencies sources to. +/deps/ + +# Where third-party dependencies like ExDoc output generated docs. +/doc/ + +# Ignore .fetch files in case you like to edit your project deps locally. +/.fetch + +# If the VM crashes, it generates a dump, let's ignore it too. +erl_crash.dump + +# Also ignore archive artifacts (built via "mix archive.build"). +*.ez + +# Ignore package tarball (built via "mix hex.build"). +exq-*.tar + +# Temporary files for e.g. tests. +/tmp/ + +# Misc. /ebin priv/tmp_downloads/ -/deps -/erl_crash.dump -/tmp -_build/ *.swp *.log .*.swp *.swo stdout -/doc dump.rdb diff --git a/CHANGELOG.md b/CHANGELOG.md index 8d5649ab..c70a5555 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -27,7 +27,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added - Queue adapter for mock testing @ananthakumaran and @samidarko -## [0.13.4] - 2019-11-3 +## [0.13.4] - 2019-11-03 ### Fixed - Remove unnecessary serialization of enqueue calls #390 by @ananthakumaran and @sb8244 @@ -66,7 +66,7 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). - Configuration for Mix Format by @chulkilee. - Use :microsecond vs :microseconds by @KalvinHom. -## Changed +### Changed - Redis options are now passed in via `redis_options` by @ryansch and @ananthakumaran. - Removed redix_sentinel dependency, now supported by new Redix version by @ananthakumaran. @@ -112,5 +112,3 @@ The format is based on [Keep a Changelog](http://keepachangelog.com/en/1.0.0/). ### Added - Redis Sentinel support by @ananthakumaran. - Make redis module name and start_link args configurable @ananthakumaran. - - diff --git a/LICENSE b/LICENSE index ba7bca1a..039f7307 100644 --- a/LICENSE +++ b/LICENSE @@ -1,4 +1,4 @@ -Copyright 2014 Alex Kira +Copyright 2014 Alex Kira Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. diff --git a/README.md b/README.md index 091cfb60..da7be484 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,12 @@ # Exq +[![CI](https://github.com/akira/exq/actions/workflows/ci.yml/badge.svg)](https://github.com/akira/exq/actions/workflows/ci.yml) [![Coveralls Coverage](https://img.shields.io/coveralls/akira/exq.svg)](https://coveralls.io/github/akira/exq) -[![Hex.pm Version](https://img.shields.io/hexpm/v/exq.svg)](https://hex.pm/packages/exq) +[![Module Version](https://img.shields.io/hexpm/v/exq.svg)](https://hex.pm/packages/exq) +[![Hex Docs](https://img.shields.io/badge/hex-docs-lightgreen.svg)](https://hexdocs.pm/exq/) +[![Total Download](https://img.shields.io/hexpm/dt/exq.svg)](https://hex.pm/packages/exq) +[![License](https://img.shields.io/hexpm/l/exq.svg)](https://github.com/akira/exq/blob/master/LICENSE) +[![Last Updated](https://img.shields.io/github/last-commit/akira/exq.svg)](https://github.com/akira/exq/commits/master) Exq is a job processing library compatible with Resque / Sidekiq for the [Elixir](http://elixir-lang.org) language. * Exq uses Redis as a store for background processing jobs. @@ -39,12 +44,14 @@ Some OTP related documentation to look at: If you need a durable jobs, retries with exponential backoffs, dynamically scheduled jobs in the future - that are all able to survive application restarts, then an externally backed queueing library such as Exq could be a good fit. -If you are starting a brand new project, I would also take a look at [Faktory](https://github.com/contribsys/faktory). It provides language indepedent queueuing system, which means this logic doesn't have to be implemented across different languages and can use a thin client such as [faktory_worker_ex](https://github.com/cjbottaro/faktory_worker_ex). +If you are starting a brand new project, I would also take a look at [Faktory](https://github.com/contribsys/faktory). It provides language independent queueing system, which means this logic doesn't have to be implemented across different languages and can use a thin client such as [faktory_worker_ex](https://github.com/cjbottaro/faktory_worker_ex). -## Getting Started: +## Getting Started + +### Pre-requisite -## Pre-requisite: This assumes you have an instance of [Redis](http://redis.io/) to use. The easiest way to install it on OSX is via brew: + ``` > brew install redis ``` @@ -54,27 +61,28 @@ To start it: ``` ### Screencast on elixircasts.io: + If you prefer video instructions, check out the screencast on elixircasts.io which details how to install and use the Exq library: https://elixircasts.io/elixir-job-processing-with-exq -### Installation: -Add exq to your mix.exs deps (replace version with the latest hex.pm package version): +### Installation +Add `:exq` to your `mix.exs` deps (replace version with the latest hex.pm package version): ```elixir - defp deps do - [ - # ... other deps - {:exq, "~> 0.14.0"} - ] - end +defp deps do + [ + # ... other deps + {:exq, "~> 0.14.0"} + ] +end ``` Then run ```mix deps.get```. -### Configuration: +### Configuration -By default, Exq will use configuration from your config.exs file. You can use this to configure your Redis host, port, password, as well as namespace (which helps isolate the data in Redis). If you would like to specify your options as a redis url, that is also an option using the `url` config key (in which case you would not need to pass the other redis options). +By default, Exq will use configuration from your config.exs file. You can use this to configure your Redis host, port, password, as well as namespace (which helps isolate the data in Redis). If you would like to specify your options as a Redis URL, that is also an option using the `url` config key (in which case you would not need to pass the other Redis options). Configuration options may optionally be given in the `{:system, "VARNAME"}` format, which will resolve to the runtime environment value. @@ -86,8 +94,8 @@ Other options include: * The `shutdown_timeout` is the number of milliseconds to wait for workers to finish processing jobs when the application is shutting down. It defaults to 5000 ms. -* The `mode` option can be used to control what components of exq are started. This would be useful if you want to only enqueue jobs in one node and run the workers in different node. - * `:default` - starts worker, enqueuer and api. +* The `mode` option can be used to control what components of Exq are started. This would be useful if you want to only enqueue jobs in one node and run the workers in different node. + * `:default` - starts worker, enqueuer and API. * `:enqueuer` - starts only the enqueuer. * `:api` - starts only the api. * `[:api, :enqueuer]` - starts both enqueuer and api. @@ -110,7 +118,7 @@ config :exq, shutdown_timeout: 5000 ``` -### Concurrency: +### Concurrency Exq supports concurrency setting per queue. You can specify the same ```concurrency``` option to apply to each queue or specify it based on a per queue basis. @@ -136,7 +144,7 @@ config :exq, ``` -### Job Retries: +### Job Retries Exq will automatically retry failed job. It will use an exponential backoff timing similar to Sidekiq or delayed_job to retry failed jobs. It can be configured via these settings: @@ -151,7 +159,7 @@ config :exq, Note that ```scheduler_enable``` has to be set to ```true``` and ```max_retries``` should be greater than ```0```. -### Dead Jobs: +### Dead Jobs Any job that has failed more than ```max_retries``` times will be moved to dead jobs queue. Dead jobs could be manually re-enqueued via @@ -165,17 +173,17 @@ config :exq, ``` -### OTP Application: +### OTP Application You can add Exq into your OTP application list, and it will start an instance of Exq along with your application startup. It will use the configuration from your ```config.exs``` file. ```elixir - def application do - [ - applications: [:logger, :exq], - #other stuff... - ] - end +def application do + [ + applications: [:logger, :exq], + #other stuff... + ] +end ``` When using Exq through OTP, it will register a process under the name ```Elixir.Exq``` - you can use this atom where expecting a process name in the Exq module. @@ -190,23 +198,22 @@ config :exq, ``` ```elixir - # Define workers and child supervisors to be supervised - children = [ - # Start the Ecto repository - supervisor(MyApp.Repo, []), - # Start the endpoint when the application starts - supervisor(MyApp.Endpoint, []), - supervisor(Exq, []), - ] +# Define workers and child supervisors to be supervised +children = [ + # Start the Ecto repository + supervisor(MyApp.Repo, []), + # Start the endpoint when the application starts + supervisor(MyApp.Endpoint, []), + supervisor(Exq, []), +] ``` -### Sentinel: +### Sentinel Exq uses [Redix](https://github.com/whatyouhide/redix) client for communication with redis server. The client can be configured to use sentinel via `redis_options`. Note: you need to have Redix 0.9.0+. - ```elixir config :exq redis_options: [ @@ -220,27 +227,30 @@ config :exq ``` -## Using iex: -If you'd like to try Exq out on the iex console, you can do this by typing -``` -mix deps.get -``` -and then +## Using IEx + +If you'd like to try Exq out on the iex console, you can do this by typing: + +```bash +> mix deps.get ``` -iex -S mix + +and then: + +```bash +> iex -S mix ``` -### Standalone Exq: +### Standalone Exq You can run Exq standalone from the command line, to run it: -``` +```bash > mix do app.start, exq.run ``` ## Workers - ### Enqueuing jobs: To enqueue jobs: @@ -266,28 +276,27 @@ You can also enqueue jobs without starting workers: {:ok, ack} = Exq.Enqueuer.enqueue(Exq.Enqueuer, "default", MyWorker, []) ``` -You can also schedule jobs to start at a future time: -You need to make sure scheduler_enable is set to true -Schedule a job to start in 5 mins +You can also schedule jobs to start at a future time. You need to make sure scheduler_enable is set to true. + +Schedule a job to start in 5 mins: + ```elixir {:ok, ack} = Exq.enqueue_in(Exq, "default", 300, MyWorker, ["arg1", "arg2"]) ``` -Schedule a job to start at 8am 2015-12-25 UTC +Schedule a job to start at 8am 2015-12-25 UTC: + ```elixir time = Timex.now() |> Timex.shift(days: 8) {:ok, ack} = Exq.enqueue_at(Exq, "default", time, MyWorker, ["arg1", "arg2"]) ``` -### Creating Workers: +### Creating Workers -To create a worker, create an elixir module matching the worker name that will be -enqueued. To process a job with "MyWorker", create a MyWorker module. Note that the perform also needs to -match the number of arguments as well. +To create a worker, create an elixir module matching the worker name that will be enqueued. To process a job with "MyWorker", create a MyWorker module. Note that the perform also needs to match the number of arguments as well. Here is an example of a worker: - ```elixir defmodule MyWorker do def perform do @@ -296,16 +305,19 @@ end ``` We could enqueue a job to this worker: + ```elixir {:ok, jid} = Exq.enqueue(Exq, "default", MyWorker, []) ``` The 'perform' method will be called with matching args. For example: + ```elixir {:ok, jid} = Exq.enqueue(Exq, "default", "MyWorker", [arg1, arg2]) ``` Would match: + ```elixir defmodule MyWorker do def perform(arg1, arg2) do @@ -326,43 +338,51 @@ defmodule MyWorker do end ``` -### Dynamic queue subscriptions: +### Dynamic queue subscriptions -The list of queues that are being monitored by Exq is determined by the config.exs file or the parameters passed to Exq.start_link. However, we can also dynamically add and remove queue subscriptions after exq has started. +The list of queues that are being monitored by Exq is determined by the ```config.exs``` file or the parameters passed to Exq.start_link. However, we can also dynamically add and remove queue subscriptions after Exq has started. To subscribe to a new queue: + ```elixir # last arg is optional and is the max concurrency for the queue :ok = Exq.subscribe(Exq, "new_queue_name", 10) ``` To unsubscribe from a queue: + ```elixir :ok = Exq.unsubscribe(Exq, "queue_to_unsubscribe") ``` To unsubscribe from all queues: + ```elixir :ok = Exq.unsubscribe_all(Exq) ``` ## Middleware Support -If you'd like to customize worker execution and/or create plugins like Sidekiq/Resque have, Exq supports custom middleware. The first step would be to define the middleware in config.exs and add your middleware into the chain: +If you'd like to customize worker execution and/or create plugins like Sidekiq/Resque have, Exq supports custom middleware. The first step would be to define the middleware in ```config.exs``` and add your middleware into the chain: + ```elixir - middleware: [Exq.Middleware.Stats, Exq.Middleware.Job, Exq.Middleware.Manager, - Exq.Middleware.Logger] +middleware: [Exq.Middleware.Stats, Exq.Middleware.Job, Exq.Middleware.Manager, + Exq.Middleware.Logger] ``` + You can then create a module that implements the middleware behavior and defines `before_work`, `after_processed_work` and `after_failed_work` functions. You can also halt execution of the chain as well. For a simple example of middleware implementation, see the [Exq Logger Middleware](https://github.com/akira/exq/blob/master/lib/exq/middleware/logger.ex). ## Using with Phoenix and Ecto If you would like to use Exq alongside Phoenix and Ecto, add `:exq` to your mix.exs application list: + ```elixir - def application do - [mod: {Chat, []}, - applications: [:phoenix, :phoenix_html, :cowboy, :logger, :exq]] - end +def application do + [ + mod: {Chat, []}, + applications: [:phoenix, :phoenix_html, :cowboy, :logger, :exq] + ] +end ``` Assuming you will be accessing the database from Exq workers, you will want to lower the concurrency level for those workers, as they are using a finite pool of connections and can potentially back up and time out. You can lower this through the ```concurrency``` setting, or perhaps use a different queue for database workers that have a lower concurrency just for that queue. Inside your worker, you would then be able to use the Repo to work with the database: @@ -377,9 +397,10 @@ end ## Using alongside Sidekiq / Resque -To use alongside Sidekiq / Resque, make sure your namespaces as configured in exq match the namespaces you are using in Sidekiq. By default, exq will use the ```exq``` namespace, so you will have to change that. +To use alongside Sidekiq / Resque, make sure your namespaces as configured in Exq match the namespaces you are using in Sidekiq. By default, Exq will use the ```exq``` namespace, so you will have to change that. Another option is to modify Sidekiq to use the Exq namespace in the sidekiq initializer in your ruby project: + ```ruby Sidekiq.configure_server do |config| config.redis = { url: 'redis://127.0.0.1:6379', namespace: 'exq' } @@ -414,6 +435,7 @@ Exq identifies each node using an identifier. By default machine's hostname is u config :exq, node_identifier: MyApp.CustomNodeIdentifier ``` + ```elixir defmodule MyApp.CustomNodeIdentifier do @behaviour Exq.NodeIdentifier.Behaviour @@ -431,7 +453,7 @@ Same node recovery is straightforward and works well if the number of worker nod Heartbeat mechanism helps in these cases. Each node registers a heartbeat at regular interval. If any node misses 5 consecutive heartbeats, it will be considered dead and all the in-progress jobs belong to that node will be re-enqueued. -This feature is disabled by default and can be enabled using the following config. +This feature is disabled by default and can be enabled using the following config:j ```elixir config :exq, @@ -440,7 +462,7 @@ config :exq, missed_heartbeats_allowed: 5 ``` -## Web UI: +## Web UI Exq has a separate repo, exq_ui which provides with a Web UI to monitor your workers: @@ -448,7 +470,7 @@ Exq has a separate repo, exq_ui which provides with a Web UI to monitor your wor See https://github.com/akira/exq_ui for more details. -## Starting Exq manually: +## Starting Exq manually Typically, Exq will start as part of the application along with the configuration you have set. However, you can also start Exq manually and set your own configuration per instance. @@ -458,8 +480,7 @@ Here is an example of how to start Exq manually: {:ok, sup} = Exq.start_link ``` -To connect with custom configuration options (if you need multiple instances of Exq for example), you can pass in options -under start_link: +To connect with custom configuration options (if you need multiple instances of Exq for example), you can pass in options under start_link: ```elixir {:ok, sup} = Exq.start_link([host: "127.0.0.1", port: 6379, namespace: "x"]) @@ -473,7 +494,7 @@ By default, Exq will register itself under the ```Elixir.Exq``` atom. You can c ## Testing -`Exq.Mock` module provides few options to test your workers. +`Exq.Mock` module provides few options to test your workers: ```elixir # change queue_adapter in config/test.exs @@ -485,26 +506,21 @@ Exq.Mock.start_link(mode: :redis) ``` `Exq.Mock` currently supports three modes. The default mode can provided -on the `Exq.Mock.start_link` call. The mode could be overriden for +on the `Exq.Mock.start_link` call. The mode could be overridden for each test by calling `Exq.Mock.set_mode(:fake)` ### redis -This could be used for integration testing. Doesn't support `async: -true` option. +This could be used for integration testing. Doesn't support `async: true` option. ### fake -The jobs get enqueued in a local queue and never get -executed. `Exq.Mock.jobs()` returns all the jobs. Supports `async: -true` option. +The jobs get enqueued in a local queue and never get executed. `Exq.Mock.jobs()` returns all the jobs. Supports `async: true` option. ### inline The jobs get executed in the same process. Supports `async: true` option. - - ## Donation To donate, send to: @@ -529,62 +545,29 @@ mix test --no-start ``` To run the full suite, including failure conditions (can have some false negatives): + ``` mix test --trace --include failure_scenarios:true --no-start ``` -## Maintainers: -Anantha Kumaran / @ananthakumaran (Lead) - -## Contributors: - -Justin McNally (j-mcnally) (structtv) - -zhongwencool (zhongwencool) - -Joe Webb (ImJoeWebb) - -Chelsea Robb (chelsea) -Nick Sanders (nicksanders) +## Maintainers -Nick Gal (nickgal) - -Ben Wilson (benwilson512) - -Mike Lawlor (disbelief) - -colbyh (colbyh) - -Udo Kramer (optikfluffel) - -Andreas Franzén (triptec) - -Josh Kalderimis (joshk) - -Daniel Perez (tuvistavie) - -Victor Rodrigues (rodrigues) - -Denis Tataurov (sineed) - -Joe Honzawa (Joe-noh) - -Aaron Jensen (aaronjensen) - -Andrew Vy (andrewvy) - -David Le (dl103) - -Roman Smirnov (romul) +Anantha Kumaran / @ananthakumaran (Lead) -Thomas Athanas (typicalpixel) +## Contributors -Wen Li (wli0503) +Justin McNally (j-mcnally) (structtv), zhongwencool (zhongwencool), Joe Webb (ImJoeWebb), Chelsea Robb (chelsea), Nick Sanders (nicksanders), Nick Gal (nickgal), Ben Wilson (benwilson512), Mike Lawlor (disbelief), colbyh (colbyh), Udo Kramer (optikfluffel), Andreas Franzén (triptec),Josh Kalderimis (joshk), Daniel Perez (tuvistavie), Victor Rodrigues (rodrigues), Denis Tataurov (sineed), Joe Honzawa (Joe-noh), Aaron Jensen (aaronjensen), Andrew Vy (andrewvy), David Le (dl103), Roman Smirnov (romul), Thomas Athanas (typicalpixel), Wen Li (wli0503), Akshay (akki91), Rob Gilson (D1plo1d), edmz (edmz), and Benjamin Tan Wei Hao (benjamintanweihao). -Akshay (akki91) +## Copyright and License -Rob Gilson (D1plo1d) +Copyright (c) 2014 Alex Kira -edmz (edmz) +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at [http://www.apache.org/licenses/LICENSE-2.0](http://www.apache.org/licenses/LICENSE-2.0) -Benjamin Tan Wei Hao (benjamintanweihao) +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. diff --git a/lib/exq/adapters/queue.ex b/lib/exq/adapters/queue.ex index 914d8f23..9801d4e8 100644 --- a/lib/exq/adapters/queue.ex +++ b/lib/exq/adapters/queue.ex @@ -1,8 +1,9 @@ defmodule Exq.Adapters.Queue do @moduledoc ~S""" - Behaviour for creating Exq queue adapters + Behaviour for creating Exq queue adapters. ## Example + defmodule Exq.Adapters.Queue.CustomAdapter do @behaviour Exq.Adapters.Queue def enqueue(pid, queue, worker, args, options) do @@ -17,6 +18,7 @@ defmodule Exq.Adapters.Queue do enqueue_in_somehow(pid, queue, offset, worker, args, options) end end + """ @typedoc "The GenServer name" diff --git a/lib/exq/adapters/queue/mock.ex b/lib/exq/adapters/queue/mock.ex index 7fcbf196..794770ea 100644 --- a/lib/exq/adapters/queue/mock.ex +++ b/lib/exq/adapters/queue/mock.ex @@ -1,6 +1,8 @@ defmodule Exq.Adapters.Queue.Mock do @moduledoc """ - Mock queue. Designed to be used when testing your application. + Mock queue. + + Designed to be used when testing your application. """ @behaviour Exq.Adapters.Queue diff --git a/lib/exq/adapters/queue/redis.ex b/lib/exq/adapters/queue/redis.ex index 7e048aec..9e415153 100644 --- a/lib/exq/adapters/queue/redis.ex +++ b/lib/exq/adapters/queue/redis.ex @@ -1,8 +1,8 @@ defmodule Exq.Adapters.Queue.Redis do @moduledoc """ - Redis based Asynchronous queue. Enqueue the job by using the GenServer API. + Redis based Asynchronous queue. - Default queue. Designed to be used in production. + Enqueue the job by using the GenServer API. Default queue. Designed to be used in production. """ alias Exq.Support.Config alias Exq.Redis.JobQueue diff --git a/lib/exq/api.ex b/lib/exq/api.ex index ee95918c..55150887 100644 --- a/lib/exq/api.ex +++ b/lib/exq/api.ex @@ -1,7 +1,8 @@ defmodule Exq.Api do @moduledoc """ Interface for retrieving Exq stats. - Pid is currently Exq.Api process + + Pid is currently Exq.Api process. """ def start_link(opts \\ []) do @@ -9,13 +10,14 @@ defmodule Exq.Api do end @doc """ - List of queues with jobs (empty queues are deleted) + List of queues with jobs (empty queues are deleted). Expected args: * `pid` - Exq.Api process Returns: - * `{:ok, queues}` - list of queue + * `{:ok, queues}` - list of queue + """ def queues(pid) do GenServer.call(pid, :queues) @@ -29,7 +31,8 @@ defmodule Exq.Api do * `queue` - Queue name Returns: - * `{:ok, queues}` - list of queue + * `{:ok, queues}` - list of queue + """ def remove_queue(pid, queue) do GenServer.call(pid, {:remove_queue, queue}) @@ -42,7 +45,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, num_busy}` - number of busy workers + * `{:ok, num_busy}` - number of busy workers + """ def busy(pid) do GenServer.call(pid, :busy) @@ -55,7 +59,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [processes]}` + * `{:ok, [processes]}` + """ def processes(pid) do GenServer.call(pid, :processes) @@ -72,7 +77,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [{queue, [jobs]}, {queue, [jobs]}]}` + * `{:ok, [{queue, [jobs]}, {queue, [jobs]}]}` + """ def jobs(pid) do GenServer.call(pid, :jobs) @@ -86,7 +92,8 @@ defmodule Exq.Api do * `queue` - Queue name Returns: - * `{:ok, [jobs]}` + * `{:ok, [jobs]}` + """ def jobs(pid, queue) do GenServer.call(pid, {:jobs, queue}) @@ -99,7 +106,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [jobs]}` + * `{:ok, [jobs]}` + """ def retries(pid) do GenServer.call(pid, :retries) @@ -112,7 +120,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [jobs]}` + * `{:ok, [jobs]}` + """ def scheduled(pid) do GenServer.call(pid, {:jobs, :scheduled}) @@ -125,7 +134,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [{job, scheduled_at}]}` + * `{:ok, [{job, scheduled_at}]}` + """ def scheduled_with_scores(pid) do GenServer.call(pid, {:jobs, :scheduled_with_scores}) @@ -144,7 +154,8 @@ defmodule Exq.Api do * `jid` - Unique identifier for the job Returns: - * `:ok` + * `:ok` + """ def remove_job(pid, queue, jid) do GenServer.call(pid, {:remove_job, queue, jid}) @@ -157,7 +168,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [{queue, num_jobs}, {queue, num_jobs}]}` + * `{:ok, [{queue, num_jobs}, {queue, num_jobs}]}` + """ def queue_size(pid) do GenServer.call(pid, :queue_size) @@ -171,7 +183,8 @@ defmodule Exq.Api do * `queue` - The name of the queue to find the number of jobs for Returns: - * `{:ok, num_jobs}` + * `{:ok, num_jobs}` + """ def queue_size(pid, queue) do GenServer.call(pid, {:queue_size, queue}) @@ -184,7 +197,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, [jobs]}` + * `{:ok, [jobs]}` + """ def failed(pid) do GenServer.call(pid, :failed) @@ -202,7 +216,8 @@ defmodule Exq.Api do * `jid` - Unique identifier for the job Returns: - * `:ok` + * `:ok` + """ def remove_failed(pid, jid) do GenServer.call(pid, {:remove_failed, jid}) @@ -219,7 +234,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, num_failed}` - number of failed jobs + * `{:ok, num_failed}` - number of failed jobs + """ def failed_size(pid) do GenServer.call(pid, :failed_size) @@ -237,7 +253,8 @@ defmodule Exq.Api do * `jid` - Unique identifier for the job Returns: - * `:ok` + * `:ok` + """ def remove_retry(pid, jid) do GenServer.call(pid, {:remove_retry, jid}) @@ -254,7 +271,8 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, num_retry}` - number of jobs to be retried + * `{:ok, num_retry}` - number of jobs to be retried + """ def retry_size(pid) do GenServer.call(pid, :retry_size) @@ -272,7 +290,8 @@ defmodule Exq.Api do * `jid` - Unique identifier for the job Returns: - * `:ok` + * `:ok` + """ def remove_scheduled(pid, jid) do GenServer.call(pid, {:remove_scheduled, jid}) @@ -289,15 +308,17 @@ defmodule Exq.Api do * `pid` - Exq.Api process Returns: - * `{:ok, num_scheduled}` - number of scheduled jobs enqueued + * `{:ok, num_scheduled}` - number of scheduled jobs enqueued + """ def scheduled_size(pid) do GenServer.call(pid, :scheduled_size) end @doc """ - Return stat for given key - Examples of keys are `processed`, `failed` + Return stat for given key. + + Examples of keys are `processed` and `failed`. Expected args: * `pid` - Exq.Api process @@ -305,7 +326,8 @@ defmodule Exq.Api do * `queue` - Queue name Returns: - * `{:ok, stat}` stat for key + * `{:ok, stat}` stat for key + """ def stats(pid, key) do GenServer.call(pid, {:stats, key}) diff --git a/lib/exq/api/server.ex b/lib/exq/api/server.ex index 34a77d8f..f849356c 100644 --- a/lib/exq/api/server.ex +++ b/lib/exq/api/server.ex @@ -1,6 +1,6 @@ defmodule Exq.Api.Server do @moduledoc """ - The Api deals with getting current stats for the UI / API. + The API deals with getting current stats for the UI / API. """ alias Exq.Support.Config @@ -18,7 +18,7 @@ defmodule Exq.Api.Server do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def init(opts) do diff --git a/lib/exq/dequeue/behaviour.ex b/lib/exq/dequeue/behaviour.ex index cd5c7446..7d90e89d 100644 --- a/lib/exq/dequeue/behaviour.ex +++ b/lib/exq/dequeue/behaviour.ex @@ -1,9 +1,9 @@ defmodule Exq.Dequeue.Behaviour do @moduledoc """ - Custom concurreny or rate limiting at a queue level can be achieved - by implementing the Dequeue behaviour + Custom concurrency or rate limiting at a queue level can be achieved + by implementing the Dequeue behaviour. - The following config can be used to customize dequeue behaviour for a queue + The following config can be used to customize dequeue behaviour for a queue: config :exq, queues: [{"default", {RateLimiter, options}}] diff --git a/lib/exq/enqueue_api.ex b/lib/exq/enqueue_api.ex index d3fe52e7..a16732b3 100644 --- a/lib/exq/enqueue_api.ex +++ b/lib/exq/enqueue_api.ex @@ -22,8 +22,9 @@ defmodule Exq.Enqueuer.EnqueueApi do * `options` - job options, for example [max_retries: `Integer`, jid: `String`] Returns: - * `{:ok, jid}` if the job was enqueued successfully, with `jid` = Job ID. - * `{:error, reason}` if there was an error enqueueing job + * `{:ok, jid}` if the job was enqueued successfully, with `jid` = Job ID. + * `{:error, reason}` if there was an error enqueueing job + """ def enqueue(pid, queue, worker, args), do: enqueue(pid, queue, worker, args, @default_options) diff --git a/lib/exq/enqueuer.ex b/lib/exq/enqueuer.ex index 821e0017..b9a958c1 100644 --- a/lib/exq/enqueuer.ex +++ b/lib/exq/enqueuer.ex @@ -1,6 +1,6 @@ defmodule Exq.Enqueuer do @moduledoc """ - Enqueuer + Enqueuer. """ # Mixin EnqueueApi diff --git a/lib/exq/enqueuer/server.ex b/lib/exq/enqueuer/server.ex index 09e68bd9..3d7953f8 100644 --- a/lib/exq/enqueuer/server.ex +++ b/lib/exq/enqueuer/server.ex @@ -1,7 +1,8 @@ defmodule Exq.Enqueuer.Server do @moduledoc """ - The Enqueuer is responsible for enqueueing jobs into Redis. It can - either be called directly by the client, or instantiated as a standalone process. + The Enqueuer is responsible for enqueueing jobs into Redis. + + It can either be called directly by the client, or instantiated as a standalone process. It supports enqueuing immediate jobs, or scheduling jobs in the future. @@ -11,6 +12,7 @@ defmodule Exq.Enqueuer.Server do * `:queues` - Array of currently active queues (TODO: Remove, I suspect it's not needed). * `:redis` - pid of Redis process. * `:scheduler_poll_timeout` - How often to poll Redis for scheduled / retry jobs. + """ require Logger @@ -27,7 +29,7 @@ defmodule Exq.Enqueuer.Server do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def init(opts) do diff --git a/lib/exq/manager/server.ex b/lib/exq/manager/server.ex index bb36061e..70fc17eb 100644 --- a/lib/exq/manager/server.ex +++ b/lib/exq/manager/server.ex @@ -1,6 +1,6 @@ defmodule Exq.Manager.Server do @moduledoc """ - The Manager module is the main orchestrator for the system + The Manager module is the main orchestrator for the system. It is also the entry point Pid process used by the client to interact with the Exq system. @@ -74,7 +74,7 @@ defmodule Exq.Manager.Server do 3. Uses the JobQueue module to fetch jobs. The JobQueue module does this through a single MULT RPOPLPUSH command issued to Redis with the targeted queue. - This command atomicaly pops an item off the queue and stores the item in a backup queue. + This command atomically pops an item off the queue and stores the item in a backup queue. The backup queue is keyed off the queue and node id, so each node would have their own backup queue. diff --git a/lib/exq/middleware/pipeline.ex b/lib/exq/middleware/pipeline.ex index 70fe19d1..6119326e 100644 --- a/lib/exq/middleware/pipeline.ex +++ b/lib/exq/middleware/pipeline.ex @@ -1,8 +1,10 @@ defmodule Exq.Middleware.Pipeline do @moduledoc """ Pipeline is a structure that is used as an argument in functions of module with - `Exq.Middleware.Behaviour` behaviour. This structure must be returned by particular function - to be used in the next middleware based on defined middleware chain. + `Exq.Middleware.Behaviour` behaviour. + + This structure must be returned by particular function to be used in the next + middleware based on defined middleware chain. Pipeline contains the following options: @@ -18,6 +20,7 @@ defmodule Exq.Middleware.Pipeline do - Exq.Middleware.Logger: Will NOT record job as done or failed with timestamp - Exq.Middleware.Manager: Will NOT update worker counter - Exq.Middleware.Stats: Will NOT remove job from processes queue + """ defstruct assigns: %{}, diff --git a/lib/exq/middleware/server.ex b/lib/exq/middleware/server.ex index adecb15b..db275d72 100644 --- a/lib/exq/middleware/server.ex +++ b/lib/exq/middleware/server.ex @@ -1,7 +1,9 @@ defmodule Exq.Middleware.Server do @moduledoc """ Middleware Server is responsible for storing middleware chain that is evaluated - when performing particular job. Middleware chain defaults to Stats, Job and Manager middlewares. + when performing particular job. + + Middleware chain defaults to Stats, Job and Manager middlewares. To push new middleware you must create module with common interface. Interface is similar to `Plug` implementation. It has three functions, every function receives `Exq.Middlewares.Pipeline` structure @@ -12,60 +14,58 @@ defmodule Exq.Middleware.Server do For example, here is a valid middleware module: - ```elixir - defmodule MyMiddleware do - @behaiour Exq.Middleware.Behaviour + defmodule MyMiddleware do + @behaiour Exq.Middleware.Behaviour - def before_work(pipeline) do - # some functionality goes here... - pipeline - end + def before_work(pipeline) do + # some functionality goes here... + pipeline + end - def after_processed_work(pipeline) do - # some functionality goes here... - pipeline - end + def after_processed_work(pipeline) do + # some functionality goes here... + pipeline + end - def after_failed_work(pipeline) do - # some functionality goes here... - pipeline + def after_failed_work(pipeline) do + # some functionality goes here... + pipeline + end end - end - ``` To add this module to middleware chain: - ```elixir - Exq.Middleware.Server.push(middleware_server_pid, MyMiddleware) - ``` + Exq.Middleware.Server.push(middleware_server_pid, MyMiddleware) + """ use GenServer @doc """ - Starts middleware server + Starts middleware server. """ def start_link(opts \\ []) do GenServer.start_link(__MODULE__, default_middleware(opts), name: server_name(opts[:name])) end @doc """ - Adds specified `middleware` module into the end of middleware list. `middleware` should have - `Exq.Middleware.Behaviour` behaviour + Adds specified `middleware` module into the end of middleware list. + + `middleware` should have `Exq.Middleware.Behaviour` behaviour. """ def push(pid, middleware) do GenServer.cast(pid, {:push, middleware}) end @doc """ - Retrieves list of middleware modules + Retrieves list of middleware modules. """ def all(pid) do GenServer.call(pid, :all) end @doc """ - Returns middleware server name + Returns middleware server name. """ def server_name(name) do name = name || Exq.Support.Config.get(:name) @@ -78,7 +78,7 @@ defmodule Exq.Middleware.Server do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def handle_cast({:push, middleware}, state) do diff --git a/lib/exq/middleware/telemetry.ex b/lib/exq/middleware/telemetry.ex index fcbaefe6..8f4e35d2 100644 --- a/lib/exq/middleware/telemetry.ex +++ b/lib/exq/middleware/telemetry.ex @@ -6,20 +6,19 @@ defmodule Exq.Middleware.Telemetry do ### Exq telemetry events The middleware emit three events, same as what `:telemetry.span/3` emits. - - - `[:exq, :job, :start]` - Is invoked whenever a job starts. + * `[:exq, :job, :start]` - Is invoked whenever a job starts. ** Measurements ** - `system_time` (integer) - System time when the job started - - `[:exq, :job, :stop]` - Is invoked whenever a job completes successfully. + * `[:exq, :job, :stop]` - Is invoked whenever a job completes successfully. ** Measurements ** - `duration` (integer) - Duration of the job execution in native unit - - `[:exq, :job, :exception]` - Is invoked whenever a job fails. + * `[:exq, :job, :exception]` - Is invoked whenever a job fails. ** Measurements ** @@ -35,47 +34,45 @@ defmodule Exq.Middleware.Telemetry do ** Metadata ** - Each event has the following common metadata - - - `enqueued_at` (`DateTime.t/0`) - datetime the job was enqueued - - `queue` (`String.t/0`) - the name of the queue the job was executed in - - `class` (`String.t/0`) - the job's class - - `jid` (`String.t/0`) - the job's jid - - `retry_count` (integer) - number of times this job has failed so far - - - ### Example: - - ``` - defmodule MyApp.Application do - def start(_type, _args) do - children = [ - # ..... - {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} - ] - - opts = [strategy: :one_for_one, name: MyApp.Supervisor] - Supervisor.start_link(children, opts) - end - - defp metrics do - [ - counter("exq.job.stop.duration"), - counter("exq.job.exception.duration"), - distribution("exq.job.stop.duration", - buckets: [0.1, 0.2, 0.3, 0.5, 0.75, 1, 2, 3, 5, 10], - unit: {:native, :millisecond} - ), - distribution("exq.job.exception.duration", - buckets: [0.1, 0.2, 0.3, 0.5, 0.75, 1, 2, 3, 5, 10], - unit: {:native, :millisecond} - ), - summary("exq.job.stop.duration", unit: {:native, :millisecond}), - summary("exq.job.exception.duration", unit: {:native, :millisecond}) - ] - end - end - ``` + Each event has the following common metadata: + * `enqueued_at` (`DateTime.t/0`) - datetime the job was enqueued + * `queue` (`String.t/0`) - the name of the queue the job was executed in + * `class` (`String.t/0`) - the job's class + * `jid` (`String.t/0`) - the job's jid + * `retry_count` (integer) - number of times this job has failed so far + + + ### Examples + + defmodule MyApp.Application do + def start(_type, _args) do + children = [ + # ..... + {Telemetry.Metrics.ConsoleReporter, metrics: metrics()} + ] + + opts = [strategy: :one_for_one, name: MyApp.Supervisor] + Supervisor.start_link(children, opts) + end + + defp metrics do + [ + counter("exq.job.stop.duration"), + counter("exq.job.exception.duration"), + distribution("exq.job.stop.duration", + buckets: [0.1, 0.2, 0.3, 0.5, 0.75, 1, 2, 3, 5, 10], + unit: {:native, :millisecond} + ), + distribution("exq.job.exception.duration", + buckets: [0.1, 0.2, 0.3, 0.5, 0.75, 1, 2, 3, 5, 10], + unit: {:native, :millisecond} + ), + summary("exq.job.stop.duration", unit: {:native, :millisecond}), + summary("exq.job.exception.duration", unit: {:native, :millisecond}) + ] + end + end + """ @behaviour Exq.Middleware.Behaviour diff --git a/lib/exq/mock.ex b/lib/exq/mock.ex index acb7ed5f..90c45ae7 100644 --- a/lib/exq/mock.ex +++ b/lib/exq/mock.ex @@ -16,7 +16,8 @@ defmodule Exq.Mock do @doc """ Start Mock server - * `mode` - The default mode that's used for all tests. See `set_mode/1` for details. + * `mode` - The default mode that's used for all tests. See `set_mode/1` for details. + """ def start_link(options \\ []) do queue_adapter = Config.get(:queue_adapter) @@ -35,16 +36,17 @@ defmodule Exq.Mock do @doc """ Set the mode for current test - * `:redis` - jobs get enqueued and processed via redis. - * `:fake` - jobs get enqueued in a local queue - * `:inline` - jobs get executed in the same process + * `:redis` - jobs get enqueued and processed via redis. + * `:fake` - jobs get enqueued in a local queue + * `:inline` - jobs get executed in the same process + """ def set_mode(mode) when mode in [:redis, :inline, :fake] do GenServer.call(__MODULE__, {:mode, self(), mode}, @timeout) end @doc """ - List of enqueued jobs + List of enqueued jobs. This only works if the mode is set to `:fake` """ diff --git a/lib/exq/redis/job_queue.ex b/lib/exq/redis/job_queue.ex index 4f65dadf..fd3a290b 100644 --- a/lib/exq/redis/job_queue.ex +++ b/lib/exq/redis/job_queue.ex @@ -9,6 +9,7 @@ defmodule Exq.Redis.JobQueue do * Fetching scheduling jobs and moving them to current job list * Retrying or failing a job * Re-hydrating jobs from a backup queue + """ require Logger diff --git a/lib/exq/support/job.ex b/lib/exq/support/job.ex index b5be67df..27aeaeaa 100644 --- a/lib/exq/support/job.ex +++ b/lib/exq/support/job.ex @@ -1,6 +1,6 @@ defmodule Exq.Support.Job do @moduledoc """ - Serializable Job format used by Exq + Serializable Job format used by Exq. """ defstruct error_message: nil, diff --git a/lib/exq/support/mode.ex b/lib/exq/support/mode.ex index 9f3806f0..b9a655d7 100644 --- a/lib/exq/support/mode.ex +++ b/lib/exq/support/mode.ex @@ -1,14 +1,16 @@ defmodule Exq.Support.Mode do @moduledoc """ - This module defines several modes in which Exq can be used. These modes are: + This module defines several modes in which Exq can be used. + These modes are: * `default` - starts the default processes * `enqueuer` - starts processes which are responsible for job enqueueing * `api` - starts processes which are responsible for API usage + """ @doc """ - Returns child list for the main Exq supervisor + Returns child list for the main Exq supervisor. """ import Exq.Support.Opts, only: [redis_worker_opts: 1] diff --git a/lib/exq/support/opts.ex b/lib/exq/support/opts.ex index ab2ed509..472514c1 100644 --- a/lib/exq/support/opts.ex +++ b/lib/exq/support/opts.ex @@ -1,9 +1,13 @@ defmodule Exq.Support.Opts do + @moduledoc """ + Exq supported options. + """ + alias Exq.Support.Coercion alias Exq.Support.Config @doc """ - Return top supervisor's name default is Exq.Sup + Returns top supervisor's name default is `Exq.Sup`. """ def top_supervisor(name) do name = name || Config.get(:name) @@ -67,7 +71,7 @@ defmodule Exq.Support.Opts do end @doc """ - Return {redis_module, redis_args, gen_server_opts} + Returns `{redis_module, redis_args, gen_server_opts}`. """ def redis_worker_opts(opts) do {redis_opts, opts} = conform_opts(opts) diff --git a/lib/exq/support/process.ex b/lib/exq/support/process.ex index 31199ced..d5fabf0b 100644 --- a/lib/exq/support/process.ex +++ b/lib/exq/support/process.ex @@ -1,13 +1,13 @@ defmodule Exq.Support.Process do @moduledoc """ - Struct for in progress worker + Struct for in progress worker. """ defstruct pid: nil, host: nil, job: nil, started_at: nil alias Exq.Support.Config @doc """ - Serialize process to JSON + Serialize process to JSON. """ def encode(%__MODULE__{} = process) do Config.serializer().encode_process(%{ @@ -19,7 +19,7 @@ defmodule Exq.Support.Process do end @doc """ - Decode JSON into process + Decode JSON into process. """ def decode(serialized) do Config.serializer().decode_process(serialized) diff --git a/lib/exq/support/randomize.ex b/lib/exq/support/randomize.ex index 881ac71a..3cdbb24f 100644 --- a/lib/exq/support/randomize.ex +++ b/lib/exq/support/randomize.ex @@ -1,4 +1,8 @@ defmodule Exq.Support.Randomize do + @moduledoc """ + Helper functions for random number. + """ + def random(number) do Enum.random(0..number) end diff --git a/lib/exq/support/time.ex b/lib/exq/support/time.ex index 43de1254..9bf08141 100644 --- a/lib/exq/support/time.ex +++ b/lib/exq/support/time.ex @@ -1,4 +1,8 @@ defmodule Exq.Support.Time do + @moduledoc """ + Helper functions for data and time. + """ + import DateTime, only: [utc_now: 0, to_unix: 2, from_unix!: 2] def offset_from_now(offset) do diff --git a/lib/exq/worker/metadata.ex b/lib/exq/worker/metadata.ex index f4e752da..4f9ed771 100644 --- a/lib/exq/worker/metadata.ex +++ b/lib/exq/worker/metadata.ex @@ -1,8 +1,9 @@ defmodule Exq.Worker.Metadata do @moduledoc """ - Provides storage functionality for job metadata. The metadata is - associated with the worker pid and automatically discarded when the - worker process exits. + Provides storage functionality for job metadata. + + The metadata is associated with the worker pid and automatically discarded + when the worker process exits. """ use GenServer @@ -21,7 +22,7 @@ defmodule Exq.Worker.Metadata do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def init(opts) do diff --git a/lib/exq/worker/server.ex b/lib/exq/worker/server.ex index 85b6cc58..b53046bf 100644 --- a/lib/exq/worker/server.ex +++ b/lib/exq/worker/server.ex @@ -1,17 +1,18 @@ defmodule Exq.Worker.Server do @moduledoc """ - Worker process is responsible for the parsing and execution of a Job. It then - broadcasts results to Stats / Manager. + Worker process is responsible for the parsing and execution of a Job. - Currently uses the `terminate` callback to track job success/failure . + It then broadcasts results to Stats / Manager. + + Currently uses the `terminate` callback to track job success/failure. ## Initialization: - * `job_serialized` - Full JSON payload of the Job. - * `manager` - Manager process pid. - * `queue` - The queue the job came from. - * `stats` - Stats process pid. - * `namespace` - Redis namespace - * `host` - Host name + * `job_serialized` - Full JSON payload of the Job. + * `manager` - Manager process pid. + * `queue` - The queue the job came from. + * `stats` - Stats process pid. + * `namespace` - Redis namespace + * `host` - Host name Expects :work message after initialization to kickoff work. """ @@ -61,7 +62,7 @@ defmodule Exq.Worker.Server do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def init({job_serialized, manager, queue, stats, namespace, host, redis, middleware, metadata}) do @@ -85,8 +86,8 @@ defmodule Exq.Worker.Server do Kickoff work associated with worker. This step handles: - * Parsing of JSON object - * Preparation of target module + * Parsing of JSON object + * Preparation of target module Calls :dispatch to then call target module. """ @@ -104,7 +105,7 @@ defmodule Exq.Worker.Server do end @doc """ - Dispatch work to the target module (call :perform method of target) + Dispatch work to the target module (call :perform method of target). """ def handle_cast(:dispatch, state) do dispatch_work( @@ -117,7 +118,7 @@ defmodule Exq.Worker.Server do end @doc """ - Worker done with normal termination message + Worker done with normal termination message. """ def handle_cast({:done, result}, state) do state = diff --git a/lib/exq/worker/supervisor.ex b/lib/exq/worker/supervisor.ex index c3e641a3..d2c22d97 100644 --- a/lib/exq/worker/supervisor.ex +++ b/lib/exq/worker/supervisor.ex @@ -1,4 +1,8 @@ defmodule Exq.Worker.Supervisor do + @moduledoc """ + Supervisor for Exq Worker. + """ + import Supervisor.Spec def start_link(opts) do diff --git a/lib/exq/worker_drainer/server.ex b/lib/exq/worker_drainer/server.ex index 7cfb65e7..8363b1a2 100644 --- a/lib/exq/worker_drainer/server.ex +++ b/lib/exq/worker_drainer/server.ex @@ -1,9 +1,10 @@ defmodule Exq.WorkerDrainer.Server do @moduledoc """ The WorkerDrainer server is responsible for gracefully draining - workers when the application is shutting down. When shutdown starts - it instructs the Manager to stop accepting new jobs and then waits - for all currently in progress jobs to complete. + workers when the application is shutting down. + + When shutdown starts it instructs the Manager to stop accepting new jobs and + then waits for all currently in progress jobs to complete. If the jobs do not complete within an allowed timeout the WorkerDrainer will shut down, allowing the rest of the supervision tree (including the @@ -11,7 +12,6 @@ defmodule Exq.WorkerDrainer.Server do The length of the grace period can be configured with the `shutdown_timeout` option, which defaults to 5000 ms. - """ use GenServer @@ -26,7 +26,7 @@ defmodule Exq.WorkerDrainer.Server do end ## =========================================================== - ## gen server callbacks + ## GenServer callbacks ## =========================================================== def start_link(opts \\ []) do @@ -56,7 +56,7 @@ defmodule Exq.WorkerDrainer.Server do end ## =========================================================== - ## functions + ## Internal Functions ## =========================================================== defp drain_workers(state) do diff --git a/mix.exs b/mix.exs index fc45a7d4..00f46070 100644 --- a/mix.exs +++ b/mix.exs @@ -1,32 +1,22 @@ defmodule Exq.Mixfile do use Mix.Project + @source_url "https://github.com/akira/exq" + @version "0.14.0" + def project do [ app: :exq, - version: "0.14.0", + version: @version, elixir: "~> 1.6", elixirc_paths: ["lib"], - package: [ - maintainers: [ - "Alex Kira", - "zhongwencool", - "Anantha Kumaran" - ], - links: %{"GitHub" => "https://github.com/akira/exq"}, - licenses: ["Apache 2.0"], - files: ~w(lib test) ++ ~w(LICENSE mix.exs README.md) - ], - description: """ - Exq is a job processing library compatible with Resque / Sidekiq for the Elixir language. - """, - deps: deps(), test_coverage: [tool: ExCoveralls], - docs: [extras: ["README.md"]] + deps: deps(), + docs: docs(), + package: package() ] end - # Configuration for the OTP application def application do [ mod: {Exq, []}, @@ -34,8 +24,6 @@ defmodule Exq.Mixfile do ] end - # Returns the list of dependencies in the format: - # { :foobar, "0.1", git: "https://github.com/elixir-lang/foobar.git" } defp deps do [ {:elixir_uuid, ">= 1.2.0"}, @@ -46,10 +34,32 @@ defmodule Exq.Mixfile do {:flaky_connection, git: "https://github.com/hamiltop/flaky_connection.git", only: :test}, # docs - {:ex_doc, "~> 0.19", only: :dev}, - {:earmark, "~> 1.0", only: :dev}, - {:benchee, "~> 1.0", only: :dev}, + {:ex_doc, ">= 0.0.0", only: :dev, runtime: false}, + {:benchee, "~> 1.0", only: :dev, runtime: false}, {:ranch, "~> 1.6", only: :test, override: true} ] end + + defp package do + [ + description: """ + Exq is a job processing library compatible with Resque / Sidekiq for the + Elixir language. + """, + maintainers: ["Alex Kira", "zhongwencool", "Anantha Kumaran"], + licenses: ["Apache-2.0"], + files: ~w(lib test) ++ ~w(LICENSE mix.exs CHANGELOG.md README.md), + links: %{"GitHub" => @source_url} + ] + end + + defp docs do + [ + extras: ["CHANGELOG.md", "README.md"], + main: "readme", + formatters: ["html"], + source_url: @source_url, + source_ref: "v#{@version}" + ] + end end diff --git a/mix.lock b/mix.lock index 92a03ab9..24dcbbef 100644 --- a/mix.lock +++ b/mix.lock @@ -3,8 +3,9 @@ "certifi": {:hex, :certifi, "2.5.2", "b7cfeae9d2ed395695dd8201c57a2d019c0c43ecaf8b8bcb9320b40d6662f340", [:rebar3], [{:parse_trans, "~>3.3", [hex: :parse_trans, repo: "hexpm", optional: false]}], "hexpm", "3b3b5f36493004ac3455966991eaf6e768ce9884693d9968055aeeeb1e575040"}, "deep_merge": {:hex, :deep_merge, "1.0.0", "b4aa1a0d1acac393bdf38b2291af38cb1d4a52806cf7a4906f718e1feb5ee961", [:mix], [], "hexpm", "ce708e5f094b9cd4e8f2be4f00d2f4250c4095be93f8cd6d018c753894885430"}, "earmark": {:hex, :earmark, "1.3.1", "73812f447f7a42358d3ba79283cfa3075a7580a3a2ed457616d6517ac3738cb9", [:mix], [], "hexpm", "000aaeff08919e95e7aea13e4af7b2b9734577b3e6a7c50ee31ee88cab6ec4fb"}, + "earmark_parser": {:hex, :earmark_parser, "1.4.12", "b245e875ec0a311a342320da0551da407d9d2b65d98f7a9597ae078615af3449", [:mix], [], "hexpm", "711e2cc4d64abb7d566d43f54b78f7dc129308a63bc103fbd88550d2174b3160"}, "elixir_uuid": {:hex, :elixir_uuid, "1.2.0", "ff26e938f95830b1db152cb6e594d711c10c02c6391236900ddd070a6b01271d", [:mix], [], "hexpm", "e4d6e26434471761ed45a3545239da87af7b70904dd4442a55f87d06b137c56b"}, - "ex_doc": {:hex, :ex_doc, "0.19.3", "3c7b0f02851f5fc13b040e8e925051452e41248f685e40250d7e40b07b9f8c10", [:mix], [{:earmark, "~> 1.2", [hex: :earmark, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.10", [hex: :makeup_elixir, repo: "hexpm", optional: false]}], "hexpm", "0e11d67e662142fc3945b0ee410c73c8c956717fbeae4ad954b418747c734973"}, + "ex_doc": {:hex, :ex_doc, "0.24.0", "2df14354835afaabdf87cb2971ea9485d8a36ff590e4b6c250b4f60c8fdf9143", [:mix], [{:earmark_parser, "~> 1.4.0", [hex: :earmark_parser, repo: "hexpm", optional: false]}, {:makeup_elixir, "~> 0.14", [hex: :makeup_elixir, repo: "hexpm", optional: false]}, {:makeup_erlang, "~> 0.1", [hex: :makeup_erlang, repo: "hexpm", optional: false]}], "hexpm", "a0f4bcff21ceebea48414e49885d2a3e542200f76a2facf3f8faa54935eeb721"}, "excoveralls": {:hex, :excoveralls, "0.13.3", "edc5f69218f84c2bf61b3609a22ddf1cec0fbf7d1ba79e59f4c16d42ea4347ed", [:mix], [{:hackney, "~> 1.16", [hex: :hackney, repo: "hexpm", optional: false]}, {:jason, "~> 1.0", [hex: :jason, repo: "hexpm", optional: false]}], "hexpm", "cc26f48d2f68666380b83d8aafda0fffc65dafcc8d8650358e0b61f6a99b1154"}, "exjsx": {:hex, :exjsx, "3.2.1", "1bc5bf1e4fd249104178f0885030bcd75a4526f4d2a1e976f4b428d347614f0f", [:mix], [{:jsx, "~> 2.8.0", [hex: :jsx, repo: "hexpm", optional: false]}], "hexpm", "b55727b206dab96feb025267e5c122ddb448f55b6648f9156b8d481215d80290"}, "flaky_connection": {:git, "https://github.com/hamiltop/flaky_connection.git", "e3a09e7198e1b155f35291ffad438966648a8156", []}, @@ -12,11 +13,12 @@ "idna": {:hex, :idna, "6.0.1", "1d038fb2e7668ce41fbf681d2c45902e52b3cb9e9c77b55334353b222c2ee50c", [:rebar3], [{:unicode_util_compat, "0.5.0", [hex: :unicode_util_compat, repo: "hexpm", optional: false]}], "hexpm", "a02c8a1c4fd601215bb0b0324c8a6986749f807ce35f25449ec9e69758708122"}, "jason": {:hex, :jason, "1.1.2", "b03dedea67a99223a2eaf9f1264ce37154564de899fd3d8b9a21b1a6fd64afe7", [:mix], [{:decimal, "~> 1.0", [hex: :decimal, repo: "hexpm", optional: true]}], "hexpm", "fdf843bca858203ae1de16da2ee206f53416bbda5dc8c9e78f43243de4bc3afe"}, "jsx": {:hex, :jsx, "2.8.2", "7acc7d785b5abe8a6e9adbde926a24e481f29956dd8b4df49e3e4e7bcc92a018", [:mix, :rebar3], [], "hexpm", "b4c5d3230b397c8d95579e4a3d72826bb6463160130ccf4182f5be8579b5f44c"}, - "makeup": {:hex, :makeup, "0.8.0", "9cf32aea71c7fe0a4b2e9246c2c4978f9070257e5c9ce6d4a28ec450a839b55f", [:mix], [{:nimble_parsec, "~> 0.5.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "5fbc8e549aa9afeea2847c0769e3970537ed302f93a23ac612602e805d9d1e7f"}, - "makeup_elixir": {:hex, :makeup_elixir, "0.13.0", "be7a477997dcac2e48a9d695ec730b2d22418292675c75aa2d34ba0909dcdeda", [:mix], [{:makeup, "~> 0.8", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "adf0218695e22caeda2820eaba703fa46c91820d53813a2223413da3ef4ba515"}, + "makeup": {:hex, :makeup, "1.0.5", "d5a830bc42c9800ce07dd97fa94669dfb93d3bf5fcf6ea7a0c67b2e0e4a7f26c", [:mix], [{:nimble_parsec, "~> 0.5 or ~> 1.0", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "cfa158c02d3f5c0c665d0af11512fed3fba0144cf1aadee0f2ce17747fba2ca9"}, + "makeup_elixir": {:hex, :makeup_elixir, "0.15.1", "b5888c880d17d1cc3e598f05cdb5b5a91b7b17ac4eaf5f297cb697663a1094dd", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}, {:nimble_parsec, "~> 1.1", [hex: :nimble_parsec, repo: "hexpm", optional: false]}], "hexpm", "db68c173234b07ab2a07f645a5acdc117b9f99d69ebf521821d89690ae6c6ec8"}, + "makeup_erlang": {:hex, :makeup_erlang, "0.1.1", "3fcb7f09eb9d98dc4d208f49cc955a34218fc41ff6b84df7c75b3e6e533cc65f", [:mix], [{:makeup, "~> 1.0", [hex: :makeup, repo: "hexpm", optional: false]}], "hexpm", "174d0809e98a4ef0b3309256cbf97101c6ec01c4ab0b23e926a9e17df2077cbb"}, "metrics": {:hex, :metrics, "1.0.1", "25f094dea2cda98213cecc3aeff09e940299d950904393b2a29d191c346a8486", [:rebar3], [], "hexpm", "69b09adddc4f74a40716ae54d140f93beb0fb8978d8636eaded0c31b6f099f16"}, "mimerl": {:hex, :mimerl, "1.2.0", "67e2d3f571088d5cfd3e550c383094b47159f3eee8ffa08e64106cdf5e981be3", [:rebar3], [], "hexpm", "f278585650aa581986264638ebf698f8bb19df297f66ad91b18910dfc6e19323"}, - "nimble_parsec": {:hex, :nimble_parsec, "0.5.0", "90e2eca3d0266e5c53f8fbe0079694740b9c91b6747f2b7e3c5d21966bba8300", [:mix], [], "hexpm", "5c040b8469c1ff1b10093d3186e2e10dbe483cd73d79ec017993fb3985b8a9b3"}, + "nimble_parsec": {:hex, :nimble_parsec, "1.1.0", "3a6fca1550363552e54c216debb6a9e95bd8d32348938e13de5eda962c0d7f89", [:mix], [], "hexpm", "08eb32d66b706e913ff748f11694b17981c0b04a33ef470e33e11b3d3ac8f54b"}, "parse_trans": {:hex, :parse_trans, "3.3.0", "09765507a3c7590a784615cfd421d101aec25098d50b89d7aa1d66646bc571c1", [:rebar3], [], "hexpm", "17ef63abde837ad30680ea7f857dd9e7ced9476cdd7b0394432af4bfc241b960"}, "poison": {:hex, :poison, "4.0.1", "bcb755a16fac91cad79bfe9fc3585bb07b9331e50cfe3420a24bcc2d735709ae", [:mix], [], "hexpm", "ba8836feea4b394bb718a161fc59a288fe0109b5006d6bdf97b6badfcf6f0f25"}, "ranch": {:hex, :ranch, "1.7.0", "9583f47160ca62af7f8d5db11454068eaa32b56eeadf984d4f46e61a076df5f2", [:rebar3], [], "hexpm", "59f7501c3a56125b2fc5684c3048fac9d043c0bf4d173941b12ca927949af189"},